From 2d5fb60abfb731bbd8a29a11a8102ff8964fffbc Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Thu, 23 Jan 2025 15:03:57 -0500 Subject: [PATCH 01/12] DOC v25.04 Updates [skip ci] --- .../cuda11.8-conda/devcontainer.json | 6 ++--- .devcontainer/cuda11.8-pip/devcontainer.json | 6 ++--- .../cuda12.5-conda/devcontainer.json | 6 ++--- .devcontainer/cuda12.5-pip/devcontainer.json | 6 ++--- .github/workflows/build.yaml | 16 ++++++------ .github/workflows/pr.yaml | 26 +++++++++---------- .github/workflows/test.yaml | 6 ++--- .../trigger-breaking-change-alert.yaml | 2 +- VERSION | 2 +- .../all_cuda-118_arch-aarch64.yaml | 2 +- .../all_cuda-118_arch-x86_64.yaml | 2 +- .../all_cuda-125_arch-aarch64.yaml | 2 +- .../all_cuda-125_arch-x86_64.yaml | 2 +- .../downstream/cmake/get_kvikio.cmake | 4 +-- dependencies.yaml | 22 ++++++++-------- java/pom.xml | 2 +- python/kvikio/pyproject.toml | 6 ++--- 17 files changed, 59 insertions(+), 59 deletions(-) diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index 3bbc6a7552..5fb2ff006d 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:25.02-cpp-cuda11.8-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.04-cpp-cuda11.8-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.02-cuda11.8-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.04-cuda11.8-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.2": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.4": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index e8fedafd2a..4a31d4b223 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:25.02-cpp-cuda11.8-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.04-cpp-cuda11.8-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.02-cuda11.8-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.04-cuda11.8-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.2": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.4": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.5-conda/devcontainer.json b/.devcontainer/cuda12.5-conda/devcontainer.json index 6bd78e8168..8124fbf31e 100644 --- a/.devcontainer/cuda12.5-conda/devcontainer.json +++ b/.devcontainer/cuda12.5-conda/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:25.02-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.04-cpp-mambaforge-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.02-cuda12.5-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.04-cuda12.5-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.2": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.4": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.5-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json index 3f5cd72a82..d1496c2f51 100644 --- a/.devcontainer/cuda12.5-pip/devcontainer.json +++ b/.devcontainer/cuda12.5-pip/devcontainer.json @@ -5,17 +5,17 @@ "args": { "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:25.02-cpp-cuda12.5-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.04-cpp-cuda12.5-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.02-cuda12.5-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.04-cuda12.5-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.2": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.4": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2568491bf1..48ed0dd3ae 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -56,7 +56,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -68,7 +68,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cpp: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} @@ -79,7 +79,7 @@ jobs: wheel-build-python: needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: wheel-publish-cpp: needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -100,7 +100,7 @@ jobs: wheel-publish-python: needs: wheel-build-python secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2275319acf..d0b800f63b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -26,7 +26,7 @@ jobs: - wheel-python-build - wheel-python-tests secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.04 if: always() with: needs: ${{ toJSON(needs) }} @@ -44,7 +44,7 @@ jobs: repo: kvikio changed-files: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.04 with: files_yaml: | test_cpp: @@ -72,24 +72,24 @@ jobs: - '!notebooks/**' checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.04 conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.04 with: build_type: pull-request conda-cpp-tests: needs: [conda-cpp-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.04 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request conda-java-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -99,20 +99,20 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.04 with: build_type: pull-request conda-python-tests: needs: [conda-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.04 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -121,7 +121,7 @@ jobs: run_script: "ci/build_docs.sh" devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.04 with: arch: '["amd64"]' cuda: '["12.5"]' @@ -132,7 +132,7 @@ jobs: sccache -s; wheel-cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request @@ -140,14 +140,14 @@ jobs: wheel-python-build: needs: wheel-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 with: build_type: pull-request script: ci/build_wheel_python.sh wheel-python-tests: needs: [wheel-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.04 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 989af1bf9f..d29ec0f11e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -24,7 +24,7 @@ jobs: sha: ${{ inputs.sha }} python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -32,7 +32,7 @@ jobs: sha: ${{ inputs.sha }} conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/.github/workflows/trigger-breaking-change-alert.yaml b/.github/workflows/trigger-breaking-change-alert.yaml index 01dd2436be..9764c62c15 100644 --- a/.github/workflows/trigger-breaking-change-alert.yaml +++ b/.github/workflows/trigger-breaking-change-alert.yaml @@ -12,7 +12,7 @@ jobs: trigger-notifier: if: contains(github.event.pull_request.labels.*.name, 'breaking') secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-25.04 with: sender_login: ${{ github.event.sender.login }} sender_avatar: ${{ github.event.sender.avatar_url }} diff --git a/VERSION b/VERSION index 72eefaf7c7..b922658ff3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -25.02.00 +25.04.00 diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index a273b0f84d..8b2912243e 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -32,7 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==25.2.*,>=0.0.0a0 +- rapids-dask-dependency==25.4.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 0cad725d17..78d51844bc 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -34,7 +34,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==25.2.*,>=0.0.0a0 +- rapids-dask-dependency==25.4.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 69654e0052..39dbe4a6c0 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -32,7 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==25.2.*,>=0.0.0a0 +- rapids-dask-dependency==25.4.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index d6a9cf88ca..680ca81703 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -32,7 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 -- rapids-dask-dependency==25.2.*,>=0.0.0a0 +- rapids-dask-dependency==25.4.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/cpp/examples/downstream/cmake/get_kvikio.cmake b/cpp/examples/downstream/cmake/get_kvikio.cmake index 45fce36bff..6f20d60afd 100644 --- a/cpp/examples/downstream/cmake/get_kvikio.cmake +++ b/cpp/examples/downstream/cmake/get_kvikio.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -30,4 +30,4 @@ function(find_and_configure_kvikio MIN_VERSION) endfunction() -find_and_configure_kvikio("25.02") +find_and_configure_kvikio("25.04") diff --git a/dependencies.yaml b/dependencies.yaml index 41f9adc26f..40452e4b51 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -170,7 +170,7 @@ dependencies: common: - output_types: conda packages: &libkvikio_packages - - libkvikio==25.2.*,>=0.0.0a0 + - libkvikio==25.4.*,>=0.0.0a0 specific: - output_types: [requirements, pyproject] matrices: @@ -178,12 +178,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - libkvikio-cu12==25.2.*,>=0.0.0a0 + - libkvikio-cu12==25.4.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - libkvikio-cu11==25.2.*,>=0.0.0a0 + - libkvikio-cu11==25.4.*,>=0.0.0a0 - {matrix: null, packages: *libkvikio_packages} build-py-wrapper: common: @@ -321,7 +321,7 @@ dependencies: common: - output_types: conda packages: - - &libkvikio_unsuffixed libkvikio==25.2.*,>=0.0.0a0 + - &libkvikio_unsuffixed libkvikio==25.4.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -335,12 +335,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - libkvikio-cu12==25.2.*,>=0.0.0a0 + - libkvikio-cu12==25.4.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - libkvikio-cu11==25.2.*,>=0.0.0a0 + - libkvikio-cu11==25.4.*,>=0.0.0a0 - {matrix: null, packages: [*libkvikio_unsuffixed]} docs: common: @@ -396,14 +396,14 @@ dependencies: common: - output_types: conda packages: - - libkvikio==25.2.*,>=0.0.0a0 - - libkvikio-tests==25.2.*,>=0.0.0a0 + - libkvikio==25.4.*,>=0.0.0a0 + - libkvikio-tests==25.4.*,>=0.0.0a0 test_kvikio: common: - output_types: conda packages: - - libkvikio==25.2.*,>=0.0.0a0 - - kvikio==25.2.*,>=0.0.0a0 + - libkvikio==25.4.*,>=0.0.0a0 + - kvikio==25.4.*,>=0.0.0a0 test_cpp: common: - output_types: conda @@ -413,7 +413,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - rapids-dask-dependency==25.2.*,>=0.0.0a0 + - rapids-dask-dependency==25.4.*,>=0.0.0a0 - pytest - pytest-cov - rangehttpserver diff --git a/java/pom.xml b/java/pom.xml index a9a2506edd..ce4c2e8bb5 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -7,7 +7,7 @@ ai.rapids.kvikio cufile - 25.02.0-SNAPSHOT + 25.04.0-SNAPSHOT cuFile diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index a1840f2d03..8f5138cb71 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -20,7 +20,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ "cupy-cuda11x>=12.0.0", - "libkvikio==25.2.*,>=0.0.0a0", + "libkvikio==25.4.*,>=0.0.0a0", "numcodecs !=0.12.0", "numpy>=1.23,<3.0a0", "nvidia-nvcomp==4.1.0.6", @@ -46,7 +46,7 @@ test = [ "pytest", "pytest-cov", "rangehttpserver", - "rapids-dask-dependency==25.2.*,>=0.0.0a0", + "rapids-dask-dependency==25.4.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] @@ -119,7 +119,7 @@ matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.0", - "libkvikio==25.2.*,>=0.0.0a0", + "libkvikio==25.4.*,>=0.0.0a0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From ff6f84178cef25a050bd72f810e5ad022d3340fb Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Thu, 30 Jan 2025 02:35:19 -0500 Subject: [PATCH 02/12] Apply east const using clang-format. Fix header issues reported by clang-tidy (#594) This small PR applies east const style using `clang-format`. This makes KvikIO consistent with cuDF in coding style. The following parameters were applied to `.clang-format` for auto-reformatting. The file `.clang-format` itself is not updated in this PR. ``` QualifierAlignment: Custom QualifierOrder: [inline, static, type, const, volatile] ``` In addition, this PR fixes minor "missing header" issues reported by `clang-tidy`. Authors: - Tianyu Liu (https://github.com/kingcrimsontianyu) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Mads R. B. Kristensen (https://github.com/madsbk) URL: https://github.com/rapidsai/kvikio/pull/594 --- cpp/include/kvikio/batch.hpp | 6 +++--- cpp/include/kvikio/buffer.hpp | 12 +++++------ cpp/include/kvikio/cufile/config.hpp | 2 +- cpp/include/kvikio/defaults.hpp | 2 +- cpp/include/kvikio/error.hpp | 10 ++++----- cpp/include/kvikio/file_handle.hpp | 10 ++++----- cpp/include/kvikio/posix_io.hpp | 20 +++++++++--------- cpp/include/kvikio/shim/cufile_h_wrapper.hpp | 4 ++-- cpp/include/kvikio/shim/libcurl.hpp | 1 + cpp/include/kvikio/shim/utils.hpp | 9 ++++---- cpp/include/kvikio/stream.hpp | 2 +- cpp/include/kvikio/utils.hpp | 14 ++++++------- cpp/src/batch.cpp | 6 +++--- cpp/src/buffer.cpp | 10 ++++----- cpp/src/cufile/config.cpp | 8 +++---- cpp/src/cufile/driver.cpp | 2 +- cpp/src/defaults.cpp | 12 +++++------ cpp/src/file_handle.cpp | 22 ++++++++++---------- cpp/src/posix_io.cpp | 4 ++-- cpp/src/shim/cuda.cpp | 2 +- cpp/src/shim/utils.cpp | 10 ++++----- cpp/src/stream.cpp | 2 +- cpp/src/utils.cpp | 14 ++++++------- cpp/tests/test_basic_io.cpp | 4 ++-- cpp/tests/test_defaults.cpp | 10 ++++----- cpp/tests/utils.hpp | 18 ++++++++-------- java/src/main/native/src/CuFileJni.cpp | 4 ++-- 27 files changed, 111 insertions(+), 109 deletions(-) diff --git a/cpp/include/kvikio/batch.hpp b/cpp/include/kvikio/batch.hpp index 9927962a65..40168ffbc3 100644 --- a/cpp/include/kvikio/batch.hpp +++ b/cpp/include/kvikio/batch.hpp @@ -78,7 +78,7 @@ class BatchHandle { /** * @brief BatchHandle support move semantic but isn't copyable */ - BatchHandle(const BatchHandle&) = delete; + BatchHandle(BatchHandle const&) = delete; BatchHandle& operator=(BatchHandle const&) = delete; BatchHandle(BatchHandle&& o) noexcept; ~BatchHandle() noexcept; @@ -96,7 +96,7 @@ class BatchHandle { * @param operations The vector of batch operations, which must not exceed the * `max_num_events`. */ - void submit(const std::vector& operations); + void submit(std::vector const& operations); /** * @brief Get status of submitted operations @@ -127,7 +127,7 @@ class BatchHandle { void close() noexcept; - void submit(const std::vector& operations); + void submit(std::vector const& operations); std::vector status(unsigned min_nr, unsigned max_nr, diff --git a/cpp/include/kvikio/buffer.hpp b/cpp/include/kvikio/buffer.hpp index 9cef45a6f0..ad026cbf38 100644 --- a/cpp/include/kvikio/buffer.hpp +++ b/cpp/include/kvikio/buffer.hpp @@ -35,17 +35,17 @@ namespace kvikio { * streaming buffer that is reused across multiple cuFile IO operations. */ /*NOLINTNEXTLINE(readability-function-cognitive-complexity)*/ -void buffer_register(const void* devPtr_base, +void buffer_register(void const* devPtr_base, std::size_t size, int flags = 0, - const std::vector& errors_to_ignore = std::vector()); + std::vector const& errors_to_ignore = std::vector()); /** * @brief deregister an already registered device memory from cuFile * * @param devPtr_base device pointer to deregister */ -void buffer_deregister(const void* devPtr_base); +void buffer_deregister(void const* devPtr_base); /** * @brief Register device memory allocation which is part of devPtr. Use this @@ -61,15 +61,15 @@ void buffer_deregister(const void* devPtr_base); * @warning This API is intended for usecases where the memory is used as * streaming buffer that is reused across multiple cuFile IO operations. */ -void memory_register(const void* devPtr, +void memory_register(void const* devPtr, int flags = 0, - const std::vector& errors_to_ignore = {}); + std::vector const& errors_to_ignore = {}); /** * @brief deregister an already registered device memory from cuFile. * * @param devPtr device pointer to deregister */ -void memory_deregister(const void* devPtr); +void memory_deregister(void const* devPtr); } // namespace kvikio diff --git a/cpp/include/kvikio/cufile/config.hpp b/cpp/include/kvikio/cufile/config.hpp index 5d48106c9b..fd721e5fe4 100644 --- a/cpp/include/kvikio/cufile/config.hpp +++ b/cpp/include/kvikio/cufile/config.hpp @@ -26,6 +26,6 @@ namespace kvikio { * * @return The filepath to the cufile.json file or the empty string if it isn't found. */ -[[nodiscard]] KVIKIO_EXPORT const std::string& config_path(); +[[nodiscard]] KVIKIO_EXPORT std::string const& config_path(); } // namespace kvikio diff --git a/cpp/include/kvikio/defaults.hpp b/cpp/include/kvikio/defaults.hpp index 6b866827a7..501c71981a 100644 --- a/cpp/include/kvikio/defaults.hpp +++ b/cpp/include/kvikio/defaults.hpp @@ -62,7 +62,7 @@ CompatMode parse_compat_mode_str(std::string_view compat_mode_str); template T getenv_or(std::string_view env_var_name, T default_val) { - const auto* env_val = std::getenv(env_var_name.data()); + auto const* env_val = std::getenv(env_var_name.data()); if (env_val == nullptr) { return default_val; } std::stringstream sstream(env_val); diff --git a/cpp/include/kvikio/error.hpp b/cpp/include/kvikio/error.hpp index 33a4730b79..25bf62a877 100644 --- a/cpp/include/kvikio/error.hpp +++ b/cpp/include/kvikio/error.hpp @@ -67,7 +67,7 @@ struct CUfileException : public std::runtime_error { namespace detail { template -void cuda_driver_try_2(CUresult error, int line_number, const char* filename) +void cuda_driver_try_2(CUresult error, int line_number, char const* filename) { if (error == CUDA_ERROR_STUB_LIBRARY) { throw Exception{std::string{"CUDA error at: "} + std::string(filename) + ":" + @@ -76,8 +76,8 @@ void cuda_driver_try_2(CUresult error, int line_number, const char* filename) "The CUDA driver loaded is a stub library)"}; } if (error != CUDA_SUCCESS) { - const char* err_name = nullptr; - const char* err_str = nullptr; + char const* err_name = nullptr; + char const* err_str = nullptr; CUresult err_name_status = cudaAPI::instance().GetErrorName(error, &err_name); CUresult err_str_status = cudaAPI::instance().GetErrorString(error, &err_str); if (err_name_status == CUDA_ERROR_INVALID_VALUE) { err_name = "unknown"; } @@ -89,7 +89,7 @@ void cuda_driver_try_2(CUresult error, int line_number, const char* filename) } template -void cufile_try_2(CUfileError_t error, int line_number, const char* filename) +void cufile_try_2(CUfileError_t error, int line_number, char const* filename) { if (error.err != CU_FILE_SUCCESS) { if (error.err == CU_FILE_CUDA_DRIVER_ERROR) { @@ -103,7 +103,7 @@ void cufile_try_2(CUfileError_t error, int line_number, const char* filename) } template -void cufile_check_bytes_done_2(ssize_t nbytes_done, int line_number, const char* filename) +void cufile_check_bytes_done_2(ssize_t nbytes_done, int line_number, char const* filename) { if (nbytes_done < 0) { auto const err = std::abs(nbytes_done); diff --git a/cpp/include/kvikio/file_handle.hpp b/cpp/include/kvikio/file_handle.hpp index d4d9e6fa66..e32136e468 100644 --- a/cpp/include/kvikio/file_handle.hpp +++ b/cpp/include/kvikio/file_handle.hpp @@ -82,15 +82,15 @@ class FileHandle { * @param mode Access modes (see `open(2)`). * @param compat_mode Set KvikIO's compatibility mode for this file. */ - FileHandle(const std::string& file_path, - const std::string& flags = "r", + FileHandle(std::string const& file_path, + std::string const& flags = "r", mode_t mode = m644, CompatMode compat_mode = defaults::compat_mode()); /** * @brief FileHandle support move semantic but isn't copyable */ - FileHandle(const FileHandle&) = delete; + FileHandle(FileHandle const&) = delete; FileHandle& operator=(FileHandle const&) = delete; FileHandle(FileHandle&& o) noexcept; FileHandle& operator=(FileHandle&& o) noexcept; @@ -216,7 +216,7 @@ class FileHandle { * case, the value of `sync_default_stream` is ignored. * @return Size of bytes that were successfully written. */ - std::size_t write(const void* devPtr_base, + std::size_t write(void const* devPtr_base, std::size_t size, std::size_t file_offset, std::size_t devPtr_offset, @@ -289,7 +289,7 @@ class FileHandle { * @note The `std::future` object's `wait()` or `get()` should not be called after the lifetime of * the FileHandle object ends. Otherwise, the behavior is undefined. */ - std::future pwrite(const void* buf, + std::future pwrite(void const* buf, std::size_t size, std::size_t file_offset = 0, std::size_t task_size = defaults::task_size(), diff --git a/cpp/include/kvikio/posix_io.hpp b/cpp/include/kvikio/posix_io.hpp index 7675285d09..99964315b3 100644 --- a/cpp/include/kvikio/posix_io.hpp +++ b/cpp/include/kvikio/posix_io.hpp @@ -69,7 +69,7 @@ class StreamsByThread { static CUstream get(); - StreamsByThread(const StreamsByThread&) = delete; + StreamsByThread(StreamsByThread const&) = delete; StreamsByThread& operator=(StreamsByThread const&) = delete; StreamsByThread(StreamsByThread&& o) = delete; StreamsByThread& operator=(StreamsByThread&& o) = delete; @@ -88,11 +88,11 @@ class StreamsByThread { * @return The number of bytes read or written (always gather than zero) */ template -ssize_t posix_host_io(int fd, const void* buf, size_t count, off_t offset) +ssize_t posix_host_io(int fd, void const* buf, size_t count, off_t offset) { off_t cur_offset = offset; size_t byte_remaining = count; - char* buffer = const_cast(static_cast(buf)); + char* buffer = const_cast(static_cast(buf)); while (byte_remaining > 0) { ssize_t nbytes = 0; if constexpr (Operation == IOOperationType::READ) { @@ -101,7 +101,7 @@ ssize_t posix_host_io(int fd, const void* buf, size_t count, off_t offset) nbytes = ::pwrite(fd, buffer, byte_remaining, cur_offset); } if (nbytes == -1) { - const std::string name = Operation == IOOperationType::READ ? "pread" : "pwrite"; + std::string const name = Operation == IOOperationType::READ ? "pread" : "pwrite"; if (errno == EBADF) { throw CUfileException{std::string{"POSIX error on " + name + " at: "} + __FILE__ + ":" + KVIKIO_STRINGIFY(__LINE__) + ": Operation not permitted"}; @@ -136,7 +136,7 @@ ssize_t posix_host_io(int fd, const void* buf, size_t count, off_t offset) */ template std::size_t posix_device_io(int fd, - const void* devPtr_base, + void const* devPtr_base, std::size_t size, std::size_t file_offset, std::size_t devPtr_offset) @@ -145,13 +145,13 @@ std::size_t posix_device_io(int fd, CUdeviceptr devPtr = convert_void2deviceptr(devPtr_base) + devPtr_offset; off_t cur_file_offset = convert_size2off(file_offset); off_t byte_remaining = convert_size2off(size); - const off_t chunk_size2 = convert_size2off(alloc.size()); + off_t const chunk_size2 = convert_size2off(alloc.size()); // Get a stream for the current CUDA context and thread CUstream stream = StreamsByThread::get(); while (byte_remaining > 0) { - const off_t nbytes_requested = std::min(chunk_size2, byte_remaining); + off_t const nbytes_requested = std::min(chunk_size2, byte_remaining); ssize_t nbytes_got = nbytes_requested; if constexpr (Operation == IOOperationType::READ) { nbytes_got = posix_host_io( @@ -209,7 +209,7 @@ std::size_t posix_host_read(int fd, void* buf, std::size_t size, std::size_t fil * @return Size of bytes that were successfully read. */ template -std::size_t posix_host_write(int fd, const void* buf, std::size_t size, std::size_t file_offset) +std::size_t posix_host_write(int fd, void const* buf, std::size_t size, std::size_t file_offset) { KVIKIO_NVTX_SCOPED_RANGE("posix_host_write()", size); return detail::posix_host_io( @@ -230,7 +230,7 @@ std::size_t posix_host_write(int fd, const void* buf, std::size_t size, std::siz * @return Size of bytes that were successfully read. */ std::size_t posix_device_read(int fd, - const void* devPtr_base, + void const* devPtr_base, std::size_t size, std::size_t file_offset, std::size_t devPtr_offset); @@ -249,7 +249,7 @@ std::size_t posix_device_read(int fd, * @return Size of bytes that were successfully written. */ std::size_t posix_device_write(int fd, - const void* devPtr_base, + void const* devPtr_base, std::size_t size, std::size_t file_offset, std::size_t devPtr_offset); diff --git a/cpp/include/kvikio/shim/cufile_h_wrapper.hpp b/cpp/include/kvikio/shim/cufile_h_wrapper.hpp index 1c13d2d8a1..66f5adbaf3 100644 --- a/cpp/include/kvikio/shim/cufile_h_wrapper.hpp +++ b/cpp/include/kvikio/shim/cufile_h_wrapper.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. + * Copyright (c) 2022-2025, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -59,7 +59,7 @@ struct CUfileDescr_t { } handle; }; -static inline const char* cufileop_status_error(CUfileOpError err) { return CUFILE_ERRSTR(err); }; +inline static char const* cufileop_status_error(CUfileOpError err) { return CUFILE_ERRSTR(err); }; CUfileError_t cuFileHandleRegister(...); CUfileError_t cuFileHandleDeregister(...); ssize_t cuFileRead(...); diff --git a/cpp/include/kvikio/shim/libcurl.hpp b/cpp/include/kvikio/shim/libcurl.hpp index a4efab02e5..ccff6d81b8 100644 --- a/cpp/include/kvikio/shim/libcurl.hpp +++ b/cpp/include/kvikio/shim/libcurl.hpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include diff --git a/cpp/include/kvikio/shim/utils.hpp b/cpp/include/kvikio/shim/utils.hpp index 52dfea42e8..f23841efe1 100644 --- a/cpp/include/kvikio/shim/utils.hpp +++ b/cpp/include/kvikio/shim/utils.hpp @@ -17,6 +17,7 @@ #include #include +#include #include namespace kvikio { @@ -44,7 +45,7 @@ namespace kvikio { * @param name Name of the library to load. * @return The library handle. */ -void* load_library(const char* name, int mode = RTLD_LAZY | RTLD_LOCAL | RTLD_NODELETE); +void* load_library(char const* name, int mode = RTLD_LAZY | RTLD_LOCAL | RTLD_NODELETE); /** * @brief Load shared library @@ -52,7 +53,7 @@ void* load_library(const char* name, int mode = RTLD_LAZY | RTLD_LOCAL | RTLD_NO * @param names Vector of names to try when loading shared library. * @return The library handle. */ -void* load_library(const std::vector& names, +void* load_library(std::vector const& names, int mode = RTLD_LAZY | RTLD_LOCAL | RTLD_NODELETE); /** @@ -64,12 +65,12 @@ void* load_library(const std::vector& names, * @param name Name of the symbol/function to load. */ template -void get_symbol(T& handle, void* lib, const char* name) +void get_symbol(T& handle, void* lib, char const* name) { ::dlerror(); // Clear old errors // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) handle = reinterpret_cast(::dlsym(lib, name)); - const char* err = ::dlerror(); + char const* err = ::dlerror(); if (err != nullptr) { throw std::runtime_error(err); } } diff --git a/cpp/include/kvikio/stream.hpp b/cpp/include/kvikio/stream.hpp index 4b3e37a980..f48f9955f7 100644 --- a/cpp/include/kvikio/stream.hpp +++ b/cpp/include/kvikio/stream.hpp @@ -66,7 +66,7 @@ class StreamFuture { /** * @brief StreamFuture support move semantic but isn't copyable */ - StreamFuture(const StreamFuture&) = delete; + StreamFuture(StreamFuture const&) = delete; StreamFuture& operator=(StreamFuture& o) = delete; StreamFuture(StreamFuture&& o) noexcept; StreamFuture& operator=(StreamFuture&& o) noexcept; diff --git a/cpp/include/kvikio/utils.hpp b/cpp/include/kvikio/utils.hpp index 5bea6f17c7..b10e54c482 100644 --- a/cpp/include/kvikio/utils.hpp +++ b/cpp/include/kvikio/utils.hpp @@ -38,7 +38,7 @@ inline constexpr std::size_t page_size = 4096; [[nodiscard]] ssize_t convert_size2ssize(std::size_t x); -[[nodiscard]] CUdeviceptr convert_void2deviceptr(const void* devPtr); +[[nodiscard]] CUdeviceptr convert_void2deviceptr(void const* devPtr); /** * @brief Help function to convert value to 64 bit signed integer @@ -72,9 +72,9 @@ template >* = nullptr> * @return The boolean answer */ #ifdef KVIKIO_CUDA_FOUND -bool is_host_memory(const void* ptr); +bool is_host_memory(void const* ptr); #else -constexpr bool is_host_memory(const void* ptr) { return true; } +constexpr bool is_host_memory(void const* ptr) { return true; } #endif /** @@ -127,7 +127,7 @@ constexpr bool is_host_memory(const void* ptr) { return true; } * @param devPtr Device pointer to query * @return Usable CUDA context */ -[[nodiscard]] CUcontext get_context_from_pointer(const void* devPtr); +[[nodiscard]] CUcontext get_context_from_pointer(void const* devPtr); /** * @brief Push CUDA context on creation and pop it on destruction @@ -138,7 +138,7 @@ class PushAndPopContext { public: PushAndPopContext(CUcontext ctx); - PushAndPopContext(const PushAndPopContext&) = delete; + PushAndPopContext(PushAndPopContext const&) = delete; PushAndPopContext& operator=(PushAndPopContext const&) = delete; PushAndPopContext(PushAndPopContext&&) = delete; PushAndPopContext&& operator=(PushAndPopContext&&) = delete; @@ -146,11 +146,11 @@ class PushAndPopContext { }; // Find the base and offset of the memory allocation `devPtr` is in -std::tuple get_alloc_info(const void* devPtr, +std::tuple get_alloc_info(void const* devPtr, CUcontext* ctx = nullptr); template -bool is_future_done(const T& future) +bool is_future_done(T const& future) { return future.wait_for(std::chrono::seconds(0)) != std::future_status::timeout; } diff --git a/cpp/src/batch.cpp b/cpp/src/batch.cpp index 8ced70cbd8..128c2d5953 100644 --- a/cpp/src/batch.cpp +++ b/cpp/src/batch.cpp @@ -52,14 +52,14 @@ void BatchHandle::close() noexcept cuFileAPI::instance().BatchIODestroy(_handle); } -void BatchHandle::submit(const std::vector& operations) +void BatchHandle::submit(std::vector const& operations) { if (convert_size2ssize(operations.size()) > _max_num_events) { throw CUfileException("Cannot submit more than the max_num_events)"); } std::vector io_batch_params; io_batch_params.reserve(operations.size()); - for (const auto& op : operations) { + for (auto const& op : operations) { if (op.file_handle.is_compat_mode_preferred()) { throw CUfileException("Cannot submit a FileHandle opened in compatibility mode"); } @@ -102,7 +102,7 @@ bool BatchHandle::closed() const noexcept { return true; } void BatchHandle::close() noexcept {} -void BatchHandle::submit(const std::vector& operations) {} +void BatchHandle::submit(std::vector const& operations) {} std::vector BatchHandle::status(unsigned min_nr, unsigned max_nr, diff --git a/cpp/src/buffer.cpp b/cpp/src/buffer.cpp index 0aa772d50f..61e4e98dec 100644 --- a/cpp/src/buffer.cpp +++ b/cpp/src/buffer.cpp @@ -27,10 +27,10 @@ namespace kvikio { -void buffer_register(const void* devPtr_base, +void buffer_register(void const* devPtr_base, std::size_t size, int flags, - const std::vector& errors_to_ignore) + std::vector const& errors_to_ignore) { if (defaults::is_compat_mode_preferred()) { return; } CUfileError_t status = cuFileAPI::instance().BufRegister(devPtr_base, size, flags); @@ -43,19 +43,19 @@ void buffer_register(const void* devPtr_base, } } -void buffer_deregister(const void* devPtr_base) +void buffer_deregister(void const* devPtr_base) { if (defaults::is_compat_mode_preferred()) { return; } CUFILE_TRY(cuFileAPI::instance().BufDeregister(devPtr_base)); } -void memory_register(const void* devPtr, int flags, const std::vector& errors_to_ignore) +void memory_register(void const* devPtr, int flags, std::vector const& errors_to_ignore) { auto [base, nbytes, offset] = get_alloc_info(devPtr); buffer_register(base, nbytes, flags, errors_to_ignore); } -void memory_deregister(const void* devPtr) +void memory_deregister(void const* devPtr) { auto [base, nbytes, offset] = get_alloc_info(devPtr); buffer_deregister(base); diff --git a/cpp/src/cufile/config.cpp b/cpp/src/cufile/config.cpp index b27475b8da..d1d36adc04 100644 --- a/cpp/src/cufile/config.cpp +++ b/cpp/src/cufile/config.cpp @@ -23,9 +23,9 @@ namespace kvikio { namespace { -[[nodiscard]] const char* lookup_config_path() +[[nodiscard]] char const* lookup_config_path() { - const char* env = std::getenv("CUFILE_ENV_PATH_JSON"); + char const* env = std::getenv("CUFILE_ENV_PATH_JSON"); if (env != nullptr && std::filesystem::exists(env)) { return env; } if (std::filesystem::exists("/etc/cufile.json")) { return "/etc/cufile.json"; } return ""; @@ -33,9 +33,9 @@ namespace { } // namespace -const std::string& config_path() +std::string const& config_path() { - static const std::string ret = lookup_config_path(); + static std::string const ret = lookup_config_path(); return ret; } diff --git a/cpp/src/cufile/driver.cpp b/cpp/src/cufile/driver.cpp index ffee213c00..aa8af0304f 100644 --- a/cpp/src/cufile/driver.cpp +++ b/cpp/src/cufile/driver.cpp @@ -48,7 +48,7 @@ DriverInitializer::~DriverInitializer() noexcept { try { cuFileAPI::instance().driver_close(); - } catch (const CUfileException& e) { + } catch (CUfileException const& e) { std::cerr << "Unable to close GDS file driver: "; std::cerr << e.what(); std::cerr << std::endl; diff --git a/cpp/src/defaults.cpp b/cpp/src/defaults.cpp index 4b431e9591..f005e86d0b 100644 --- a/cpp/src/defaults.cpp +++ b/cpp/src/defaults.cpp @@ -54,12 +54,12 @@ CompatMode parse_compat_mode_str(std::string_view compat_mode_str) template <> bool getenv_or(std::string_view env_var_name, bool default_val) { - const auto* env_val = std::getenv(env_var_name.data()); + auto const* env_val = std::getenv(env_var_name.data()); if (env_val == nullptr) { return default_val; } try { // Try parsing `env_var_name` as a integer return static_cast(std::stoi(env_val)); - } catch (const std::invalid_argument&) { + } catch (std::invalid_argument const&) { } // Convert to lowercase std::string str{env_val}; @@ -93,7 +93,7 @@ CompatMode getenv_or(std::string_view env_var_name, CompatMode default_val) unsigned int defaults::get_num_threads_from_env() { - const int ret = getenv_or("KVIKIO_NTHREADS", 1); + int const ret = getenv_or("KVIKIO_NTHREADS", 1); if (ret <= 0) { throw std::invalid_argument("KVIKIO_NTHREADS has to be a positive integer greater than zero"); } @@ -108,7 +108,7 @@ defaults::defaults() } // Determine the default value of `task_size` { - const ssize_t env = getenv_or("KVIKIO_TASK_SIZE", 4 * 1024 * 1024); + ssize_t const env = getenv_or("KVIKIO_TASK_SIZE", 4 * 1024 * 1024); if (env <= 0) { throw std::invalid_argument( "KVIKIO_TASK_SIZE has to be a positive integer greater than zero"); @@ -117,7 +117,7 @@ defaults::defaults() } // Determine the default value of `gds_threshold` { - const ssize_t env = getenv_or("KVIKIO_GDS_THRESHOLD", 1024 * 1024); + ssize_t const env = getenv_or("KVIKIO_GDS_THRESHOLD", 1024 * 1024); if (env < 0) { throw std::invalid_argument("KVIKIO_GDS_THRESHOLD has to be a positive integer"); } @@ -125,7 +125,7 @@ defaults::defaults() } // Determine the default value of `bounce_buffer_size` { - const ssize_t env = getenv_or("KVIKIO_BOUNCE_BUFFER_SIZE", 16 * 1024 * 1024); + ssize_t const env = getenv_or("KVIKIO_BOUNCE_BUFFER_SIZE", 16 * 1024 * 1024); if (env <= 0) { throw std::invalid_argument( "KVIKIO_BOUNCE_BUFFER_SIZE has to be a positive integer greater than zero"); diff --git a/cpp/src/file_handle.cpp b/cpp/src/file_handle.cpp index 9c025c8d45..d6b96f3d6d 100644 --- a/cpp/src/file_handle.cpp +++ b/cpp/src/file_handle.cpp @@ -40,7 +40,7 @@ namespace { * @throw std::invalid_argument if the specified flags are not supported. * @throw std::invalid_argument if `o_direct` is true, but `O_DIRECT` is not supported. */ -int open_fd_parse_flags(const std::string& flags, bool o_direct) +int open_fd_parse_flags(std::string const& flags, bool o_direct) { int file_flags = -1; if (flags.empty()) { throw std::invalid_argument("Unknown file open flag"); } @@ -76,7 +76,7 @@ int open_fd_parse_flags(const std::string& flags, bool o_direct) * @param mode Access modes * @return File descriptor */ -int open_fd(const std::string& file_path, const std::string& flags, bool o_direct, mode_t mode) +int open_fd(std::string const& file_path, std::string const& flags, bool o_direct, mode_t mode) { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) int fd = ::open(file_path.c_str(), open_fd_parse_flags(flags, o_direct), mode); @@ -116,8 +116,8 @@ int open_fd(const std::string& file_path, const std::string& flags, bool o_direc } // namespace -FileHandle::FileHandle(const std::string& file_path, - const std::string& flags, +FileHandle::FileHandle(std::string const& file_path, + std::string const& flags, mode_t mode, CompatMode compat_mode) : _fd_direct_off{open_fd(file_path, flags, false, mode)}, @@ -139,9 +139,9 @@ FileHandle::FileHandle(const std::string& file_path, try { _fd_direct_on = open_fd(file_path, flags, true, mode); - } catch (const std::system_error&) { + } catch (std::system_error const&) { handle_o_direct_except(); - } catch (const std::invalid_argument&) { + } catch (std::invalid_argument const&) { handle_o_direct_except(); } @@ -242,7 +242,7 @@ std::size_t FileHandle::read(void* devPtr_base, return ret; } -std::size_t FileHandle::write(const void* devPtr_base, +std::size_t FileHandle::write(void const* devPtr_base, std::size_t size, std::size_t file_offset, std::size_t devPtr_offset, @@ -318,7 +318,7 @@ std::future FileHandle::pread(void* buf, return parallel_io(task, devPtr_base, size, file_offset, task_size, devPtr_offset); } -std::future FileHandle::pwrite(const void* buf, +std::future FileHandle::pwrite(void const* buf, std::size_t size, std::size_t file_offset, std::size_t task_size, @@ -327,11 +327,11 @@ std::future FileHandle::pwrite(const void* buf, { KVIKIO_NVTX_MARKER("FileHandle::pwrite()", size); if (is_host_memory(buf)) { - auto op = [this](const void* hostPtr_base, + auto op = [this](void const* hostPtr_base, std::size_t size, std::size_t file_offset, std::size_t hostPtr_offset) -> std::size_t { - const char* buf = static_cast(hostPtr_base) + hostPtr_offset; + char const* buf = static_cast(hostPtr_base) + hostPtr_offset; return detail::posix_host_write( _fd_direct_off, buf, size, file_offset); }; @@ -357,7 +357,7 @@ std::future FileHandle::pwrite(const void* buf, } // Regular case that use the threadpool and run the tasks in parallel - auto op = [this, ctx](const void* devPtr_base, + auto op = [this, ctx](void const* devPtr_base, std::size_t size, std::size_t file_offset, std::size_t devPtr_offset) -> std::size_t { diff --git a/cpp/src/posix_io.cpp b/cpp/src/posix_io.cpp index d4ee2944e5..9576f284dc 100644 --- a/cpp/src/posix_io.cpp +++ b/cpp/src/posix_io.cpp @@ -55,7 +55,7 @@ CUstream StreamsByThread::get() } std::size_t posix_device_read(int fd, - const void* devPtr_base, + void const* devPtr_base, std::size_t size, std::size_t file_offset, std::size_t devPtr_offset) @@ -66,7 +66,7 @@ std::size_t posix_device_read(int fd, } std::size_t posix_device_write(int fd, - const void* devPtr_base, + void const* devPtr_base, std::size_t size, std::size_t file_offset, std::size_t devPtr_offset) diff --git a/cpp/src/shim/cuda.cpp b/cpp/src/shim/cuda.cpp index b6a1b3babf..85b27d4f9f 100644 --- a/cpp/src/shim/cuda.cpp +++ b/cpp/src/shim/cuda.cpp @@ -62,7 +62,7 @@ bool is_cuda_available() { try { cudaAPI::instance(); - } catch (const std::runtime_error&) { + } catch (std::runtime_error const&) { return false; } return true; diff --git a/cpp/src/shim/utils.cpp b/cpp/src/shim/utils.cpp index de7ec6a875..91f4f28d19 100644 --- a/cpp/src/shim/utils.cpp +++ b/cpp/src/shim/utils.cpp @@ -24,7 +24,7 @@ namespace kvikio { -void* load_library(const char* name, int mode) +void* load_library(char const* name, int mode) { ::dlerror(); // Clear old errors void* ret = ::dlopen(name, mode); @@ -32,14 +32,14 @@ void* load_library(const char* name, int mode) return ret; } -void* load_library(const std::vector& names, int mode) +void* load_library(std::vector const& names, int mode) { std::stringstream ss; - for (const char* name : names) { + for (char const* name : names) { ss << name << " "; try { return load_library(name, mode); - } catch (const std::runtime_error&) { + } catch (std::runtime_error const&) { } } throw std::runtime_error("cannot open shared object file, tried: " + ss.str()); @@ -51,7 +51,7 @@ bool is_running_in_wsl() noexcept struct utsname buf {}; int err = ::uname(&buf); if (err == 0) { - const std::string name(static_cast(buf.release)); + std::string const name(static_cast(buf.release)); // 'Microsoft' for WSL1 and 'microsoft' for WSL2 return name.find("icrosoft") != std::string::npos; } diff --git a/cpp/src/stream.cpp b/cpp/src/stream.cpp index ac0f8138e4..40a4d9d26d 100644 --- a/cpp/src/stream.cpp +++ b/cpp/src/stream.cpp @@ -92,7 +92,7 @@ StreamFuture::~StreamFuture() noexcept if (_val != nullptr) { try { check_bytes_done(); - } catch (const kvikio::CUfileException& e) { + } catch (kvikio::CUfileException const& e) { std::cerr << e.what() << std::endl; } std::free(_val); diff --git a/cpp/src/utils.cpp b/cpp/src/utils.cpp index 4ba5a757a2..bed2cbafbc 100644 --- a/cpp/src/utils.cpp +++ b/cpp/src/utils.cpp @@ -49,14 +49,14 @@ ssize_t convert_size2ssize(std::size_t x) return static_cast(x); } -CUdeviceptr convert_void2deviceptr(const void* devPtr) +CUdeviceptr convert_void2deviceptr(void const* devPtr) { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) return reinterpret_cast(devPtr); } #ifdef KVIKIO_CUDA_FOUND -bool is_host_memory(const void* ptr) +bool is_host_memory(void const* ptr) { CUpointer_attribute attrs[1] = { CU_POINTER_ATTRIBUTE_MEMORY_TYPE, @@ -108,7 +108,7 @@ CUcontext get_primary_cuda_context(int ordinal) std::optional get_context_associated_pointer(CUdeviceptr dev_ptr) { CUcontext ctx = nullptr; - const CUresult err = + CUresult const err = cudaAPI::instance().PointerGetAttribute(&ctx, CU_POINTER_ATTRIBUTE_CONTEXT, dev_ptr); if (err == CUDA_SUCCESS && ctx != nullptr) { return ctx; } if (err != CUDA_ERROR_INVALID_VALUE) { CUDA_DRIVER_TRY(err); } @@ -118,14 +118,14 @@ std::optional get_context_associated_pointer(CUdeviceptr dev_ptr) bool current_context_can_access_pointer(CUdeviceptr dev_ptr) { CUdeviceptr current_ctx_dev_ptr{}; - const CUresult err = cudaAPI::instance().PointerGetAttribute( + CUresult const err = cudaAPI::instance().PointerGetAttribute( ¤t_ctx_dev_ptr, CU_POINTER_ATTRIBUTE_DEVICE_POINTER, dev_ptr); if (err == CUDA_SUCCESS && current_ctx_dev_ptr == dev_ptr) { return true; } if (err != CUDA_ERROR_INVALID_VALUE) { CUDA_DRIVER_TRY(err); } return false; } -CUcontext get_context_from_pointer(const void* devPtr) +CUcontext get_context_from_pointer(void const* devPtr) { CUdeviceptr dev_ptr = convert_void2deviceptr(devPtr); @@ -157,12 +157,12 @@ PushAndPopContext::~PushAndPopContext() { try { CUDA_DRIVER_TRY(cudaAPI::instance().CtxPopCurrent(&_ctx), CUfileException); - } catch (const CUfileException& e) { + } catch (CUfileException const& e) { std::cerr << e.what() << std::endl; } } -std::tuple get_alloc_info(const void* devPtr, CUcontext* ctx) +std::tuple get_alloc_info(void const* devPtr, CUcontext* ctx) { auto dev = convert_void2deviceptr(devPtr); CUdeviceptr base_ptr{}; diff --git a/cpp/tests/test_basic_io.cpp b/cpp/tests/test_basic_io.cpp index c884ec6230..e16bd99d83 100644 --- a/cpp/tests/test_basic_io.cpp +++ b/cpp/tests/test_basic_io.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, NVIDIA CORPORATION. + * Copyright (c) 2024-2025, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -77,7 +77,7 @@ TEST_F(BasicIOTest, write_read_async) // Explicitly set compatibility mode std::array compat_modes{kvikio::CompatMode::AUTO, kvikio::CompatMode::ON}; - for (const auto& compat_mode : compat_modes) { + for (auto const& compat_mode : compat_modes) { { kvikio::FileHandle f(_filepath, "w", kvikio::FileHandle::m644, compat_mode); auto stream_future = f.write_async(_dev_a.ptr, _dev_a.nbytes, 0, 0, stream); diff --git a/cpp/tests/test_defaults.cpp b/cpp/tests/test_defaults.cpp index c4a88775e4..c95e9d1d11 100644 --- a/cpp/tests/test_defaults.cpp +++ b/cpp/tests/test_defaults.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, NVIDIA CORPORATION. + * Copyright (c) 2024-2025, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ TEST(Defaults, parse_compat_mode_str) { std::vector inputs{ "ON", "on", "On", "TRUE", "true", "True", "YES", "yes", "Yes", "1"}; - for (const auto& input : inputs) { + for (auto const& input : inputs) { EXPECT_EQ(kvikio::detail::parse_compat_mode_str(input), kvikio::CompatMode::ON); } } @@ -32,21 +32,21 @@ TEST(Defaults, parse_compat_mode_str) { std::vector inputs{ "OFF", "off", "oFf", "FALSE", "false", "False", "NO", "no", "No", "0"}; - for (const auto& input : inputs) { + for (auto const& input : inputs) { EXPECT_EQ(kvikio::detail::parse_compat_mode_str(input), kvikio::CompatMode::OFF); } } { std::vector inputs{"AUTO", "auto", "aUtO"}; - for (const auto& input : inputs) { + for (auto const& input : inputs) { EXPECT_EQ(kvikio::detail::parse_compat_mode_str(input), kvikio::CompatMode::AUTO); } } { std::vector inputs{"", "invalidOption", "11", "*&^Yes"}; - for (const auto& input : inputs) { + for (auto const& input : inputs) { EXPECT_THROW(kvikio::detail::parse_compat_mode_str(input), std::invalid_argument); } } diff --git a/cpp/tests/utils.hpp b/cpp/tests/utils.hpp index 1c671a82bc..5722d3db25 100644 --- a/cpp/tests/utils.hpp +++ b/cpp/tests/utils.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, NVIDIA CORPORATION. + * Copyright (c) 2024-2025, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -72,7 +72,7 @@ namespace kvikio::test { */ class TempDir { public: - TempDir(const bool cleanup = true) : _cleanup{cleanup} + TempDir(bool const cleanup = true) : _cleanup{cleanup} { std::string tpl{std::filesystem::temp_directory_path() / "kvikio.XXXXXX"}; if (mkdtemp(tpl.data()) == nullptr) { @@ -91,17 +91,17 @@ class TempDir { } } - TempDir(const TempDir&) = delete; + TempDir(TempDir const&) = delete; TempDir& operator=(TempDir const&) = delete; - TempDir(const TempDir&&) = delete; + TempDir(TempDir const&&) = delete; TempDir&& operator=(TempDir const&&) = delete; - const std::filesystem::path& path() { return _dir_path; } + std::filesystem::path const& path() { return _dir_path; } operator std::string() { return path(); } private: - const bool _cleanup; + bool const _cleanup; std::filesystem::path _dir_path{}; }; @@ -120,7 +120,7 @@ class DevBuffer { { KVIKIO_CHECK_CUDA(cudaMalloc(&ptr, nbytes)); } - DevBuffer(const std::vector& host_buffer) : DevBuffer{host_buffer.size()} + DevBuffer(std::vector const& host_buffer) : DevBuffer{host_buffer.size()} { KVIKIO_CHECK_CUDA(cudaMemcpy(ptr, host_buffer.data(), nbytes, cudaMemcpyHostToDevice)); } @@ -149,7 +149,7 @@ class DevBuffer { return DevBuffer{host_buffer}; } - [[nodiscard]] static DevBuffer zero_like(const DevBuffer& prototype) + [[nodiscard]] static DevBuffer zero_like(DevBuffer const& prototype) { DevBuffer ret{prototype.nelem}; KVIKIO_CHECK_CUDA(cudaMemset(ret.ptr, 0, ret.nbytes)); @@ -176,7 +176,7 @@ class DevBuffer { /** * @brief Check that two buffers are equal */ -inline void expect_equal(const DevBuffer& a, const DevBuffer& b) +inline void expect_equal(DevBuffer const& a, DevBuffer const& b) { EXPECT_EQ(a.nbytes, b.nbytes); auto a_vec = a.to_vector(); diff --git a/java/src/main/native/src/CuFileJni.cpp b/java/src/main/native/src/CuFileJni.cpp index 9aa4131fe2..829fcbf5f9 100644 --- a/java/src/main/native/src/CuFileJni.cpp +++ b/java/src/main/native/src/CuFileJni.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, NVIDIA CORPORATION. + * Copyright (c) 2024-2025, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -168,7 +168,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_kvikio_cufile_CuFileDriver_create(JNIEnv* { try { return reinterpret_cast(new cufile_driver()); - } catch (const std::exception& e) { + } catch (std::exception const& e) { jlong default_ret_val = 0; if (env->ExceptionOccurred()) { return default_ret_val; } From 6b0836e3c3efcc12a110edf44dc0920ceabaa8e4 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 31 Jan 2025 16:31:25 -0600 Subject: [PATCH 03/12] Migrate to NVKS for amd64 CI runners (#605) This migrates amd64 CI jobs (PRs and nightlies) to use L4 GPUs from the NVKS cluster. xref: https://github.com/rapidsai/build-infra/issues/184 Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Gil Forsyth (https://github.com/gforsyth) URL: https://github.com/rapidsai/kvikio/pull/605 --- .github/workflows/build.yaml | 18 +++++------ .github/workflows/pr.yaml | 30 +++++++++---------- .github/workflows/test.yaml | 8 ++--- .../trigger-breaking-change-alert.yaml | 2 +- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 48ed0dd3ae..dedd74ea26 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -56,19 +56,19 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners with: arch: "amd64" branch: ${{ inputs.branch }} build_type: ${{ inputs.build_type || 'branch' }} container_image: "rapidsai/ci-conda:latest" date: ${{ inputs.date }} - node_type: "gpu-v100-latest-1" + node_type: "gpu-l4-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} wheel-build-cpp: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} @@ -79,7 +79,7 @@ jobs: wheel-build-python: needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: wheel-publish-cpp: needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -100,7 +100,7 @@ jobs: wheel-publish-python: needs: wheel-build-python secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@nvks-runners with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a5a5007339..98dacf67d7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -26,7 +26,7 @@ jobs: - wheel-python-build - wheel-python-tests secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@nvks-runners if: always() with: needs: ${{ toJSON(needs) }} @@ -44,7 +44,7 @@ jobs: repo: kvikio changed-files: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@nvks-runners with: files_yaml: | test_cpp: @@ -72,56 +72,56 @@ jobs: - '!notebooks/**' checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@nvks-runners conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@nvks-runners with: build_type: pull-request conda-cpp-tests: needs: [conda-cpp-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@nvks-runners if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request conda-java-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners with: build_type: pull-request - node_type: "gpu-v100-latest-1" + node_type: "gpu-l4-latest-1" arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/test_java.sh" conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@nvks-runners with: build_type: pull-request conda-python-tests: needs: [conda-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@nvks-runners if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners with: build_type: pull-request - node_type: "gpu-v100-latest-1" + node_type: "gpu-l4-latest-1" arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@nvks-runners with: arch: '["amd64"]' cuda: '["12.8"]' @@ -132,7 +132,7 @@ jobs: sccache -s; wheel-cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request @@ -140,14 +140,14 @@ jobs: wheel-python-build: needs: wheel-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners with: build_type: pull-request script: ci/build_wheel_python.sh wheel-python-tests: needs: [wheel-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@nvks-runners if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d29ec0f11e..638c02dedd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@nvks-runners with: build_type: nightly branch: ${{ inputs.branch }} @@ -24,7 +24,7 @@ jobs: sha: ${{ inputs.sha }} python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@nvks-runners with: build_type: nightly branch: ${{ inputs.branch }} @@ -32,13 +32,13 @@ jobs: sha: ${{ inputs.sha }} conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - node_type: "gpu-v100-latest-1" + node_type: "gpu-l4-latest-1" arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/test_java.sh" diff --git a/.github/workflows/trigger-breaking-change-alert.yaml b/.github/workflows/trigger-breaking-change-alert.yaml index 9764c62c15..7b5b4810fb 100644 --- a/.github/workflows/trigger-breaking-change-alert.yaml +++ b/.github/workflows/trigger-breaking-change-alert.yaml @@ -12,7 +12,7 @@ jobs: trigger-notifier: if: contains(github.event.pull_request.labels.*.name, 'breaking') secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-25.04 + uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@nvks-runners with: sender_login: ${{ github.event.sender.login }} sender_avatar: ${{ github.event.sender.avatar_url }} From 3630b8c1d8a186688a58bf0123a271f0e977be9d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 3 Feb 2025 09:13:55 -0600 Subject: [PATCH 04/12] disallow fallback to Make in Python builds (#612) Contributes to https://github.com/rapidsai/build-planning/issues/146 Proposes: * setting `[tool.scikit-build].ninja.make-fallback = false`, so `scikit-build-core` will not silently fallback to using GNU Make if `ninja` is not available Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/kvikio/pull/612 --- python/kvikio/pyproject.toml | 2 +- python/libkvikio/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index 8f5138cb71..201245a621 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -128,7 +128,7 @@ build-dir = "build/{wheel_tag}" cmake.build-type = "Release" cmake.version = "CMakeLists.txt" minimum-version = "build-system.requires" -ninja.make-fallback = true +ninja.make-fallback = false sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["kvikio"] diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index 9504cb3755..8a194ae3ef 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -36,7 +36,7 @@ build-dir = "build/{wheel_tag}" cmake.build-type = "Release" cmake.version = "CMakeLists.txt" minimum-version = "build-system.requires" -ninja.make-fallback = true +ninja.make-fallback = false sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["libkvikio"] From 255ed481899dd358b4f93f509556b33621be6969 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Tue, 4 Feb 2025 15:38:16 -0500 Subject: [PATCH 05/12] Add RAII file wrappers to avoid resource leak (#614) Closes #607 Authors: - Tianyu Liu (https://github.com/kingcrimsontianyu) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Mads R. B. Kristensen (https://github.com/madsbk) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/kvikio/pull/614 --- cpp/CMakeLists.txt | 1 + cpp/include/kvikio/error.hpp | 3 + cpp/include/kvikio/file_handle.hpp | 20 +-- cpp/include/kvikio/file_utils.hpp | 165 +++++++++++++++++++++++++ cpp/src/error.cpp | 15 +++ cpp/src/file_handle.cpp | 187 +++++++++-------------------- cpp/src/file_utils.cpp | 164 +++++++++++++++++++++++++ 7 files changed, 416 insertions(+), 139 deletions(-) create mode 100644 cpp/include/kvikio/file_utils.hpp create mode 100644 cpp/src/file_utils.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1a5890071d..f17db71439 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -140,6 +140,7 @@ set(SOURCES "src/defaults.cpp" "src/error.cpp" "src/file_handle.cpp" + "src/file_utils.cpp" "src/posix_io.cpp" "src/shim/cuda.cpp" "src/shim/cufile.cpp" diff --git a/cpp/include/kvikio/error.hpp b/cpp/include/kvikio/error.hpp index 25bf62a877..e38d05054f 100644 --- a/cpp/include/kvikio/error.hpp +++ b/cpp/include/kvikio/error.hpp @@ -115,6 +115,9 @@ void cufile_check_bytes_done_2(ssize_t nbytes_done, int line_number, char const* } } +#define KVIKIO_LOG_ERROR(err_msg) kvikio::detail::log_error(err_msg, __LINE__, __FILE__) +void log_error(std::string_view err_msg, int line_number, char const* filename); + } // namespace detail } // namespace kvikio diff --git a/cpp/include/kvikio/file_handle.hpp b/cpp/include/kvikio/file_handle.hpp index e32136e468..ba44992b53 100644 --- a/cpp/include/kvikio/file_handle.hpp +++ b/cpp/include/kvikio/file_handle.hpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -44,12 +45,12 @@ namespace kvikio { class FileHandle { private: // We use two file descriptors, one opened with the O_DIRECT flag and one without. - int _fd_direct_on{-1}; - int _fd_direct_off{-1}; + FileWrapper _fd_direct_on{}; + FileWrapper _fd_direct_off{}; bool _initialized{false}; CompatMode _compat_mode{CompatMode::AUTO}; mutable std::size_t _nbytes{0}; // The size of the underlying file, zero means unknown. - CUfileHandle_t _handle{}; + CUFileHandleWrapper _cufile_handle{}; /** * @brief Given a requested compatibility mode, whether it is expected to reduce to `ON` for @@ -122,23 +123,24 @@ class FileHandle { * @brief Get one of the file descriptors * * Notice, FileHandle maintains two file descriptors - one opened with the - * `O_DIRECT` flag and one without. This function returns one of them but - * it is unspecified which one. + * `O_DIRECT` flag and one without. * + * @param o_direct Whether to get the file descriptor opened with the `O_DIRECT` flag. * @return File descriptor */ - [[nodiscard]] int fd() const noexcept; + [[nodiscard]] int fd(bool o_direct = false) const noexcept; /** * @brief Get the flags of one of the file descriptors (see open(2)) * * Notice, FileHandle maintains two file descriptors - one opened with the - * `O_DIRECT` flag and one without. This function returns the flags of one of - * them but it is unspecified which one. + * `O_DIRECT` flag and one without. * + * @param o_direct Whether to get the flags of the file descriptor opened with the `O_DIRECT` + * flag. * @return File descriptor */ - [[nodiscard]] int fd_open_flags() const; + [[nodiscard]] int fd_open_flags(bool o_direct = false) const; /** * @brief Get the file size diff --git a/cpp/include/kvikio/file_utils.hpp b/cpp/include/kvikio/file_utils.hpp new file mode 100644 index 0000000000..9cf3cae380 --- /dev/null +++ b/cpp/include/kvikio/file_utils.hpp @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +#include + +namespace kvikio { +/** + * @brief Class that provides RAII for file handling. + */ +class FileWrapper { + private: + int _fd{-1}; + + public: + /** + * @brief Open file. + * + * @param file_path File path. + * @param flags Open flags given as a string. + * @param o_direct Append O_DIRECT to `flags`. + * @param mode Access modes. + */ + FileWrapper(std::string const& file_path, std::string const& flags, bool o_direct, mode_t mode); + + /** + * @brief Construct an empty file wrapper object without opening a file. + */ + FileWrapper() noexcept = default; + + ~FileWrapper() noexcept; + FileWrapper(FileWrapper const&) = delete; + FileWrapper& operator=(FileWrapper const&) = delete; + FileWrapper(FileWrapper&& o) noexcept; + FileWrapper& operator=(FileWrapper&& o) noexcept; + + /** + * @brief Open file using `open(2)` + * + * @param file_path File path. + * @param flags Open flags given as a string. + * @param o_direct Append O_DIRECT to `flags`. + * @param mode Access modes. + */ + void open(std::string const& file_path, std::string const& flags, bool o_direct, mode_t mode); + + /** + * @brief Check if the file has been opened. + * + * @return A boolean answer indicating if the file has been opened. + */ + bool opened() const noexcept; + + /** + * @brief Close the file if it is opened; do nothing otherwise. + */ + void close() noexcept; + + /** + * @brief Return the file descriptor. + * + * @return File descriptor. + */ + int fd() const noexcept; +}; + +/** + * @brief Class that provides RAII for the cuFile handle. + */ +class CUFileHandleWrapper { + private: + CUfileHandle_t _handle{}; + bool _registered{false}; + + public: + CUFileHandleWrapper() noexcept = default; + ~CUFileHandleWrapper() noexcept; + CUFileHandleWrapper(CUFileHandleWrapper const&) = delete; + CUFileHandleWrapper& operator=(CUFileHandleWrapper const&) = delete; + CUFileHandleWrapper(CUFileHandleWrapper&& o) noexcept; + CUFileHandleWrapper& operator=(CUFileHandleWrapper&& o) noexcept; + + /** + * @brief Register the file handle given the file descriptor. + * + * @param fd File descriptor. + * @return Return the cuFile error code from handle register. If the handle has already been + * registered by calling `register_handle()`, return `std::nullopt`. + */ + std::optional register_handle(int fd) noexcept; + + /** + * @brief Check if the handle has been registered. + * + * @return A boolean answer indicating if the handle has been registered. + */ + bool registered() const noexcept; + + /** + * @brief Return the cuFile handle. + * + * @return The cuFile handle. + */ + CUfileHandle_t handle() const noexcept; + + /** + * @brief Unregister the handle if it has been registered; do nothing otherwise. + */ + void unregister_handle() noexcept; +}; + +/** + * @brief Parse open file flags given as a string and return oflags + * + * @param flags The flags + * @param o_direct Append O_DIRECT to the open flags + * @return oflags + * + * @throw std::invalid_argument if the specified flags are not supported. + * @throw std::invalid_argument if `o_direct` is true, but `O_DIRECT` is not supported. + */ +int open_fd_parse_flags(std::string const& flags, bool o_direct); + +/** + * @brief Open file using `open(2)` + * + * @param flags Open flags given as a string + * @param o_direct Append O_DIRECT to `flags` + * @param mode Access modes + * @return File descriptor + */ +int open_fd(std::string const& file_path, std::string const& flags, bool o_direct, mode_t mode); + +/** + * @brief Get the flags of the file descriptor (see `open(2)`) + * + * @return Open flags + */ +[[nodiscard]] int open_flags(int fd); + +/** + * @brief Get file size from file descriptor `fstat(3)` + * + * @param file_descriptor Open file descriptor + * @return The number of bytes + */ +[[nodiscard]] std::size_t get_file_size(int file_descriptor); + +} // namespace kvikio diff --git a/cpp/src/error.cpp b/cpp/src/error.cpp index 21ce736a65..116a77cd28 100644 --- a/cpp/src/error.cpp +++ b/cpp/src/error.cpp @@ -14,4 +14,19 @@ * limitations under the License. */ +#include + #include + +namespace kvikio { + +namespace detail { + +void log_error(std::string_view err_msg, int line_number, char const* filename) +{ + std::cerr << "KvikIO error at: " << filename << ":" << line_number << ": " << err_msg << "\n"; +} + +} // namespace detail + +} // namespace kvikio diff --git a/cpp/src/file_handle.cpp b/cpp/src/file_handle.cpp index d6b96f3d6d..0e65afb7fd 100644 --- a/cpp/src/file_handle.cpp +++ b/cpp/src/file_handle.cpp @@ -25,104 +25,15 @@ #include #include +#include namespace kvikio { -namespace { - -/** - * @brief Parse open file flags given as a string and return oflags - * - * @param flags The flags - * @param o_direct Append O_DIRECT to the open flags - * @return oflags - * - * @throw std::invalid_argument if the specified flags are not supported. - * @throw std::invalid_argument if `o_direct` is true, but `O_DIRECT` is not supported. - */ -int open_fd_parse_flags(std::string const& flags, bool o_direct) -{ - int file_flags = -1; - if (flags.empty()) { throw std::invalid_argument("Unknown file open flag"); } - switch (flags[0]) { - case 'r': - file_flags = O_RDONLY; - if (flags[1] == '+') { file_flags = O_RDWR; } - break; - case 'w': - file_flags = O_WRONLY; - if (flags[1] == '+') { file_flags = O_RDWR; } - file_flags |= O_CREAT | O_TRUNC; - break; - case 'a': throw std::invalid_argument("Open flag 'a' isn't supported"); - default: throw std::invalid_argument("Unknown file open flag"); - } - file_flags |= O_CLOEXEC; - if (o_direct) { -#if defined(O_DIRECT) - file_flags |= O_DIRECT; -#else - throw std::invalid_argument("'o_direct' flag unsupported on this platform"); -#endif - } - return file_flags; -} - -/** - * @brief Open file using `open(2)` - * - * @param flags Open flags given as a string - * @param o_direct Append O_DIRECT to `flags` - * @param mode Access modes - * @return File descriptor - */ -int open_fd(std::string const& file_path, std::string const& flags, bool o_direct, mode_t mode) -{ - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) - int fd = ::open(file_path.c_str(), open_fd_parse_flags(flags, o_direct), mode); - if (fd == -1) { throw std::system_error(errno, std::generic_category(), "Unable to open file"); } - return fd; -} - -/** - * @brief Get the flags of the file descriptor (see `open(2)`) - * - * @return Open flags - */ -[[nodiscard]] int open_flags(int fd) -{ - int ret = fcntl(fd, F_GETFL); // NOLINT(cppcoreguidelines-pro-type-vararg) - if (ret == -1) { - throw std::system_error(errno, std::generic_category(), "Unable to retrieve open flags"); - } - return ret; -} - -/** - * @brief Get file size from file descriptor `fstat(3)` - * - * @param file_descriptor Open file descriptor - * @return The number of bytes - */ -[[nodiscard]] std::size_t get_file_size(int file_descriptor) -{ - struct stat st {}; - int ret = fstat(file_descriptor, &st); - if (ret == -1) { - throw std::system_error(errno, std::generic_category(), "Unable to query file size"); - } - return static_cast(st.st_size); -} - -} // namespace - FileHandle::FileHandle(std::string const& file_path, std::string const& flags, mode_t mode, CompatMode compat_mode) - : _fd_direct_off{open_fd(file_path, flags, false, mode)}, - _initialized{true}, - _compat_mode{compat_mode} + : _fd_direct_off{file_path, flags, false, mode}, _initialized{true}, _compat_mode{compat_mode} { if (is_compat_mode_preferred()) { return; // Nothing to do in compatibility mode @@ -138,7 +49,7 @@ FileHandle::FileHandle(std::string const& file_path, }; try { - _fd_direct_on = open_fd(file_path, flags, true, mode); + _fd_direct_on.open(file_path, flags, true, mode); } catch (std::system_error const&) { handle_o_direct_except(); } catch (std::invalid_argument const&) { @@ -147,40 +58,37 @@ FileHandle::FileHandle(std::string const& file_path, if (_compat_mode == CompatMode::ON) { return; } - // Create a cuFile handle, if not in compatibility mode - CUfileDescr_t desc{}; // It is important to set to zero! - desc.type = CU_FILE_HANDLE_TYPE_OPAQUE_FD; - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) - desc.handle.fd = _fd_direct_on; + CUFileHandleWrapper handle; + auto error_code = handle.register_handle(_fd_direct_on.fd()); + assert(error_code.has_value()); - auto error_code = cuFileAPI::instance().HandleRegister(&_handle, &desc); // For the AUTO mode, if the first cuFile API call fails, fall back to the compatibility // mode. - if (_compat_mode == CompatMode::AUTO && error_code.err != CU_FILE_SUCCESS) { + if (_compat_mode == CompatMode::AUTO && error_code.value().err != CU_FILE_SUCCESS) { _compat_mode = CompatMode::ON; } else { - CUFILE_TRY(error_code); + CUFILE_TRY(error_code.value()); } } FileHandle::FileHandle(FileHandle&& o) noexcept - : _fd_direct_on{std::exchange(o._fd_direct_on, -1)}, - _fd_direct_off{std::exchange(o._fd_direct_off, -1)}, + : _fd_direct_on{std::exchange(o._fd_direct_on, {})}, + _fd_direct_off{std::exchange(o._fd_direct_off, {})}, _initialized{std::exchange(o._initialized, false)}, _compat_mode{std::exchange(o._compat_mode, CompatMode::AUTO)}, _nbytes{std::exchange(o._nbytes, 0)}, - _handle{std::exchange(o._handle, CUfileHandle_t{})} + _cufile_handle{std::exchange(o._cufile_handle, {})} { } FileHandle& FileHandle::operator=(FileHandle&& o) noexcept { - _fd_direct_on = std::exchange(o._fd_direct_on, -1); - _fd_direct_off = std::exchange(o._fd_direct_off, -1); + _fd_direct_on = std::exchange(o._fd_direct_on, {}); + _fd_direct_off = std::exchange(o._fd_direct_off, {}); _initialized = std::exchange(o._initialized, false); _compat_mode = std::exchange(o._compat_mode, CompatMode::AUTO); _nbytes = std::exchange(o._nbytes, 0); - _handle = std::exchange(o._handle, CUfileHandle_t{}); + _cufile_handle = std::exchange(o._cufile_handle, {}); return *this; } @@ -193,13 +101,11 @@ void FileHandle::close() noexcept try { if (closed()) { return; } - if (!is_compat_mode_preferred()) { cuFileAPI::instance().HandleDeregister(_handle); } + _cufile_handle.unregister_handle(); _compat_mode = CompatMode::AUTO; - ::close(_fd_direct_off); - if (_fd_direct_on != -1) { ::close(_fd_direct_on); } - _fd_direct_on = -1; - _fd_direct_off = -1; - _initialized = false; + _fd_direct_off.close(); + _fd_direct_on.close(); + _initialized = false; } catch (...) { } } @@ -210,17 +116,20 @@ CUfileHandle_t FileHandle::handle() if (is_compat_mode_preferred()) { throw CUfileException("The underlying cuFile handle isn't available in compatibility mode"); } - return _handle; + return _cufile_handle.handle(); } -int FileHandle::fd() const noexcept { return _fd_direct_off; } +int FileHandle::fd(bool o_direct) const noexcept +{ + return o_direct ? _fd_direct_on.fd() : _fd_direct_off.fd(); +} -int FileHandle::fd_open_flags() const { return open_flags(_fd_direct_off); } +int FileHandle::fd_open_flags(bool o_direct) const { return open_flags(fd(o_direct)); } std::size_t FileHandle::nbytes() const { if (closed()) { return 0; } - if (_nbytes == 0) { _nbytes = get_file_size(_fd_direct_off); } + if (_nbytes == 0) { _nbytes = get_file_size(_fd_direct_off.fd()); } return _nbytes; } @@ -231,13 +140,17 @@ std::size_t FileHandle::read(void* devPtr_base, bool sync_default_stream) { if (is_compat_mode_preferred()) { - return detail::posix_device_read(_fd_direct_off, devPtr_base, size, file_offset, devPtr_offset); + return detail::posix_device_read( + _fd_direct_off.fd(), devPtr_base, size, file_offset, devPtr_offset); } if (sync_default_stream) { CUDA_DRIVER_TRY(cudaAPI::instance().StreamSynchronize(nullptr)); } KVIKIO_NVTX_SCOPED_RANGE("cufileRead()", size); - ssize_t ret = cuFileAPI::instance().Read( - _handle, devPtr_base, size, convert_size2off(file_offset), convert_size2off(devPtr_offset)); + ssize_t ret = cuFileAPI::instance().Read(_cufile_handle.handle(), + devPtr_base, + size, + convert_size2off(file_offset), + convert_size2off(devPtr_offset)); CUFILE_CHECK_BYTES_DONE(ret); return ret; } @@ -252,13 +165,16 @@ std::size_t FileHandle::write(void const* devPtr_base, if (is_compat_mode_preferred()) { return detail::posix_device_write( - _fd_direct_off, devPtr_base, size, file_offset, devPtr_offset); + _fd_direct_off.fd(), devPtr_base, size, file_offset, devPtr_offset); } if (sync_default_stream) { CUDA_DRIVER_TRY(cudaAPI::instance().StreamSynchronize(nullptr)); } KVIKIO_NVTX_SCOPED_RANGE("cufileWrite()", size); - ssize_t ret = cuFileAPI::instance().Write( - _handle, devPtr_base, size, convert_size2off(file_offset), convert_size2off(devPtr_offset)); + ssize_t ret = cuFileAPI::instance().Write(_cufile_handle.handle(), + devPtr_base, + size, + convert_size2off(file_offset), + convert_size2off(devPtr_offset)); if (ret == -1) { throw std::system_error(errno, std::generic_category(), "Unable to write file"); } @@ -283,7 +199,8 @@ std::future FileHandle::pread(void* buf, std::size_t file_offset, std::size_t hostPtr_offset) -> std::size_t { char* buf = static_cast(hostPtr_base) + hostPtr_offset; - return detail::posix_host_read(_fd_direct_off, buf, size, file_offset); + return detail::posix_host_read( + _fd_direct_off.fd(), buf, size, file_offset); }; return parallel_io(op, buf, size, file_offset, task_size, 0); @@ -295,7 +212,7 @@ std::future FileHandle::pread(void* buf, if (size < gds_threshold) { auto task = [this, ctx, buf, size, file_offset]() -> std::size_t { PushAndPopContext c(ctx); - return detail::posix_device_read(_fd_direct_off, buf, size, file_offset, 0); + return detail::posix_device_read(_fd_direct_off.fd(), buf, size, file_offset, 0); }; return std::async(std::launch::deferred, task); } @@ -333,7 +250,7 @@ std::future FileHandle::pwrite(void const* buf, std::size_t hostPtr_offset) -> std::size_t { char const* buf = static_cast(hostPtr_base) + hostPtr_offset; return detail::posix_host_write( - _fd_direct_off, buf, size, file_offset); + _fd_direct_off.fd(), buf, size, file_offset); }; return parallel_io(op, buf, size, file_offset, task_size, 0); @@ -345,7 +262,7 @@ std::future FileHandle::pwrite(void const* buf, if (size < gds_threshold) { auto task = [this, ctx, buf, size, file_offset]() -> std::size_t { PushAndPopContext c(ctx); - return detail::posix_device_write(_fd_direct_off, buf, size, file_offset, 0); + return detail::posix_device_write(_fd_direct_off.fd(), buf, size, file_offset, 0); }; return std::async(std::launch::deferred, task); } @@ -380,8 +297,13 @@ void FileHandle::read_async(void* devPtr_base, *bytes_read_p = static_cast(read(devPtr_base, *size_p, *file_offset_p, *devPtr_offset_p)); } else { - CUFILE_TRY(cuFileAPI::instance().ReadAsync( - _handle, devPtr_base, size_p, file_offset_p, devPtr_offset_p, bytes_read_p, stream)); + CUFILE_TRY(cuFileAPI::instance().ReadAsync(_cufile_handle.handle(), + devPtr_base, + size_p, + file_offset_p, + devPtr_offset_p, + bytes_read_p, + stream)); } } @@ -407,8 +329,13 @@ void FileHandle::write_async(void* devPtr_base, *bytes_written_p = static_cast(write(devPtr_base, *size_p, *file_offset_p, *devPtr_offset_p)); } else { - CUFILE_TRY(cuFileAPI::instance().WriteAsync( - _handle, devPtr_base, size_p, file_offset_p, devPtr_offset_p, bytes_written_p, stream)); + CUFILE_TRY(cuFileAPI::instance().WriteAsync(_cufile_handle.handle(), + devPtr_base, + size_p, + file_offset_p, + devPtr_offset_p, + bytes_written_p, + stream)); } } diff --git a/cpp/src/file_utils.cpp b/cpp/src/file_utils.cpp new file mode 100644 index 0000000000..ca00f254b2 --- /dev/null +++ b/cpp/src/file_utils.cpp @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace kvikio { + +FileWrapper::FileWrapper(std::string const& file_path, + std::string const& flags, + bool o_direct, + mode_t mode) +{ + open(file_path, flags, o_direct, mode); +} + +FileWrapper::~FileWrapper() noexcept { close(); } + +FileWrapper::FileWrapper(FileWrapper&& o) noexcept : _fd(std::exchange(o._fd, -1)) {} + +FileWrapper& FileWrapper::operator=(FileWrapper&& o) noexcept +{ + _fd = std::exchange(o._fd, -1); + return *this; +} + +void FileWrapper::open(std::string const& file_path, + std::string const& flags, + bool o_direct, + mode_t mode) +{ + if (!opened()) { _fd = open_fd(file_path, flags, o_direct, mode); } +} + +bool FileWrapper::opened() const noexcept { return _fd != -1; } + +void FileWrapper::close() noexcept +{ + if (opened()) { + if (::close(_fd) != 0) { KVIKIO_LOG_ERROR("File cannot be closed"); } + _fd = -1; + } +} + +int FileWrapper::fd() const noexcept { return _fd; } + +CUFileHandleWrapper::~CUFileHandleWrapper() noexcept { unregister_handle(); } + +CUFileHandleWrapper::CUFileHandleWrapper(CUFileHandleWrapper&& o) noexcept + : _handle{std::exchange(o._handle, {})}, _registered{std::exchange(o._registered, false)} +{ +} + +CUFileHandleWrapper& CUFileHandleWrapper::operator=(CUFileHandleWrapper&& o) noexcept +{ + _handle = std::exchange(o._handle, {}); + _registered = std::exchange(o._registered, false); + return *this; +} + +std::optional CUFileHandleWrapper::register_handle(int fd) noexcept +{ + std::optional error_code; + if (registered()) { return error_code; } + + // Create a cuFile handle, if not in compatibility mode + CUfileDescr_t desc{}; // It is important to set to zero! + desc.type = CU_FILE_HANDLE_TYPE_OPAQUE_FD; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) + desc.handle.fd = fd; + error_code = cuFileAPI::instance().HandleRegister(&_handle, &desc); + if (error_code.value().err == CU_FILE_SUCCESS) { _registered = true; } + return error_code; +} + +bool CUFileHandleWrapper::registered() const noexcept { return _registered; } + +CUfileHandle_t CUFileHandleWrapper::handle() const noexcept { return _handle; } + +void CUFileHandleWrapper::unregister_handle() noexcept +{ + if (registered()) { + cuFileAPI::instance().HandleDeregister(_handle); + _registered = false; + } +} + +int open_fd_parse_flags(std::string const& flags, bool o_direct) +{ + int file_flags = -1; + if (flags.empty()) { throw std::invalid_argument("Unknown file open flag"); } + switch (flags[0]) { + case 'r': + file_flags = O_RDONLY; + if (flags[1] == '+') { file_flags = O_RDWR; } + break; + case 'w': + file_flags = O_WRONLY; + if (flags[1] == '+') { file_flags = O_RDWR; } + file_flags |= O_CREAT | O_TRUNC; + break; + case 'a': throw std::invalid_argument("Open flag 'a' isn't supported"); + default: throw std::invalid_argument("Unknown file open flag"); + } + file_flags |= O_CLOEXEC; + if (o_direct) { +#if defined(O_DIRECT) + file_flags |= O_DIRECT; +#else + throw std::invalid_argument("'o_direct' flag unsupported on this platform"); +#endif + } + return file_flags; +} + +int open_fd(std::string const& file_path, std::string const& flags, bool o_direct, mode_t mode) +{ + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + int fd = ::open(file_path.c_str(), open_fd_parse_flags(flags, o_direct), mode); + if (fd == -1) { throw std::system_error(errno, std::generic_category(), "Unable to open file"); } + return fd; +} + +[[nodiscard]] int open_flags(int fd) +{ + int ret = fcntl(fd, F_GETFL); // NOLINT(cppcoreguidelines-pro-type-vararg) + if (ret == -1) { + throw std::system_error(errno, std::generic_category(), "Unable to retrieve open flags"); + } + return ret; +} + +[[nodiscard]] std::size_t get_file_size(int file_descriptor) +{ + struct stat st {}; + int ret = fstat(file_descriptor, &st); + if (ret == -1) { + throw std::system_error(errno, std::generic_category(), "Unable to query file size"); + } + return static_cast(st.st_size); +} + +} // namespace kvikio From b00141379b8ec777b0f3081069986c70552f880e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 5 Feb 2025 14:34:51 -0800 Subject: [PATCH 06/12] Fix package versioning (#616) The nightly package versions are currently not being correctly assigned. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Mads R. B. Kristensen (https://github.com/madsbk) - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/kvikio/pull/616 --- ci/build_cpp.sh | 4 ++-- ci/build_python.sh | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 8fb3a35991..ca1c20c7b0 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail @@ -18,7 +18,7 @@ conda config --set path_conflict prevent sccache --zero-stats -rapids-conda-retry mambabuild conda/recipes/libkvikio +RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libkvikio sccache --show-adv-stats diff --git a/ci/build_python.sh b/ci/build_python.sh index 7e0fc0bf93..b15f2a9d75 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail @@ -13,6 +13,8 @@ export CMAKE_GENERATOR=Ninja rapids-print-env +rapids-generate-version > ./VERSION + rapids-logger "Begin py build" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) @@ -20,7 +22,7 @@ conda config --set path_conflict prevent sccache --zero-stats -rapids-conda-retry mambabuild \ +RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ --channel "${CPP_CHANNEL}" \ conda/recipes/kvikio From 099657c3ace4ade72e8fb320c76901bdbc2ba4bd Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 6 Feb 2025 09:55:07 -0500 Subject: [PATCH 07/12] Add `shellcheck` to pre-commit and fix warnings (#621) `shellcheck` is a fast, static analysis tool for shell scripts. It's good at flagging up unused variables, unintentional glob expansions, and other potential execution and security headaches that arise from the wonders of `bash` (and other shlangs). This PR adds a `pre-commit` hook to run `shellcheck` on all of the `sh-lang` files in the `ci/` directory, and the changes requested by `shellcheck` to make the existing files pass the check. xref: rapidsai/build-planning#135 Authors: - Gil Forsyth (https://github.com/gforsyth) Approvers: - James Lamb (https://github.com/jameslamb) - Mads R. B. Kristensen (https://github.com/madsbk) URL: https://github.com/rapidsai/kvikio/pull/621 --- .pre-commit-config.yaml | 7 +++++++ ci/build_docs.sh | 5 +++-- ci/build_wheel_cpp.sh | 4 ++-- ci/build_wheel_python.sh | 4 ++-- ci/check_style.sh | 6 +++--- ci/release/update-version.sh | 12 ++++-------- ci/test_cpp.sh | 4 ++-- ci/test_python.sh | 4 ++-- ci/test_wheel.sh | 8 ++++---- ci/validate_wheel.sh | 6 +++--- 10 files changed, 32 insertions(+), 28 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 50f26c7bc0..7c21cdf377 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -108,6 +108,13 @@ repos: hooks: - id: rapids-dependency-file-generator args: ["--clean"] + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck + args: ["--severity=warning"] + files: ^ci/ + default_language_version: python: python3 diff --git a/ci/build_docs.sh b/ci/build_docs.sh index ea408b6940..8055f375a2 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail @@ -22,7 +22,8 @@ conda activate docs rapids-print-env -export RAPIDS_DOCS_DIR="$(mktemp -d)" +RAPIDS_DOCS_DIR="$(mktemp -d)" +export RAPIDS_DOCS_DIR rapids-logger "Build CPP docs" pushd cpp/doxygen diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index ca27717769..ce154f24b4 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail @@ -28,7 +28,7 @@ export PIP_NO_BUILD_ISOLATION=0 export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON" ./ci/build_wheel.sh "${package_name}" "${package_dir}" -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" mkdir -p ${package_dir}/final_dist python -m auditwheel repair \ diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 088e8e8e8f..6caf967a01 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -1,12 +1,12 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail package_name="kvikio" package_dir="python/kvikio" -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" # Ensure 'kvikio' wheel builds always use the 'libkvikio' just built in the same CI run # diff --git a/ci/check_style.sh b/ci/check_style.sh index a17065c1f9..675f251098 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. set -euo pipefail @@ -18,8 +18,8 @@ RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION_MAJOR_MINOR}/cmake-format-rapids-cmake.json export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json -mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) -wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL} +mkdir -p "$(dirname ${RAPIDS_CMAKE_FORMAT_FILE})" +wget -O ${RAPIDS_CMAKE_FORMAT_FILE} "${FORMAT_FILE_URL}" # Run pre-commit checks pre-commit run --hook-stage manual --all-files --show-diff-on-failure diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 102beaa2ba..1ad3f06a8c 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. ########################## # KvikIO Version Updater # ########################## @@ -12,14 +12,10 @@ NEXT_FULL_TAG=$1 # Get current version CURRENT_TAG=$(git tag --merged HEAD | grep -xE '^v.*' | sort --version-sort | tail -n 1 | tr -d 'v') -CURRENT_MAJOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[1]}') -CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}') -CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}') -CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} # Get . for next version -NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}') -NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}') +NEXT_MAJOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[1]}') +NEXT_MINOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[2]}') NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} # Need to distutils-normalize the original version @@ -29,7 +25,7 @@ echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" # Inplace sed replace; workaround for Linux and Mac function sed_runner() { - sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak + sed -i.bak ''"$1"'' "$2" && rm -f "${2}".bak } # Centralized version file update diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index ef7933f150..9f1fea1e5d 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail @@ -35,4 +35,4 @@ nvidia-smi && EXITCODE=$? || EXITCODE=$?; rapids-logger "Test script exiting with value: $EXITCODE" -exit ${EXITCODE} +exit "${EXITCODE}" diff --git a/ci/test_python.sh b/ci/test_python.sh index df16f20f9f..f7651f8d60 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail @@ -46,4 +46,4 @@ rapids-logger "pytest kvikio" && EXITCODE=$? || EXITCODE=$?; rapids-logger "Test script exiting with value: $EXITCODE" -exit ${EXITCODE} +exit "${EXITCODE}" diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 46ae5cbcf2..fb84af4379 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -1,17 +1,17 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -eou pipefail -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" # Download and install the libkvikio and kvikio wheels built in the previous step RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist python -m pip install -v \ - "$(echo ./dist/libkvikio_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./dist/kvikio_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" + "$(echo ./dist/libkvikio_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo ./dist/kvikio_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" # If running CUDA 11.8 on arm64, we skip tests marked "cufile" since # cuFile didn't support arm until 12.4 diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 5910a5c59f..40bb27feeb 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. set -euo pipefail @@ -12,10 +12,10 @@ rapids-logger "validate packages with 'pydistcheck'" pydistcheck \ --inspect \ - "$(echo ${wheel_dir_relative_path}/*.whl)" + "$(echo "${wheel_dir_relative_path}"/*.whl)" rapids-logger "validate packages with 'twine'" twine check \ --strict \ - "$(echo ${wheel_dir_relative_path}/*.whl)" + "$(echo "${wheel_dir_relative_path}"/*.whl)" From 6b1a71fabb781e9ab15e40fee32db5c86d791790 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 6 Feb 2025 14:27:36 -0500 Subject: [PATCH 08/12] Add build_type input field for `test.yaml` (#620) Exposes `build_type` as an input in `test.yaml` so that `test.yaml` can be manually run against a specific branch/commit as needed. The default value is still `nightly`, and without maintainer intervention, that is what will run each night. xref rapidsai/build-planning#147 Authors: - Gil Forsyth (https://github.com/gforsyth) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/kvikio/pull/620 --- .github/workflows/test.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 638c02dedd..e4c7e5c928 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,13 +12,16 @@ on: sha: required: true type: string + build_type: + type: string + default: nightly jobs: cpp-tests: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@nvks-runners with: - build_type: nightly + build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} @@ -26,7 +29,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@nvks-runners with: - build_type: nightly + build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} @@ -34,7 +37,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners with: - build_type: nightly + build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} From 5b57e493397df28fdb1eaf6147e81d4dec29a5fe Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 7 Feb 2025 09:14:58 -0800 Subject: [PATCH 09/12] Use shared-workflows branch-25.04 (#623) This completes the migration to NVKS runners now that all libraries have been tested and https://github.com/rapidsai/shared-workflows/pull/273 has been merged. xref: https://github.com/rapidsai/build-infra/issues/184 Authors: - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/kvikio/pull/623 --- .github/workflows/build.yaml | 16 ++++++------ .github/workflows/pr.yaml | 26 +++++++++---------- .github/workflows/test.yaml | 6 ++--- .../trigger-breaking-change-alert.yaml | 2 +- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index dedd74ea26..ec9127e5e4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -56,7 +56,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -68,7 +68,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cpp: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} @@ -79,7 +79,7 @@ jobs: wheel-build-python: needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: wheel-publish-cpp: needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -100,7 +100,7 @@ jobs: wheel-publish-python: needs: wheel-build-python secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 98dacf67d7..f03335a211 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -26,7 +26,7 @@ jobs: - wheel-python-build - wheel-python-tests secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.04 if: always() with: needs: ${{ toJSON(needs) }} @@ -44,7 +44,7 @@ jobs: repo: kvikio changed-files: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.04 with: files_yaml: | test_cpp: @@ -72,24 +72,24 @@ jobs: - '!notebooks/**' checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.04 conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-25.04 with: build_type: pull-request conda-cpp-tests: needs: [conda-cpp-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.04 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request conda-java-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 with: build_type: pull-request node_type: "gpu-l4-latest-1" @@ -99,20 +99,20 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.04 with: build_type: pull-request conda-python-tests: needs: [conda-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.04 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 with: build_type: pull-request node_type: "gpu-l4-latest-1" @@ -121,7 +121,7 @@ jobs: run_script: "ci/build_docs.sh" devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.04 with: arch: '["amd64"]' cuda: '["12.8"]' @@ -132,7 +132,7 @@ jobs: sccache -s; wheel-cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request @@ -140,14 +140,14 @@ jobs: wheel-python-build: needs: wheel-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.04 with: build_type: pull-request script: ci/build_wheel_python.sh wheel-python-tests: needs: [wheel-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.04 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e4c7e5c928..5037318440 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,7 +19,7 @@ on: jobs: cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-25.04 with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -27,7 +27,7 @@ jobs: sha: ${{ inputs.sha }} python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.04 with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -35,7 +35,7 @@ jobs: sha: ${{ inputs.sha }} conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/trigger-breaking-change-alert.yaml b/.github/workflows/trigger-breaking-change-alert.yaml index 7b5b4810fb..9764c62c15 100644 --- a/.github/workflows/trigger-breaking-change-alert.yaml +++ b/.github/workflows/trigger-breaking-change-alert.yaml @@ -12,7 +12,7 @@ jobs: trigger-notifier: if: contains(github.event.pull_request.labels.*.name, 'breaking') secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-25.04 with: sender_login: ${{ github.event.sender.login }} sender_avatar: ${{ github.event.sender.avatar_url }} From 7dcf2165a27335f494e59cb0c363f430a556dd37 Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Fri, 7 Feb 2025 16:40:40 -0600 Subject: [PATCH 10/12] add telemetry (#615) Enables telemetry during kvikio CI runs. This is done by parsing GitHub Actions run log metadata and should have no impact on build or test times. xref https://github.com/rapidsai/build-infra/issues/139 Authors: - Mike Sarahan (https://github.com/msarahan) - James Lamb (https://github.com/jameslamb) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/kvikio/pull/615 --- .github/workflows/pr.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f03335a211..60d65026c8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,6 +25,7 @@ jobs: - wheel-cpp-build - wheel-python-build - wheel-python-tests + - telemetry-setup secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.04 if: always() @@ -35,6 +36,7 @@ jobs: # provides at least Python 3.11 (see # https://docs.python.org/3/library/datetime.html#datetime.date.fromisoformat) runs-on: ubuntu-24.04 + needs: telemetry-setup env: RAPIDS_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -70,9 +72,23 @@ jobs: - '!README.md' - '!docs/**' - '!notebooks/**' + telemetry-setup: + runs-on: ubuntu-latest + continue-on-error: true + env: + OTEL_SERVICE_NAME: "pr-kvikio" + steps: + - name: Telemetry setup + # This gate is here and not at the job level because we need the job to not be skipped, + # since other jobs depend on it. + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main checks: secrets: inherit + needs: telemetry-setup uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.04 + with: + ignored_pr_jobs: telemetry-summarize conda-cpp-build: needs: checks secrets: inherit @@ -120,6 +136,7 @@ jobs: container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" devcontainer: + needs: telemetry-setup secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.04 with: @@ -152,3 +169,12 @@ jobs: with: build_type: pull-request script: ci/test_wheel.sh + telemetry-summarize: + # This job must use a self-hosted runner to record telemetry traces. + runs-on: linux-amd64-cpu4 + needs: pr-builder + if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} + continue-on-error: true + steps: + - name: Telemetry summarize + uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main From 5a825df50e53d546c0b08f2c57f0a578e42fd328 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 7 Feb 2025 16:14:23 -0800 Subject: [PATCH 11/12] Use rapids-generate-version for package versions (#628) This uses the `RAPIDS_PACKAGE_VERSION` values set in #616. This ensures we have consistent nightly versions. This PR is independent of #622 (it is needed regardless of whether that PR is closed or merged). Authors: - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/kvikio/pull/628 --- conda/recipes/kvikio/meta.yaml | 2 +- conda/recipes/libkvikio/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/kvikio/meta.yaml b/conda/recipes/kvikio/meta.yaml index c59640cbb8..01c98fd528 100644 --- a/conda/recipes/kvikio/meta.yaml +++ b/conda/recipes/kvikio/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2023-2025, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} {% set py_version = environ['CONDA_PY'] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} diff --git a/conda/recipes/libkvikio/meta.yaml b/conda/recipes/libkvikio/meta.yaml index a616292262..a2f5579865 100644 --- a/conda/recipes/libkvikio/meta.yaml +++ b/conda/recipes/libkvikio/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2023-2025, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} {% set date_string = environ['RAPIDS_DATE_STRING'] %} From ecb81c2c3c17252fcd5a902af9789debb234426b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 14 Feb 2025 15:49:54 -0500 Subject: [PATCH 12/12] Consolidate more Conda solves in CI (#636) Issue: https://github.com/rapidsai/build-planning/issues/22 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/kvikio/pull/636 --- ci/test_java.sh | 16 +++++++--------- dependencies.yaml | 1 + 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ci/test_java.sh b/ci/test_java.sh index e473dd6915..05c4e3346d 100755 --- a/ci/test_java.sh +++ b/ci/test_java.sh @@ -1,15 +1,20 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. set -euo pipefail . /opt/conda/etc/profile.d/conda.sh +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) + rapids-logger "Generate java testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_java \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \ + --prepend-channel "${CPP_CHANNEL}" \ + | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test @@ -18,15 +23,8 @@ set +u conda activate test set -u -rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) - rapids-print-env -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - libkvikio - rapids-logger "Check GPU usage" nvidia-smi diff --git a/dependencies.yaml b/dependencies.yaml index be30907dff..f855576305 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -104,6 +104,7 @@ files: - cuda_version - cuda - test_java + - depends_on_libkvikio channels: - rapidsai - rapidsai-nightly