diff --git a/.github/workflows/trigger-breaking-change-alert.yaml b/.github/workflows/trigger-breaking-change-alert.yaml index 3b972f31ca..01dd2436be 100644 --- a/.github/workflows/trigger-breaking-change-alert.yaml +++ b/.github/workflows/trigger-breaking-change-alert.yaml @@ -12,7 +12,7 @@ jobs: trigger-notifier: if: contains(github.event.pull_request.labels.*.name, 'breaking') secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-25.02 with: sender_login: ${{ github.event.sender.login }} sender_avatar: ${{ github.event.sender.avatar_url }} diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 8c2ee8ae95..ea408b6940 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -6,28 +6,22 @@ set -euo pipefail rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh -export RAPIDS_VERSION="$(rapids-version)" +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-dependency-file-generator \ --output conda \ --file-key docs \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \ + | tee env.yaml -rapids-mamba-retry env create --yes -f env.yaml -n docs +rapids-mamba-retry env create -yq -f env.yaml -n docs conda activate docs rapids-print-env -rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) - -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - "kvikio=${RAPIDS_VERSION}" \ - "libkvikio=${RAPIDS_VERSION}" - export RAPIDS_DOCS_DIR="$(mktemp -d)" rapids-logger "Build CPP docs" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index fb4b1660fa..102beaa2ba 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -20,12 +20,10 @@ CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} # Get . for next version NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}') NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}') -NEXT_PATCH=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[3]}') NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} # Need to distutils-normalize the original version NEXT_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_SHORT_TAG}'))") -PATCH_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_PATCH}'))") echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" @@ -38,7 +36,10 @@ function sed_runner() { echo "${NEXT_FULL_TAG}" > VERSION DEPENDENCIES=( + kvikio libkvikio + libkvikio-tests + rapids-dask-dependency ) for DEP in "${DEPENDENCIES[@]}"; do for FILE in dependencies.yaml conda/environments/*.yaml; do @@ -54,10 +55,6 @@ for FILE in .github/workflows/*.yaml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" done -# Java files -NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}-SNAPSHOT" -sed_runner "s|.*-SNAPSHOT|${NEXT_FULL_JAVA_TAG}|g" java/pom.xml - # .devcontainer files find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r -d '' filename; do sed_runner "s@rapidsai/devcontainers:[0-9.]*@rapidsai/devcontainers:${NEXT_SHORT_TAG}@g" "${filename}" diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 7dfde2187f..ef7933f150 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -5,33 +5,28 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-version)" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_cpp \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \ + --prepend-channel "${CPP_CHANNEL}" \ + | tee env.yaml -rapids-mamba-retry env create --yes -f env.yaml -n test +rapids-mamba-retry env create -qy -f env.yaml -n test # Temporarily allow unbound variables for conda activation. set +u conda activate test set -u -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" -SUITEERROR=0 rapids-print-env -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - "libkvikio=${RAPIDS_VERSION}" \ - "libkvikio-tests=${RAPIDS_VERSION}" - rapids-logger "Check GPU usage" nvidia-smi diff --git a/ci/test_python.sh b/ci/test_python.sh index 1f4ff6d2f1..df16f20f9f 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -8,37 +8,31 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-version)" +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_python \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \ + | tee env.yaml -rapids-mamba-retry env create --yes -f env.yaml -n test +rapids-mamba-retry env create -qy -f env.yaml -n test # Temporarily allow unbound variables for conda activation. set +u conda activate test set -u -rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) - RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" rapids-print-env -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - "libkvikio=${RAPIDS_VERSION}" \ - "kvikio=${RAPIDS_VERSION}" - rapids-logger "Check GPU usage" nvidia-smi diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 2de4597657..328ae17a46 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -9,13 +9,12 @@ dependencies: - boto3>=1.21.21 - c-compiler - cmake>=3.26.4,!=3.30.0 -- cuda-python>=11.7.1,<12.0a0,<=11.8.3 +- cuda-python>=11.8.5,<12.0a0 - cuda-version=11.8 - cudatoolkit - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-aarch64=11.* - libcurl>=7.87.0 @@ -33,6 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8b63a6726b..d1dad00c50 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -9,13 +9,12 @@ dependencies: - boto3>=1.21.21 - c-compiler - cmake>=3.26.4,!=3.30.0 -- cuda-python>=11.7.1,<12.0a0,<=11.8.3 +- cuda-python>=11.8.5,<12.0a0 - cuda-version=11.8 - cudatoolkit - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-64=11.* - libcufile-dev=1.4.0.31 @@ -35,6 +34,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 87759faf53..53ad0ab929 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -10,12 +10,11 @@ dependencies: - c-compiler - cmake>=3.26.4,!=3.30.0 - cuda-nvcc -- cuda-python>=12.0,<13.0a0,<=12.6.0 +- cuda-python>=12.6.2,<13.0a0 - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-aarch64=11.* - libcufile-dev @@ -33,6 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index e0a10af655..68b9a72f6b 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -10,12 +10,11 @@ dependencies: - c-compiler - cmake>=3.26.4,!=3.30.0 - cuda-nvcc -- cuda-python>=12.0,<13.0a0,<=12.6.0 +- cuda-python>=12.6.2,<13.0a0 - cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-64=11.* - libcufile-dev @@ -33,6 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==25.2.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/recipes/kvikio/conda_build_config.yaml b/conda/recipes/kvikio/conda_build_config.yaml index 8af5f6f9c2..639a56f509 100644 --- a/conda/recipes/kvikio/conda_build_config.yaml +++ b/conda/recipes/kvikio/conda_build_config.yaml @@ -4,6 +4,9 @@ c_compiler_version: cxx_compiler_version: - 11 +cmake_version: + - ">=3.26.4,!=3.30.0" + cuda_compiler: - cuda-nvcc @@ -16,8 +19,18 @@ c_stdlib: c_stdlib_version: - "2.17" -cmake_version: - - ">=3.26.4,!=3.30.0" +# The CTK libraries below are missing from the conda-forge::cudatoolkit package +# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages +# and the "*_run_*" version specifiers correspond to `11.x` packages. + +cuda11_libcufile_host_version: + - "1.4.0.31" + +cuda11_libcufile_run_version: + - ">=1.0.0.82,<=1.4.0.31" + +libcurl_version: + - "==7.87.0" nvcomp_version: - "=4.1.0.6" diff --git a/conda/recipes/kvikio/meta.yaml b/conda/recipes/kvikio/meta.yaml index 3c41af3310..271712ac51 100644 --- a/conda/recipes/kvikio/meta.yaml +++ b/conda/recipes/kvikio/meta.yaml @@ -37,6 +37,8 @@ build: - {{ compiler('cuda11') }} {% else %} - {{ compiler('cuda') }} + - cuda-cudart-dev + - libcufile-dev # [linux] {% endif %} requirements: @@ -58,13 +60,17 @@ requirements: - cython >=3.0.0 {% if cuda_major == "11" %} - cudatoolkit + - libcufile {{ cuda11_libcufile_run_version }} # [linux64] + {% else %} + - cuda-cudart-dev + - libcufile-dev # [linux] {% endif %} - cuda-version ={{ cuda_version }} - nvcomp {{ nvcomp_version }} - rapids-build-backend >=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.10.0 - libkvikio ={{ version }} - - libcurl==7.87.0 + - libcurl {{ libcurl_version }} run: - python - numpy >=1.23,<3.0a0 @@ -76,6 +82,10 @@ requirements: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit + - libcufile {{ cuda11_libcufile_run_version }} # [linux64] + {% else %} + - cuda-cudart + - libcufile # [linux] {% endif %} test: diff --git a/conda/recipes/libkvikio/conda_build_config.yaml b/conda/recipes/libkvikio/conda_build_config.yaml index 9cf2923599..b895b842f3 100644 --- a/conda/recipes/libkvikio/conda_build_config.yaml +++ b/conda/recipes/libkvikio/conda_build_config.yaml @@ -28,3 +28,6 @@ cuda11_libcufile_host_version: cuda11_libcufile_run_version: - ">=1.0.0.82,<=1.4.0.31" + +libcurl_version: + - "==7.87.0" diff --git a/conda/recipes/libkvikio/meta.yaml b/conda/recipes/libkvikio/meta.yaml index 999b9fc2c1..4019a55ec8 100644 --- a/conda/recipes/libkvikio/meta.yaml +++ b/conda/recipes/libkvikio/meta.yaml @@ -52,7 +52,7 @@ requirements: {% else %} - libcufile-dev # [linux] {% endif %} - - libcurl==7.87.0 + - libcurl {{ libcurl_version }} outputs: - name: libkvikio @@ -75,7 +75,7 @@ outputs: - cmake {{ cmake_version }} host: - cuda-version ={{ cuda_version }} - - libcurl==7.87.0 + - libcurl {{ libcurl_version }} run: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9c1450518b..a0639c5382 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -80,7 +80,7 @@ if(KvikIO_CUDA_SUPPORT) else() set(cuFile_FOUND 1) - # Check batch and stream API support (cuFile_BATCH_API_FOUND and cuFile_STREAM_API_FOUND) + # Check API support try_compile( cuFile_BATCH_API_FOUND SOURCE_FROM_CONTENT batch.cpp @@ -109,6 +109,20 @@ if(KvikIO_CUDA_SUPPORT) OUTPUT_VARIABLE stream_output ) message(STATUS "Found cuFile Stream API: ${cuFile_STREAM_API_FOUND}") + try_compile( + cuFile_VERSION_API_FOUND SOURCE_FROM_CONTENT + version.cpp + [[#include + int main() { + int version; + cuFileGetVersion(&version); + return 0; + } + ]] + LINK_LIBRARIES CUDA::cuFile rt ${CMAKE_DL_LIBS} + OUTPUT_VARIABLE version_output + ) + message(STATUS "Found cuFile Version API: ${cuFile_VERSION_API_FOUND}") endif() endif() @@ -154,6 +168,7 @@ target_compile_definitions( $<$:KVIKIO_CUFILE_FOUND> $<$:KVIKIO_CUFILE_BATCH_API_FOUND> $<$:KVIKIO_CUFILE_STREAM_API_FOUND> + $<$:KVIKIO_CUFILE_VERSION_API_FOUND> ) set_target_properties( diff --git a/cpp/examples/basic_io.cpp b/cpp/examples/basic_io.cpp index 4d04391404..39bfc315cd 100644 --- a/cpp/examples/basic_io.cpp +++ b/cpp/examples/basic_io.cpp @@ -181,7 +181,7 @@ int main() cout << "Parallel POSIX read (" << kvikio::defaults::thread_pool_nthreads() << " threads): " << read << endl; } - if (kvikio::is_batch_and_stream_available() && !kvikio::defaults::is_compat_mode_preferred()) { + if (kvikio::is_batch_api_available() && !kvikio::defaults::is_compat_mode_preferred()) { std::cout << std::endl; Timer timer; // Here we use the batch API to read "/tmp/test-file" into `b_dev` by diff --git a/cpp/include/kvikio/file_handle.hpp b/cpp/include/kvikio/file_handle.hpp index abc6660de6..4880bd4f20 100644 --- a/cpp/include/kvikio/file_handle.hpp +++ b/cpp/include/kvikio/file_handle.hpp @@ -62,23 +62,20 @@ class FileHandle { */ bool is_compat_mode_preferred_for_async(CompatMode requested_compat_mode) { - if (!defaults::is_compat_mode_preferred(requested_compat_mode)) { - if (!is_batch_and_stream_available()) { - if (requested_compat_mode == CompatMode::AUTO) { return true; } - throw std::runtime_error("Missing cuFile batch or stream library symbol."); - } - - // When checking for availability, we also check if cuFile's config file exist. This is - // because even when the stream API is available, it doesn't work if no config file exist. - if (config_path().empty()) { - if (requested_compat_mode == CompatMode::AUTO) { return true; } - throw std::runtime_error("Missing cuFile configuration file."); - } - - return false; + if (defaults::is_compat_mode_preferred(requested_compat_mode)) { return true; } + + if (!is_stream_api_available()) { + if (requested_compat_mode == CompatMode::AUTO) { return true; } + throw std::runtime_error("Missing the cuFile stream api."); } - return true; + // When checking for availability, we also check if cuFile's config file exists. This is + // because even when the stream API is available, it doesn't work if no config file exists. + if (config_path().empty()) { + if (requested_compat_mode == CompatMode::AUTO) { return true; } + throw std::runtime_error("Missing cuFile configuration file."); + } + return false; } public: @@ -670,7 +667,7 @@ class FileHandle { */ [[nodiscard]] bool is_compat_mode_preferred_for_async() const noexcept { - static bool is_extra_symbol_available = is_batch_and_stream_available(); + static bool is_extra_symbol_available = is_stream_api_available(); static bool is_config_path_empty = config_path().empty(); return is_compat_mode_preferred() || !is_extra_symbol_available || is_config_path_empty; } diff --git a/cpp/include/kvikio/remote_handle.hpp b/cpp/include/kvikio/remote_handle.hpp index 8ac2798f31..e1b152b23c 100644 --- a/cpp/include/kvikio/remote_handle.hpp +++ b/cpp/include/kvikio/remote_handle.hpp @@ -41,7 +41,7 @@ namespace detail { * @note Is not thread-safe. */ class BounceBufferH2D { - CUstream _stream; // The CUDA steam to use. + CUstream _stream; // The CUDA stream to use. CUdeviceptr _dev; // The output device buffer. AllocRetain::Alloc _host_buffer; // The host buffer to bounce data on. std::ptrdiff_t _dev_offset{0}; // Number of bytes written to `_dev`. diff --git a/cpp/include/kvikio/shim/cufile.hpp b/cpp/include/kvikio/shim/cufile.hpp index 7f12c29c3d..5194d45e74 100644 --- a/cpp/include/kvikio/shim/cufile.hpp +++ b/cpp/include/kvikio/shim/cufile.hpp @@ -57,8 +57,11 @@ class cuFileAPI { decltype(cuFileDriverOpen)* DriverOpen{nullptr}; decltype(cuFileDriverClose)* DriverClose{nullptr}; + // Don't call `GetVersion` directly, use `cuFileAPI::instance().version`. + decltype(cuFileGetVersion)* GetVersion{nullptr}; + public: - bool stream_available = false; + int version{0}; private: #ifdef KVIKIO_CUFILE_FOUND @@ -88,33 +91,39 @@ class cuFileAPI { get_symbol(DriverSetMaxCacheSize, lib, KVIKIO_STRINGIFY(cuFileDriverSetMaxCacheSize)); get_symbol(DriverSetMaxPinnedMemSize, lib, KVIKIO_STRINGIFY(cuFileDriverSetMaxPinnedMemSize)); -#ifdef KVIKIO_CUFILE_BATCH_API_FOUND - get_symbol(BatchIOSetUp, lib, KVIKIO_STRINGIFY(cuFileBatchIOSetUp)); - get_symbol(BatchIOSubmit, lib, KVIKIO_STRINGIFY(cuFileBatchIOSubmit)); - get_symbol(BatchIOGetStatus, lib, KVIKIO_STRINGIFY(cuFileBatchIOGetStatus)); - get_symbol(BatchIOCancel, lib, KVIKIO_STRINGIFY(cuFileBatchIOCancel)); - get_symbol(BatchIODestroy, lib, KVIKIO_STRINGIFY(cuFileBatchIODestroy)); -#endif - -#ifdef KVIKIO_CUFILE_STREAM_API_FOUND - get_symbol(ReadAsync, lib, KVIKIO_STRINGIFY(cuFileReadAsync)); - get_symbol(WriteAsync, lib, KVIKIO_STRINGIFY(cuFileWriteAsync)); - get_symbol(StreamRegister, lib, KVIKIO_STRINGIFY(cuFileStreamRegister)); - get_symbol(StreamDeregister, lib, KVIKIO_STRINGIFY(cuFileStreamDeregister)); +#ifdef KVIKIO_CUFILE_VERSION_API_FOUND try { - void* s{}; - get_symbol(s, lib, "cuFileReadAsync"); - stream_available = true; - } catch (const std::runtime_error&) { + get_symbol(GetVersion, lib, KVIKIO_STRINGIFY(cuFileGetVersion)); + int ver; + CUfileError_t const error = GetVersion(&ver); + if (error.err == CU_FILE_SUCCESS) { version = ver; } + } catch (std::runtime_error const&) { } #endif + // Some symbols were introduced in later versions, so version guards are required. + // Note: `version` is 0 for cuFile versions prior to v1.8 because `cuFileGetVersion` + // did not exist. As a result, the batch and stream APIs are not loaded in versions + // 1.6 and 1.7, respectively, even though they are available. This trade-off is made + // for improved robustness. + if (version >= 1060) { + get_symbol(BatchIOSetUp, lib, KVIKIO_STRINGIFY(cuFileBatchIOSetUp)); + get_symbol(BatchIOSubmit, lib, KVIKIO_STRINGIFY(cuFileBatchIOSubmit)); + get_symbol(BatchIOGetStatus, lib, KVIKIO_STRINGIFY(cuFileBatchIOGetStatus)); + get_symbol(BatchIOCancel, lib, KVIKIO_STRINGIFY(cuFileBatchIOCancel)); + get_symbol(BatchIODestroy, lib, KVIKIO_STRINGIFY(cuFileBatchIODestroy)); + } + if (version >= 1070) { + get_symbol(ReadAsync, lib, KVIKIO_STRINGIFY(cuFileReadAsync)); + get_symbol(WriteAsync, lib, KVIKIO_STRINGIFY(cuFileWriteAsync)); + get_symbol(StreamRegister, lib, KVIKIO_STRINGIFY(cuFileStreamRegister)); + get_symbol(StreamDeregister, lib, KVIKIO_STRINGIFY(cuFileStreamDeregister)); + } + // cuFile is supposed to open and close the driver automatically but // because of a bug in cuFile v1.4 (CUDA v11.8) it sometimes segfaults: // . - // We use the stream API as a version indicator of cuFile since it was introduced - // in cuFile v1.7 (CUDA v12.2). - if (!stream_available) { driver_open(); } + if (version < 1050) { driver_open(); } } // Notice, we have to close the driver at program exit (if we opened it) even though we are @@ -124,7 +133,7 @@ class cuFileAPI { // [1] ~cuFileAPI() { - if (!stream_available) { driver_close(); } + if (version < 1050) { driver_close(); } } #else cuFileAPI() { throw std::runtime_error("KvikIO not compiled with cuFile.h"); } @@ -205,25 +214,49 @@ inline bool is_cufile_available() } /** - * @brief Check if cuFile's batch and stream API is available + * @brief Get cufile version (or zero if older than v1.8). * - * Technically, the batch API is available in CUDA 12.1 but since there is no good - * way to check CUDA version using the driver API, we check for the existing of the - * `cuFileReadAsync` symbol, which is defined in CUDA 12.2+. + * The version is returned as (1000*major + 10*minor). E.g., cufile v1.8.0 would + * be represented by 1080. * - * @return The boolean answer + * Notice, this is not the version of the CUDA toolkit. cufile is part of the + * toolkit but follows its own version scheme. + * + * @return The version (1000*major + 10*minor) or zero if older than 1080. */ -#if defined(KVIKIO_CUFILE_STREAM_API_FOUND) && defined(KVIKIO_CUFILE_STREAM_API_FOUND) -inline bool is_batch_and_stream_available() noexcept +#ifdef KVIKIO_CUFILE_FOUND +inline int cufile_version() { try { - return is_cufile_available() && cuFileAPI::instance().stream_available; - } catch (const std::runtime_error&) { - return false; + return cuFileAPI::instance().version; + } catch (std::runtime_error const&) { + return 0; } } #else -constexpr bool is_batch_and_stream_available() { return false; } +constexpr int cufile_version() { return 0; } #endif +/** + * @brief Check if cuFile's batch API is available. + * + * Since `cuFileGetVersion()` first became available in cufile v1.8 (CTK v12.3), + * this function returns false for versions older than v1.8 even though the batch + * API became available in v1.6. + * + * @return The boolean answer + */ +inline bool is_batch_api_available() noexcept { return cufile_version() >= 1060; } + +/** + * @brief Check if cuFile's stream (async) API is available. + * + * Since `cuFileGetVersion()` first became available in cufile v1.8 (CTK v12.3), + * this function returns false for versions older than v1.8 even though the stream + * API became available in v1.7. + * + * @return The boolean answer + */ +inline bool is_stream_api_available() noexcept { return cufile_version() >= 1070; } + } // namespace kvikio diff --git a/cpp/include/kvikio/shim/cufile_h_wrapper.hpp b/cpp/include/kvikio/shim/cufile_h_wrapper.hpp index 33c3fee9a2..1c13d2d8a1 100644 --- a/cpp/include/kvikio/shim/cufile_h_wrapper.hpp +++ b/cpp/include/kvikio/shim/cufile_h_wrapper.hpp @@ -75,7 +75,7 @@ CUfileError_t cuFileDriverSetMaxPinnedMemSize(...); #endif -// If the Batch API isn't defined, we define some of the data types here. +// If some cufile APIs aren't defined, we define some of the data types here. // Notice, this doesn't need to be ABI compatible with the cufile definitions and // the lack of definitions is not a problem because the linker will never look for // these symbols because the "real" function calls are made through the shim instance. @@ -105,10 +105,13 @@ CUfileError_t cuFileBatchIOCancel(...); CUfileError_t cuFileBatchIODestroy(...); #endif -// If the Stream API isn't defined, we define some of the data types here. #ifndef KVIKIO_CUFILE_STREAM_API_FOUND CUfileError_t cuFileReadAsync(...); CUfileError_t cuFileWriteAsync(...); CUfileError_t cuFileStreamRegister(...); CUfileError_t cuFileStreamDeregister(...); #endif + +#ifndef KVIKIO_CUFILE_VERSION_API_FOUND +CUfileError_t cuFileGetVersion(...); +#endif diff --git a/dependencies.yaml b/dependencies.yaml index 32d4f73c79..6a715e08d7 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -24,12 +24,14 @@ files: output: none includes: - cuda_version + - test_libkvikio - test_cpp test_python: output: none includes: - cuda_version - py_version + - test_kvikio - test_python checks: output: none @@ -42,6 +44,7 @@ files: - cuda_version - docs - py_version + - test_kvikio py_build_kvikio: output: pyproject pyproject_dir: python/kvikio @@ -366,6 +369,18 @@ dependencies: # See https://github.com/zarr-developers/numcodecs/pull/475 - numcodecs !=0.12.0 - packaging + test_libkvikio: + common: + - output_types: conda + packages: + - libkvikio==25.2.*,>=0.0.0a0 + - libkvikio-tests==25.2.*,>=0.0.0a0 + test_kvikio: + common: + - output_types: conda + packages: + - libkvikio==25.2.*,>=0.0.0a0 + - kvikio==25.2.*,>=0.0.0a0 test_cpp: common: - output_types: conda @@ -375,7 +390,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - &dask dask>=2022.05.2 + - rapids-dask-dependency==25.2.*,>=0.0.0a0 - pytest - pytest-cov - rangehttpserver @@ -392,10 +407,10 @@ dependencies: - matrix: cuda: "12.*" packages: - - cuda-python>=12.0,<13.0a0,<=12.6.0 + - cuda-python>=12.6.2,<13.0a0 - matrix: # All CUDA 11 versions packages: - - cuda-python>=11.7.1,<12.0a0,<=11.8.3 + - cuda-python>=11.8.5,<12.0a0 test_java: common: - output_types: conda diff --git a/python/kvikio/kvikio/_lib/cufile_driver.pyx b/python/kvikio/kvikio/_lib/cufile_driver.pyx index 29302a0104..0488eb3b20 100644 --- a/python/kvikio/kvikio/_lib/cufile_driver.pyx +++ b/python/kvikio/kvikio/_lib/cufile_driver.pyx @@ -9,10 +9,15 @@ from libcpp cimport bool cdef extern from "" nogil: + cdef int cpp_libcufile_version "kvikio::cufile_version"() except + cdef void cpp_driver_open "kvikio::cuFileAPI::instance().driver_open"() except + cdef void cpp_driver_close "kvikio::cuFileAPI::instance().driver_close"() except + +def libcufile_version() -> int: + return cpp_libcufile_version() + + def driver_open(): cpp_driver_open() diff --git a/python/kvikio/kvikio/benchmarks/utils.py b/python/kvikio/kvikio/benchmarks/utils.py index 23c7731f24..fa25c361a4 100644 --- a/python/kvikio/kvikio/benchmarks/utils.py +++ b/python/kvikio/kvikio/benchmarks/utils.py @@ -27,6 +27,7 @@ def drop_vm_cache() -> None: def pprint_sys_info() -> None: """Pretty print system information""" + version = kvikio.cufile_driver.libcufile_version() props = kvikio.cufile_driver.DriverProperties() try: import pynvml @@ -41,6 +42,10 @@ def pprint_sys_info() -> None: gpu_name = f"{pynvml.nvmlDeviceGetName(dev)} (dev #0)" mem_total = format_bytes(pynvml.nvmlDeviceGetMemoryInfo(dev).total) bar1_total = format_bytes(pynvml.nvmlDeviceGetBAR1MemoryInfo(dev).bar1Total) + if version == (0, 0): + libcufile_version = "unknown (earlier than cuFile 1.8)" + else: + libcufile_version = f"{version[0]}.{version[1]}" gds_version = "N/A (Compatibility Mode)" if props.is_gds_available: gds_version = f"v{props.major_version}.{props.minor_version}" @@ -61,6 +66,7 @@ def pprint_sys_info() -> None: print(f"GPU | {gpu_name}") print(f"GPU Memory Total | {mem_total}") print(f"BAR1 Memory Total | {bar1_total}") + print(f"libcufile version | {libcufile_version}") print(f"GDS driver | {gds_version}") print(f"GDS config.json | {gds_config_json_path}") diff --git a/python/kvikio/kvikio/cufile_driver.py b/python/kvikio/kvikio/cufile_driver.py index e78242a514..fb32be347a 100644 --- a/python/kvikio/kvikio/cufile_driver.py +++ b/python/kvikio/kvikio/cufile_driver.py @@ -2,6 +2,7 @@ # See file LICENSE for terms. import atexit +from typing import Tuple from kvikio._lib import cufile_driver # type: ignore @@ -10,6 +11,26 @@ DriverProperties = cufile_driver.DriverProperties +def libcufile_version() -> Tuple[int, int]: + """Get the libcufile version. + + Returns (0, 0) for cuFile versions prior to v1.8. + + Notes + ----- + This is not the version of the CUDA toolkit. cufile is part of the + toolkit but follows its own version scheme. + + Returns + ------- + The version as a tuple (MAJOR, MINOR). + """ + v = cufile_driver.libcufile_version() + # Convert the integer version like 1080 to (1, 8). + major, minor = divmod(v, 1000) + return (major, minor // 10) + + def driver_open() -> None: """Open the cuFile driver diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index 68d003c4d0..c9a9ab80d2 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -41,12 +41,12 @@ classifiers = [ [project.optional-dependencies] test = [ "boto3>=1.21.21", - "cuda-python>=11.7.1,<12.0a0,<=11.8.3", - "dask>=2022.05.2", + "cuda-python>=11.8.5,<12.0a0", "moto[server]>=4.0.8", "pytest", "pytest-cov", "rangehttpserver", + "rapids-dask-dependency==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] diff --git a/python/kvikio/tests/test_cufile_driver.py b/python/kvikio/tests/test_cufile_driver.py index 0a64bf0952..a1dc3a6454 100644 --- a/python/kvikio/tests/test_cufile_driver.py +++ b/python/kvikio/tests/test_cufile_driver.py @@ -6,6 +6,12 @@ import kvikio.cufile_driver +def test_version(): + major, minor = kvikio.cufile_driver.libcufile_version() + assert major >= 0 + assert minor >= 0 + + @pytest.mark.cufile def test_open_and_close(): kvikio.cufile_driver.driver_open()