From e0ba3345abb26d8a32b3a79b54e1feff7361eec6 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 5 Feb 2025 15:18:37 -0500 Subject: [PATCH] Add `shellcheck` to pre-commit and fix warnings --- .pre-commit-config.yaml | 6 ++++++ ci/build_docs.sh | 9 ++++++--- ci/build_python.sh | 3 --- ci/build_rust.sh | 3 ++- ci/build_wheel.sh | 4 ++-- ci/build_wheel_cuvs.sh | 2 +- ci/build_wheel_libcuvs.sh | 2 -- ci/check_style.sh | 4 ++-- ci/checks/black_lists.sh | 16 ++++++++-------- ci/release/update-version.sh | 15 +++++++-------- ci/test_wheel_cuvs.sh | 2 +- ci/validate_wheel.sh | 5 ++--- 12 files changed, 37 insertions(+), 34 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1124d3814..bbfe73401 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -124,6 +124,12 @@ repos: hooks: - id: rapids-dependency-file-generator args: ["--clean"] + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck + args: ["--severity=warning"] + files: ^ci/ default_language_version: python: python3 diff --git a/ci/build_docs.sh b/ci/build_docs.sh index bce93c605..053e0200a 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -7,7 +7,8 @@ rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh RAPIDS_VERSION="$(rapids-version)" -export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" +RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" +export RAPIDS_VERSION_MAJOR_MINOR rapids-dependency-file-generator \ --output conda \ @@ -34,7 +35,8 @@ rapids-mamba-retry install \ "libcuvs=${RAPIDS_VERSION}" \ "cuvs=${RAPIDS_VERSION}" -export RAPIDS_DOCS_DIR="$(mktemp -d)" +RAPIDS_DOCS_DIR="$(mktemp -d)" +export RAPIDS_DOCS_DIR rapids-logger "Build CPP docs" pushd cpp/doxygen @@ -43,7 +45,8 @@ popd rapids-logger "Build Rust docs" pushd rust -export LIBCLANG_PATH=$(dirname $(find /opt/conda -name libclang.so | head -n 1)) +LIBCLANG_PATH=$(dirname "$(find /opt/conda -name libclang.so | head -n 1)") +export LIBCLANG_PATH cargo doc -p cuvs --no-deps popd diff --git a/ci/build_python.sh b/ci/build_python.sh index 3241a2c2b..d9ea64de9 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -15,9 +15,6 @@ rapids-print-env rapids-logger "Begin py build" -package_name="cuvs" -package_dir="python" - CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) version=$(rapids-generate-version) diff --git a/ci/build_rust.sh b/ci/build_rust.sh index 309501c32..0d0832031 100755 --- a/ci/build_rust.sh +++ b/ci/build_rust.sh @@ -25,7 +25,8 @@ rapids-print-env # we need to set up LIBCLANG_PATH to allow rust bindgen to work, # grab it from the conda env -export LIBCLANG_PATH=$(dirname $(find /opt/conda -name libclang.so | head -n 1)) +LIBCLANG_PATH=$(dirname "$(find /opt/conda -name libclang.so | head -n 1)") +export LIBCLANG_PATH echo "LIBCLANG_PATH=$LIBCLANG_PATH" rapids-logger "Downloading artifacts from previous jobs" diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 91cb50a50..f3b68c0b1 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -11,7 +11,7 @@ underscore_package_name=$(echo "${package_name}" | tr "-" "_") source rapids-configure-sccache source rapids-date-string -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" rapids-generate-version > ./VERSION @@ -50,4 +50,4 @@ sccache --show-adv-stats mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_type} final_dist +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 "${package_type}" final_dist diff --git a/ci/build_wheel_cuvs.sh b/ci/build_wheel_cuvs.sh index fb40d1459..8557d65d1 100755 --- a/ci/build_wheel_cuvs.sh +++ b/ci/build_wheel_cuvs.sh @@ -5,7 +5,7 @@ set -euo pipefail package_dir="python/cuvs" -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" # Downloads libcuvs wheels from this current build, # then ensures 'cuvs' wheel builds always use the 'libcuvs' just built in the same CI run. diff --git a/ci/build_wheel_libcuvs.sh b/ci/build_wheel_libcuvs.sh index 3bceed423..0d45a11b1 100755 --- a/ci/build_wheel_libcuvs.sh +++ b/ci/build_wheel_libcuvs.sh @@ -26,7 +26,5 @@ rapids-pip-retry install \ # 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) export PIP_NO_BUILD_ISOLATION=0 -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - ci/build_wheel.sh libcuvs ${package_dir} cpp ci/validate_wheel.sh ${package_dir} final_dist libcuvs diff --git a/ci/check_style.sh b/ci/check_style.sh index 952e94bf1..95f1e3742 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -18,8 +18,8 @@ conda activate checks RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" FORMAT_FILE_URL="https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION_MAJOR_MINOR}/cmake-format-rapids-cmake.json" export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json -mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) -wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL} +mkdir -p "$(dirname "${RAPIDS_CMAKE_FORMAT_FILE}")" +wget -O ${RAPIDS_CMAKE_FORMAT_FILE} "${FORMAT_FILE_URL}" # Run pre-commit checks pre-commit run --all-files --show-diff-on-failure diff --git a/ci/checks/black_lists.sh b/ci/checks/black_lists.sh index ed61527c3..ed9f3745b 100755 --- a/ci/checks/black_lists.sh +++ b/ci/checks/black_lists.sh @@ -20,16 +20,16 @@ set +H RETVAL=0 for black_listed in cudaDeviceSynchronize cudaMalloc cudaMallocManaged cudaFree cudaMallocHost cudaHostAlloc cudaFreeHost; do - TMP=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" $PR_TARGET_BRANCH | grep '^+' | grep -v '^+++' | grep "$black_listed"` + TMP=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" "$PR_TARGET_BRANCH" | grep '^+' | grep -v '^+++' | grep "$black_listed") if [ "$TMP" != "" ]; then - for filename in `git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$black_listed" $PR_TARGET_BRANCH`; do + for filename in $(git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$black_listed" "$PR_TARGET_BRANCH"); do basefilename=$(basename -- "$filename") filext="${basefilename##*.}" if [ "$filext" != "md" ] && [ "$filext" != "sh" ]; then - TMP2=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" $PR_TARGET_BRANCH -- $filename | grep '^+' | grep -v '^+++' | grep "$black_listed" | grep -vE "^\+[[:space:]]*/{2,}.*$black_listed"` + TMP2=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" "$PR_TARGET_BRANCH" -- "$filename" | grep '^+' | grep -v '^+++' | grep "$black_listed" | grep -vE "^\+[[:space:]]*/{2,}.*$black_listed") if [ "$TMP2" != "" ]; then echo "=== ERROR: black listed function call $black_listed added to $filename ===" - git --no-pager diff --ignore-submodules -w --minimal -S"$black_listed" $PR_TARGET_BRANCH -- $filename + git --no-pager diff --ignore-submodules -w --minimal -S"$black_listed" "$PR_TARGET_BRANCH" -- "$filename" echo "=== END ERROR ===" RETVAL=1 fi @@ -39,17 +39,17 @@ for black_listed in cudaDeviceSynchronize cudaMalloc cudaMallocManaged cudaFree done for cond_black_listed in cudaMemcpy cudaMemset; do - TMP=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" $PR_TARGET_BRANCH | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)"` + TMP=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" "$PR_TARGET_BRANCH" | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)") if [ "$TMP" != "" ]; then - for filename in `git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$cond_black_listed" $PR_TARGET_BRANCH`; do + for filename in $(git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$cond_black_listed" "$PR_TARGET_BRANCH"); do basefilename=$(basename -- "$filename") filext="${basefilename##*.}" if [ "$filext" != "md" ] && [ "$filext" != "sh" ]; then - TMP2=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" $PR_TARGET_BRANCH -- $filename | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)" | grep -vE "^\+[[:space:]]*/{2,}.*$cond_black_listed"` + TMP2=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" "$PR_TARGET_BRANCH" -- "$filename" | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)" | grep -vE "^\+[[:space:]]*/{2,}.*$cond_black_listed") if [ "$TMP2" != "" ]; then echo "=== ERROR: black listed function call $cond_black_listed added to $filename ===" - git --no-pager diff --ignore-submodules -w --minimal -S"$cond_black_listed" $PR_TARGET_BRANCH -- $filename + git --no-pager diff --ignore-submodules -w --minimal -S"$cond_black_listed" "$PR_TARGET_BRANCH" -- "$filename" echo "=== END ERROR ===" RETVAL=1 fi diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 3385d4b61..9bc916a87 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -13,17 +13,16 @@ NEXT_FULL_TAG=$1 # Get current version CURRENT_TAG=$(git tag --merged HEAD | grep -xE '^v.*' | sort --version-sort | tail -n 1 | tr -d 'v') -CURRENT_MAJOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[1]}') -CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}') -CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}') +CURRENT_MAJOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[1]}') +CURRENT_MINOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[2]}') CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} # Get . for next version -NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}') -NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}') -NEXT_PATCH=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[3]}') +NEXT_MAJOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[1]}') +NEXT_MINOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[2]}') +NEXT_PATCH=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[3]}') NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} -NEXT_UCXX_SHORT_TAG="$(curl -sL https://version.gpuci.io/rapids/${NEXT_SHORT_TAG})" +NEXT_UCXX_SHORT_TAG="$(curl -sL https://version.gpuci.io/rapids/"${NEXT_SHORT_TAG}")" # Need to distutils-normalize the original version NEXT_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_SHORT_TAG}'))") @@ -34,7 +33,7 @@ echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" # Inplace sed replace; workaround for Linux and Mac function sed_runner() { - sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak + sed -i.bak ''"$1"'' "$2" && rm -f "${2}".bak } sed_runner "s/set(RAPIDS_VERSION .*)/set(RAPIDS_VERSION \"${NEXT_SHORT_TAG}\")/g" examples/cmake/thirdparty/fetch_rapids.cmake diff --git a/ci/test_wheel_cuvs.sh b/ci/test_wheel_cuvs.sh index c2d02a30e..382a42b2f 100755 --- a/ci/test_wheel_cuvs.sh +++ b/ci/test_wheel_cuvs.sh @@ -4,7 +4,7 @@ set -euo pipefail mkdir -p ./dist -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" RAPIDS_PY_WHEEL_NAME="libcuvs_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libcuvs-dep RAPIDS_PY_WHEEL_NAME="cuvs_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 19d413fa2..5c2c5f2cf 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -6,7 +6,6 @@ set -euo pipefail package_dir=$1 wheel_dir_relative_path=$2 -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" cd "${package_dir}" @@ -14,10 +13,10 @@ rapids-logger "validate packages with 'pydistcheck'" pydistcheck \ --inspect \ - "$(echo ${wheel_dir_relative_path}/*.whl)" + "$(echo "${wheel_dir_relative_path}"/*.whl)" rapids-logger "validate packages with 'twine'" twine check \ --strict \ - "$(echo ${wheel_dir_relative_path}/*.whl)" + "$(echo "${wheel_dir_relative_path}"/*.whl)"