diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9b5301d88..7aa4074c6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -72,7 +72,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@test_pypi_trusted_publish with: - build_type: ${{ inputs.build_type || 'branch' }} + build_type: pull-request branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} diff --git a/ci/_rapids-wheels-prepare.sh b/ci/_rapids-wheels-prepare.sh new file mode 100755 index 000000000..6be334f51 --- /dev/null +++ b/ci/_rapids-wheels-prepare.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# A utility script to download and untar Python wheel packages from S3. +# Positional Arguments: +# 1) wheel name +# 2) package type (one of: 'cpp', 'python'). If not provided, defaults to 'python' for compatibility with older code where python was the only behavior. +# +# [usage] +# +# # download and untar any wheels found in CI artifacts with names like '*wheel_python_sparkly-unicorn*.tar.gz' +# _rapids-wheels-prepare 'sparkly-unicorn' 'python' +# +set -eou pipefail +source rapids-constants +if [ -z "$1" ]; then + rapids-echo-stderr "Must specify input arguments: WHEEL_NAME" + exit 1 +fi +WHEEL_NAME="$1" +PKG_TYPE="${2:-python}" +case "${PKG_TYPE}" in + cpp) + ;; + python) + ;; + *) + rapids-echo-stderr 'Pass one of the following package types: "cpp", "python"' + exit 1 + ;; +esac +WHEEL_SEARCH_KEY="wheel_${PKG_TYPE}_${WHEEL_NAME}" +WHEEL_DIR="./dist" +mkdir -p "${WHEEL_DIR}" +S3_PATH=$(rapids-s3-path) +BUCKET_PREFIX=${S3_PATH/s3:\/\/${RAPIDS_DOWNLOADS_BUCKET}\//} # removes s3://rapids-downloads/ from s3://rapids-downloads/ci/rmm/... +# shellcheck disable=SC2016 +WHEEL_TARBALLS=$( + set -eo pipefail; + aws \ + --output json \ + s3api list-objects \ + --bucket "${RAPIDS_DOWNLOADS_BUCKET}" \ + --prefix "${BUCKET_PREFIX}" \ + --page-size 100 \ + --query "Contents[?contains(Key, '${WHEEL_SEARCH_KEY}')].Key" \ + | jq -c +) +export WHEEL_TARBALLS +# first untar them all +for OBJ in $(jq -nr 'env.WHEEL_TARBALLS | fromjson | .[]'); do + FILENAME=$(basename "${OBJ}") + S3_URI="${S3_PATH}${FILENAME}" + rapids-echo-stderr "Untarring ${S3_URI} into ${WHEEL_DIR}" + aws s3 cp --only-show-errors "${S3_URI}" - | tar xzf - -C "${WHEEL_DIR}" +done diff --git a/ci/mint-pypi-token.sh b/ci/mint-pypi-token.sh new file mode 100755 index 000000000..200f2baaa --- /dev/null +++ b/ci/mint-pypi-token.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# retrieve the ambient OIDC token +resp=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ +"$ACTIONS_ID_TOKEN_REQUEST_URL&audience=pypi") +oidc_token=$(jq -r '.value' <<< "${resp}") + +# exchange the OIDC token for an API token +resp=$(curl -X POST https://test.pypi.org/_/oidc/mint-token -d "{\"token\": \"${oidc_token}\"}") +api_token=$(jq -r '.token' <<< "${resp}") + +# mask the newly minted API token, so that we don't accidentally leak it +echo "::add-mask::${api_token}" + +# see the next step in the workflow for an example of using this step output +echo "api-token=${api_token}" >> "${GITHUB_OUTPUT}" diff --git a/ci/rapids-wheels-anaconda.sh b/ci/rapids-wheels-anaconda.sh new file mode 100755 index 000000000..46eedc295 --- /dev/null +++ b/ci/rapids-wheels-anaconda.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# A utility script to upload Python wheel packages to Anaconda repository using anaconda-client. +# Positional Arguments: +# 1) wheel name +# 2) package type (one of: 'cpp', 'python'). If not provided, defaults to 'python' for compatibility with older code where python was the only behavior. +# +# [usage] +# +# # upload any wheels found in CI artifacts with names like '*wheel_python_sparkly-unicorn*.tar.gz' +# rapids-wheels-anaconda 'sparkly-unicorn' 'python' +# +set -eou pipefail +source rapids-constants +export RAPIDS_SCRIPT_NAME="rapids-wheels-anaconda" +WHEEL_NAME="$1" +PKG_TYPE="${2:-python}" +WHEEL_DIR="./dist" +_rapids-wheels-prepare "${WHEEL_NAME}" "${PKG_TYPE}" +export RAPIDS_RETRY_SLEEP=180 +# shellcheck disable=SC2086 +# rapids-retry anaconda \ +# -t "${RAPIDS_CONDA_TOKEN}" \ +# upload \ +# --skip-existing \ +# --no-progress \ +# "${WHEEL_DIR}"/*.whl + +echo "all good!" diff --git a/ci/rapids-wheels-pypi.sh b/ci/rapids-wheels-pypi.sh new file mode 100755 index 000000000..58730d075 --- /dev/null +++ b/ci/rapids-wheels-pypi.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# A utility script to upload Python wheel packages to PyPI repository using an OIDC token. +# Positional Arguments: +# 1) wheel name +# 2) package type (one of: 'cpp', 'python'). If not provided, defaults to 'python' for compatibility with older code where python was the only behavior. +# +# [usage] +# +# # upload any wheels found in CI artifacts with names like '*wheel_python_sparkly-unicorn*.tar.gz' +# rapids-wheels-pypi 'sparkly-unicorn' 'python' +# +set -eou pipefail +source rapids-constants +export RAPIDS_SCRIPT_NAME="rapids-wheels-pypi" +WHEEL_NAME="$1" +PKG_TYPE="${2:-python}" +WHEEL_DIR="./dist" +_rapids-wheels-prepare "${WHEEL_NAME}" "${PKG_TYPE}" + +if [ -z "${PYPI_TOKEN}" ]; then + rapids-echo-stderr "Must specify input arguments: PYPI_TOKEN" + exit 1 +fi + +# shellcheck disable=SC2086 +rapids-retry python -m twine \ + upload \ + --repository testpypi \ + --disable-progress-bar \ + --non-interactive \ + --skip-existing \ + "${WHEEL_DIR}"/*.whl + +echo ""