Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main'
Browse files Browse the repository at this point in the history
  • Loading branch information
guangzegu committed Feb 14, 2025
2 parents ed7b184 + f0e3832 commit 78857d9
Show file tree
Hide file tree
Showing 220 changed files with 6,153 additions and 2,277 deletions.
47 changes: 29 additions & 18 deletions .github/actions/build_cmake/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ inputs:
description: 'Enable GPU support.'
required: false
default: OFF
raft:
description: 'Enable RAFT support.'
cuvs:
description: 'Enable cuVS support.'
required: false
default: OFF
rocm:
Expand All @@ -23,49 +23,55 @@ runs:
uses: conda-incubator/setup-miniconda@v3
with:
python-version: '3.11'
miniconda-version: latest
miniforge-version: latest # ensures conda-forge channel is used.
channels: conda-forge
conda-remove-defaults: 'true'
# Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64.
# They are the same thing, just named differently.
architecture: ${{ runner.arch == 'ARM64' && 'aarch64' || runner.arch }}
- name: Configure build environment
shell: bash
run: |
# initialize Conda
conda config --set solver libmamba
# Ensure starting packages are from conda-forge.
conda list --show-channel-urls
conda update -y -q conda
echo "$CONDA/bin" >> $GITHUB_PATH
conda install -y -q python=3.11 cmake make swig numpy scipy pytest gflags
conda install -y -q python=3.11 cmake=3.26 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2
# install base packages for ARM64
if [ "${{ runner.arch }}" = "ARM64" ]; then
conda install -y -q -c conda-forge openblas gxx_linux-aarch64 sysroot_linux-aarch64
conda install -y -q -c conda-forge openblas=0.3 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
fi
# install base packages for X86_64
if [ "${{ runner.arch }}" = "X64" ]; then
# TODO: merge this with ARM64
conda install -y -q -c conda-forge gxx_linux-64 sysroot_linux-64
conda install -y -q mkl=2023 mkl-devel=2023
conda install -y -q -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17
conda install -y -q mkl=2022.2.1 mkl-devel=2022.2.1
fi
# no CUDA needed for ROCm so skip this
if [ "${{ inputs.rocm }}" = "ON" ]; then
:
# regular CUDA for GPU builds
elif [ "${{ inputs.gpu }}" = "ON" ] && [ "${{ inputs.raft }}" = "OFF" ]; then
conda install -y -q cuda-toolkit -c "nvidia/label/cuda-12.4.0"
# and CUDA from RAFT channel for RAFT builds
elif [ "${{ inputs.raft }}" = "ON" ]; then
conda install -y -q libraft=24.06 cuda-version=12.4 cuda-toolkit -c rapidsai -c "nvidia/label/cuda-12.4.0" -c conda-forge
elif [ "${{ inputs.gpu }}" = "ON" ] && [ "${{ inputs.cuvs }}" = "OFF" ]; then
conda install -y -q cuda-toolkit=12.4 -c "nvidia/label/cuda-12.4.0"
# and CUDA from cuVS channel for cuVS builds
elif [ "${{ inputs.cuvs }}" = "ON" ]; then
conda install -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c conda-forge
fi
# install test packages
conda install -y pytest
if [ "${{ inputs.rocm }}" = "ON" ]; then
: # skip torch install via conda, we need to install via pip to get
# ROCm-enabled version until it's supported in conda by PyTorch
elif [ "${{ inputs.gpu }}" = "ON" ]; then
conda install -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c nvidia/label/cuda-12.4.0
conda install -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c "nvidia/label/cuda-12.4.0"
else
conda install -y -q pytorch -c pytorch
conda install -y -q "pytorch<2.5" -c pytorch
fi
- name: ROCm - Install dependencies
if: inputs.rocm == 'ON'
Expand Down Expand Up @@ -102,7 +108,7 @@ runs:
sudo apt-get -qq clean >/dev/null
sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
- name: Symblink system dependencies
if: inputs.raft == 'ON' || inputs.rocm == 'ON'
if: inputs.rocm == 'ON'
shell: bash
run: |
# symblink system libraries for HIP compiler
Expand All @@ -119,7 +125,7 @@ runs:
-DBUILD_TESTING=ON \
-DBUILD_SHARED_LIBS=ON \
-DFAISS_ENABLE_GPU=${{ inputs.gpu }} \
-DFAISS_ENABLE_RAFT=${{ inputs.raft }} \
-DFAISS_ENABLE_CUVS=${{ inputs.cuvs }} \
-DFAISS_ENABLE_ROCM=${{ inputs.rocm }} \
-DFAISS_OPT_LEVEL=${{ inputs.opt_level }} \
-DFAISS_ENABLE_C_API=ON \
Expand Down Expand Up @@ -174,5 +180,10 @@ runs:
if: always()
uses: actions/upload-artifact@v4
with:
name: test-results-arch=${{ runner.arch }}-opt=${{ inputs.opt_level }}-gpu=${{ inputs.gpu }}-raft=${{ inputs.raft }}-rocm=${{ inputs.rocm }}
name: test-results-arch=${{ runner.arch }}-opt=${{ inputs.opt_level }}-gpu=${{ inputs.gpu }}-cuvs=${{ inputs.cuvs }}-rocm=${{ inputs.rocm }}
path: test-results
- name: Check installed packages channel
shell: bash
run: |
# Shows that all installed packages are from conda-forge.
conda list --show-channel-urls
47 changes: 31 additions & 16 deletions .github/actions/build_conda/action.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: Conda build
description: Builds FAISS inside a Conda environment and uploads to repository when label is provided.
description: Builds Faiss inside a Conda environment and uploads to repository when label is provided.
inputs:
label:
description: "The label to be used for uploads to Conda."
Expand All @@ -9,8 +9,8 @@ inputs:
description: "CUDA toolkit version to use."
default: ""
required: false
raft:
description: "Enable RAFT support."
cuvs:
description: "Enable cuVS support."
default: ""
required: false
runs:
Expand All @@ -30,12 +30,22 @@ runs:
uses: conda-incubator/setup-miniconda@v3
with:
python-version: '3.11'
miniconda-version: latest
miniforge-version: latest # ensures conda-forge channel is used.
channels: conda-forge
conda-remove-defaults: 'true'
# Set to runner.arch=aarch64 if we're on arm64 because
# there's no miniforge ARM64 package, just aarch64.
# They are the same thing, just named differently.
# However there is an ARM64 for macOS, so exclude that.
architecture: ${{ (runner.arch == 'ARM64' && runner.os != 'macOS') && 'aarch64' || runner.arch }}
- name: Install conda build tools
shell: ${{ steps.choose_shell.outputs.shell }}
run: |
conda update -y -q conda
conda install -y -q conda-build
# Ensure starting packages are from conda-forge.
conda list --show-channel-urls
conda install -y -q "conda!=24.11.0"
conda install -y -q "conda-build!=24.11.0"
conda list --show-channel-urls
- name: Enable anaconda uploads
if: inputs.label != ''
shell: ${{ steps.choose_shell.outputs.shell }}
Expand All @@ -59,34 +69,39 @@ runs:
run: |
conda build faiss --user pytorch --label ${{ inputs.label }} -c pytorch
- name: Conda build (GPU)
if: inputs.label == '' && inputs.cuda != '' && inputs.raft == ''
if: inputs.label == '' && inputs.cuda != '' && inputs.cuvs == ''
shell: ${{ steps.choose_shell.outputs.shell }}
working-directory: conda
run: |
conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
-c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia
- name: Conda build (GPU) w/ anaconda upload
if: inputs.label != '' && inputs.cuda != '' && inputs.raft == ''
if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs == ''
shell: ${{ steps.choose_shell.outputs.shell }}
working-directory: conda
env:
PACKAGE_TYPE: ${{ inputs.label }}
run: |
conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
--user pytorch --label ${{ inputs.label }} -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia
- name: Conda build (GPU w/ RAFT)
if: inputs.label == '' && inputs.cuda != '' && inputs.raft != ''
- name: Conda build (GPU w/ cuVS)
if: inputs.label == '' && inputs.cuda != '' && inputs.cuvs != ''
shell: ${{ steps.choose_shell.outputs.shell }}
working-directory: conda
run: |
conda build faiss-gpu-raft --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
-c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia -c rapidsai -c rapidsai-nightly -c conda-forge
- name: Conda build (GPU w/ RAFT) w/ anaconda upload
if: inputs.label != '' && inputs.cuda != '' && inputs.raft != ''
conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
-c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia
- name: Conda build (GPU w/ cuVS) w/ anaconda upload
if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs != ''
shell: ${{ steps.choose_shell.outputs.shell }}
working-directory: conda
env:
PACKAGE_TYPE: ${{ inputs.label }}
run: |
conda build faiss-gpu-raft --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
--user pytorch --label ${{ inputs.label }} -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia -c rapidsai -c rapidsai-nightly -c conda-forge
conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
--user pytorch --label ${{ inputs.label }} -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia
- name: Check installed packages channel
shell: ${{ steps.choose_shell.outputs.shell }}
run: |
# Shows that all installed packages are from conda-forge.
conda list --show-channel-urls
3 changes: 1 addition & 2 deletions .github/workflows/autoclose.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Close inactive issues
name: Close Inactive Issues
on:
schedule:
- cron: "30 1 * * *"
Expand All @@ -13,7 +13,6 @@ jobs:
- uses: actions/stale@v5
with:
only-labels: autoclose
labels-to-remove-when-unstale: autoclose
days-before-issue-stale: 7
days-before-issue-close: 7
stale-issue-label: "stale"
Expand Down
169 changes: 169 additions & 0 deletions .github/workflows/build-pull-request.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
on:
workflow_call:
env:
OMP_NUM_THREADS: '10'
MKL_THREADING_LAYER: GNU
jobs:
format:
name: Format
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install clang-format
run: |
sudo apt-get update -y
sudo apt-get install -y wget
sudo apt install -y lsb-release wget software-properties-common gnupg
wget https://apt.llvm.org/llvm.sh
chmod u+x llvm.sh
sudo ./llvm.sh 18
sudo apt-get install -y git-core clang-format-18
- name: Verify clang-format
run: |
git ls-files | grep -E '\.(cpp|h|cu|cuh)$' | xargs clang-format-18 -i
if git diff --quiet; then
echo "Formatting OK!"
else
echo "Formatting not OK!"
echo "------------------"
git --no-pager diff --color
exit 1
fi
linux-x86_64-cmake:
name: Linux x86_64 (cmake)
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build and Test (cmake)
uses: ./.github/actions/build_cmake
linux-x86_64-AVX2-cmake:
name: Linux x86_64 AVX2 (cmake)
needs: linux-x86_64-cmake
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build and Test (cmake)
uses: ./.github/actions/build_cmake
with:
opt_level: avx2
linux-x86_64-AVX512-cmake:
name: Linux x86_64 AVX512 (cmake)
needs: linux-x86_64-cmake
runs-on: faiss-aws-m7i.large
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build and Test (cmake)
uses: ./.github/actions/build_cmake
with:
opt_level: avx512
linux-x86_64-AVX512_SPR-cmake:
name: Linux x86_64 AVX512_SPR (cmake)
needs: linux-x86_64-cmake
runs-on: faiss-aws-m7i.large
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build and Test (cmake)
uses: ./.github/actions/build_cmake
with:
opt_level: avx512_spr
linux-x86_64-GPU-cmake:
name: Linux x86_64 GPU (cmake)
needs: linux-x86_64-cmake
runs-on: 4-core-ubuntu-gpu-t4
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build and Test (cmake)
uses: ./.github/actions/build_cmake
with:
gpu: ON
linux-x86_64-GPU-w-CUVS-cmake:
name: Linux x86_64 GPU w/ cuVS (cmake)
needs: linux-x86_64-cmake
runs-on: 4-core-ubuntu-gpu-t4
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build and Test (cmake)
uses: ./.github/actions/build_cmake
with:
gpu: ON
cuvs: ON
linux-x86_64-GPU-w-ROCm-cmake:
name: Linux x86_64 GPU w/ ROCm (cmake)
needs: linux-x86_64-cmake
runs-on: faiss-amd-MI200
container:
image: ubuntu:22.04
options: --device=/dev/kfd --device=/dev/dri --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN
steps:
- name: Container setup
run: |
if [ -f /.dockerenv ]; then
apt-get update && apt-get install -y sudo && apt-get install -y git
git config --global --add safe.directory '*'
else
echo 'Skipping. Current job is not running inside a container.'
fi
- name: Checkout
uses: actions/checkout@v4
- name: Build and Test (cmake)
uses: ./.github/actions/build_cmake
with:
gpu: ON
rocm: ON
linux-arm64-SVE-cmake:
name: Linux arm64 SVE (cmake)
needs: linux-x86_64-cmake
runs-on: faiss-aws-r8g.large
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build and Test (cmake)
uses: ./.github/actions/build_cmake
with:
opt_level: sve
env:
# Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction
OPENBLAS_NUM_THREADS: '1'
linux-x86_64-conda:
name: Linux x86_64 (conda)
needs: linux-x86_64-cmake
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
fetch-tags: true
- name: Build and Package (conda)
uses: ./.github/actions/build_conda
windows-x86_64-conda:
name: Windows x86_64 (conda)
needs: linux-x86_64-cmake
runs-on: windows-2019
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
fetch-tags: true
- name: Build and Package (conda)
uses: ./.github/actions/build_conda
linux-arm64-conda:
name: Linux arm64 (conda)
needs: linux-x86_64-cmake
runs-on: 2-core-ubuntu-arm
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
fetch-tags: true
- name: Build and Package (conda)
uses: ./.github/actions/build_conda
Loading

0 comments on commit 78857d9

Please sign in to comment.