Merge branch 'main' into parquet/support-write-bloom-filter

# Conflicts: # cpp/src/parquet/file_writer.cc # cpp/src/parquet/properties.h
mapleFU · Feb 5, 2025 · 0898466 · 0898466
2 parents c5b1fb1 + 2cbc7d6
commit 0898466
Show file tree

Hide file tree

Showing 315 changed files with 5,966 additions and 4,767 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -23,7 +23,6 @@ github:
     - benibus
     - jbonofre
     - js8544
-    - laurentgo
     - vibhatha
     - ZhangHuiGui
 

diff --git a/.env b/.env
@@ -62,7 +62,7 @@ HDFS=3.2.1
 JDK=11
 KARTOTHEK=latest
 # LLVM 12 and GCC 11 reports -Wmismatched-new-delete.
-LLVM=14
+LLVM=18
 MAVEN=3.8.7
 NODE=18
 NUMBA=latest
@@ -89,17 +89,17 @@ TZ=UTC
 # Used through docker-compose.yml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
 # docker tags more readable.
-VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01"    # 2024.04.26 Release
+VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625"    # 2024.06.15 Release
 
 # This must be updated when we update
-# ci/docker/python-*-windows-*.dockerfile.
+# ci/docker/python-*-windows-*.dockerfile or the vcpkg config.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-01-08
-PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-01-08
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-02-03
+PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-01-27
 
 # Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan".
 # See https://github.com/conan-io/conan-docker-tools#readme and
 # https://hub.docker.com/u/conanio for available images.
-CONAN_BASE=gcc10
-CONAN_VERSION=1.62.0
+CONAN_BASE=gcc11-ubuntu16.04
+CONAN_VERSION=2.12.1
diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml
@@ -65,7 +65,7 @@ jobs:
         shell: bash
         run: git branch $ARCHERY_DEFAULT_BRANCH origin/$ARCHERY_DEFAULT_BRANCH || true
       - name: Setup Python
-        uses: actions/setup-python@v5.3.0
+        uses: actions/setup-python@v5.4.0
         with:
           python-version: '3.9'
       - name: Install pygit2 binary wheel

diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml
@@ -42,7 +42,7 @@ jobs:
           # fetch the tags for version number generation
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: 3.12
       - name: Install Archery and Crossbow dependencies

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -69,65 +69,37 @@ env:
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
-  docker-targets:
-    name: Docker targets
-    runs-on: ubuntu-latest
-    outputs:
-      targets: ${{ steps.detect-targets.outputs.targets }}
-    steps:
-      - name: Detect targets
-        id: detect-targets
-        run: |
-          echo "targets<<JSON" >> "$GITHUB_OUTPUT"
-          echo "[" >> "$GITHUB_OUTPUT"
-          cat <<JSON >> "$GITHUB_OUTPUT"
-          {
-            "arch": "amd64",
-            "clang-tools": "14",
-            "image": "conda-cpp",
-            "llvm": "14",
-            "runs-on": "ubuntu-latest",
-            "simd-level": "AVX2",
-            "title": "AMD64 Conda C++ AVX2",
-            "ubuntu": "22.04"
-          },
-          {
-            "arch": "amd64",
-            "clang-tools": "14",
-            "image": "ubuntu-cpp-sanitizer",
-            "llvm": "14",
-            "runs-on": "ubuntu-latest",
-            "title": "AMD64 Ubuntu 22.04 C++ ASAN UBSAN",
-            "ubuntu": "22.04"
-          }
-          JSON
-          if [ "$GITHUB_REPOSITORY_OWNER" = "apache" ]; then
-            echo "," >> "$GITHUB_OUTPUT"
-            cat <<JSON >> "$GITHUB_OUTPUT"
-          {
-            "arch": "arm64v8",
-            "clang-tools": "10",
-            "image": "ubuntu-cpp",
-            "llvm": "10",
-            "runs-on": ["self-hosted", "arm", "linux"],
-            "title": "ARM64 Ubuntu 20.04 C++",
-            "ubuntu": "20.04"
-          }
-          JSON
-          fi
-          echo "]" >> "$GITHUB_OUTPUT"
-          echo "JSON" >> "$GITHUB_OUTPUT"
-
   docker:
     name: ${{ matrix.title }}
-    needs: docker-targets
     runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 75
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
+        include:
+          - arch: amd64
+            clang-tools: 14
+            image: conda-cpp
+            llvm: 14
+            runs-on: ubuntu-latest
+            simd-level: AVX2
+            title: AMD64 Conda C++ AVX2
+            ubuntu: 22.04
+          - arch: amd64
+            clang-tools: 14
+            image: ubuntu-cpp-sanitizer
+            llvm: 14
+            runs-on: ubuntu-latest
+            title: AMD64 Ubuntu 22.04 C++ ASAN UBSAN
+            ubuntu: 22.04
+          - arch: arm64v8
+            clang-tools: 10
+            image: ubuntu-cpp
+            llvm: 10
+            runs-on: ubuntu-24.04-arm
+            title: ARM64 Ubuntu 20.04 C++
+            ubuntu: 20.04
     env:
       ARCH: ${{ matrix.arch }}
       ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
@@ -149,7 +121,7 @@ jobs:
       - name: Setup Python on hosted runner
         if: |
           matrix.runs-on == 'ubuntu-latest'
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: 3
       - name: Setup Python on self-hosted runner
@@ -262,7 +234,7 @@ jobs:
           $(brew --prefix bash)/bin/bash \
             ci/scripts/install_minio.sh latest ${ARROW_HOME}
       - name: Set up Python
-        uses: actions/setup-python@v5.3.0
+        uses: actions/setup-python@v5.4.0
         with:
           python-version: 3.12
       - name: Install Google Cloud Storage Testbench
@@ -481,7 +453,7 @@ jobs:
             https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z
           chmod +x /usr/local/bin/minio.exe
       - name: Set up Python
-        uses: actions/setup-python@v5.3.0
+        uses: actions/setup-python@v5.4.0
         id: python-install
         with:
           python-version: 3.9

diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml
@@ -54,11 +54,11 @@ jobs:
         dotnet: ['8.0.x']
     steps:
       - name: Install C#
-        uses: actions/setup-dotnet@v4.2.0
+        uses: actions/setup-dotnet@v4.3.0
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Setup Python
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: 3
       - name: Checkout Arrow
@@ -86,7 +86,7 @@ jobs:
         dotnet: ['8.0.x']
     steps:
       - name: Install C#
-        uses: actions/setup-dotnet@v4.2.0
+        uses: actions/setup-dotnet@v4.3.0
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Checkout Arrow
@@ -113,11 +113,11 @@ jobs:
         dotnet: ['8.0.x']
     steps:
       - name: Install C#
-        uses: actions/setup-dotnet@v4.2.0
+        uses: actions/setup-dotnet@v4.3.0
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Setup Python
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: 3.12
       - name: Checkout Arrow
@@ -133,3 +133,88 @@ jobs:
       - name: Test
         shell: bash
         run: ci/scripts/csharp_test.sh $(pwd)
+
+  package:
+    name: Package
+    # Branch or RC tag
+    if: github.ref_type != 'tag' || contains(github.ref_name, 'rc')
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout for utilities
+        if: github.ref_type == 'tag'
+        uses: actions/checkout@v4
+        with:
+          path: arrow
+      - name: Download source archive
+        if: github.ref_type == 'tag'
+        run: |
+          arrow/dev/release/utils-watch-gh-workflow.sh \
+            ${GITHUB_REF_NAME} \
+            release_candidate.yml
+          gh release download ${GITHUB_REF_NAME} \
+            --pattern "*.tar.gz" \
+            --repo ${GITHUB_REPOSITORY}
+          tar -xf *.tar.gz --strip-components=1
+          mv csharp/dummy.git .git
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Checkout
+        if: github.ref_type != 'tag'
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Prepare version
+        if: github.ref_type != 'tag'
+        run: |
+          # apache-arrow-20.0.0.dev-9-g758867f907 ->
+          #              20.0.0.dev-9-g758867f907 ->
+          #              20.0.0.dev-9             ->
+          #              20.0.0-dev-9
+          semver="$(git describe --tags | \
+                     sed -E \
+                         -e 's/^apache-arrow-//' \
+                         -e 's/-[^-]*$//' \
+                         -e 's/^([0-9]*\.[0-9]*\.[0-9])\./\1-/')"
+          sed -i'' -E -e \
+            "s/^    <Version>.+<\/Version>/    <Version>${semver}<\/Version>/" \
+            csharp/Directory.Build.props
+      - name: Setup Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3
+      - name: Setup Archery
+        run: |
+          python3 -m pip install -e 'dev/archery[docker]'
+      - name: Build
+        run: |
+          archery docker run ubuntu-csharp
+      - name: Prepare artifacts
+        run: |
+          shopt -s globstar
+          cp csharp/artifacts/**/*.{,s}nupkg ./
+          for artifact in *.{,s}nupkg; do
+            dev/release/utils-generate-checksum.sh "${artifact}"
+          done
+      - name: Upload
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: nuget
+          path: |
+            *.nupkg
+            *.sha256
+            *.sha512
+            *.snupkg
+      - name: Publish
+        if: github.ref_type == 'tag'
+        run: |
+          gh release upload ${GITHUB_REF_NAME} \
+            --repo ${GITHUB_REPOSITORY} \
+            *.nupkg \
+            *.sha256 \
+            *.sha512 \
+            *.snupkg
+        env:
+          GH_TOKEN: ${{ github.token }}
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -50,7 +50,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup Python
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: 3.12
       - name: Install pre-commit
@@ -109,22 +109,22 @@ jobs:
         with:
           fetch-depth: 0
       - name: Install Python
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: '3.12'
       - name: Install Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ruby
       - name: Install .NET
-        uses: actions/setup-dotnet@87b7050bc53ea08284295505d98d2aa94301e852 # v4.2.0
+        uses: actions/setup-dotnet@3951f0dfe7a07e2313ec93c75700083e2005cbab # v4.3.0
         with:
           dotnet-version: '8.0.x'
       - name: Install Dependencies
         shell: bash
         run: |
           gem install test-unit
-          pip install "cython>=0.29.31" setuptools pytest requests setuptools-scm
+          pip install "cython>=3" setuptools pytest requests setuptools-scm
       - name: Run Release Test
         env:
           ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -52,7 +52,7 @@ jobs:
           key: debian-docs-${{ hashFiles('cpp/**') }}
           restore-keys: debian-docs-
       - name: Setup Python
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: 3.12
       - name: Setup Archery

diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml
@@ -59,7 +59,7 @@ jobs:
           key: conda-docs-${{ hashFiles('cpp/**') }}
           restore-keys: conda-docs-
       - name: Setup Python
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: 3.12
       - name: Setup Archery

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -106,7 +106,7 @@ jobs:
           key: conda-${{ hashFiles('cpp/**') }}
           restore-keys: conda-
       - name: Setup Python
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: 3.12
       - name: Setup Archery

diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml
@@ -61,7 +61,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup Python
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: 3.12
       - name: Setup Archery