Merge branch 'main' into minor/remove-experimental-for-parquet-record…

…-reader
mapleFU · Jul 3, 2024 · 08265f2 · 08265f2
2 parents 79361dd + fd6bf5a
commit 08265f2
Show file tree

Hide file tree

Showing 2,919 changed files with 118,135 additions and 72,821 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -21,10 +21,12 @@ github:
   collaborators:
     - anjakefala
     - benibus
-    - danepitkin
     - davisusanibar
+    - jbonofre
     - js8544
     - vibhatha
+    - zanmato1984
+    - ZhangHuiGui
 
 notifications:
   commits:      commits@arrow.apache.org

diff --git a/.env b/.env
@@ -56,7 +56,7 @@ UBUNTU=20.04
 CLANG_TOOLS=14
 CUDA=11.2.2
 DASK=latest
-DOTNET=7.0
+DOTNET=8.0
 GCC_VERSION=""
 GO=1.21.8
 STATICCHECK=v0.4.7
@@ -71,12 +71,12 @@ NUMBA=latest
 NUMPY=latest
 PANDAS=latest
 PYTHON=3.8
-R=4.2
+R=4.4
 SPARK=master
 TURBODBC=latest
 
-# These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-gcc-release:latest
-R_IMAGE=ubuntu-gcc-release
+# These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-release:latest
+R_IMAGE=ubuntu-release
 R_ORG=rhub
 R_TAG=latest
 
@@ -86,21 +86,19 @@ ARROW_R_DEV=TRUE
 R_PRUNE_DEPS=FALSE
 TZ=UTC
 
-# Any non-empty string will install devtoolset-${DEVTOOLSET_VERSION}
-DEVTOOLSET_VERSION=
-
 # Used through docker-compose.yml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
 # docker tags more readable.
-VCPKG="a42af01b72c28a8e1d7b48107b33e4f286a55ef6"    # 2023.11.20 Release
+VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01"    # 2024.04.26 Release
 
 # This must be updated when we update
 # ci/docker/python-wheel-windows-vs2019.dockerfile.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-03-19
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-06-18
 
-# Use conanio/${CONAN} for "docker-compose run --rm conan". See
-# https://github.com/conan-io/conan-docker-tools#readme for available
-# images.
-CONAN=gcc10
+# Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker-compose run --rm conan".
+# See https://github.com/conan-io/conan-docker-tools#readme and
+# https://hub.docker.com/u/conanio for available images.
+CONAN_BASE=gcc10
+CONAN_VERSION=1.62.0
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -37,7 +37,7 @@
 /go/ @zeroshade
 /java/ @lidavidm
 /js/ @domoritz @trxcllnt
-/matlab/ @kevingurney @kou
+/matlab/ @kevingurney @kou @sgilmore10
 /python/pyarrow/_flight.pyx @lidavidm
 /python/pyarrow/**/*gandiva* @wjones127
 /r/ @paleolimbot @thisisnic

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -23,30 +23,35 @@ updates:
       interval: "weekly"
     commit-message:
       prefix: "MINOR: [CI] "
+    open-pull-requests-limit: 10
   - package-ecosystem: "gomod"
     directory: "/go/"
     schedule:
       interval: "weekly"
     commit-message:
       prefix: "MINOR: [Go] "
+    open-pull-requests-limit: 10
   - package-ecosystem: "maven"
     directory: "/java/"
     schedule:
       interval: "weekly"
     commit-message:
       prefix: "MINOR: [Java] "
+    open-pull-requests-limit: 10
   - package-ecosystem: "npm"
     directory: "/js/"
     schedule:
       interval: "monthly"
     commit-message:
       prefix: "MINOR: [JS] "
+    open-pull-requests-limit: 10
   - package-ecosystem: "nuget"
     directory: "/csharp/"
     schedule:
       interval: "weekly"
     commit-message:
       prefix: "MINOR: [C#] "
+    open-pull-requests-limit: 10
     ignore:
       - dependency-name: "Microsoft.Extensions.*"
         update-types:

diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml
@@ -32,7 +32,9 @@ on:
       - 'docker-compose.yml'
 
 env:
+  ARCHERY_DEBUG: 1
   ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+  ARCHERY_USE_DOCKER_CLI: 1
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
@@ -57,9 +59,9 @@ jobs:
         shell: bash
         run: git branch $ARCHERY_DEFAULT_BRANCH origin/$ARCHERY_DEFAULT_BRANCH || true
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v5.1.0
         with:
-          python-version: '3.12'
+          python-version: '3.9'
       - name: Install pygit2 binary wheel
         run: pip install pygit2 --only-binary pygit2
       - name: Install Archery, Crossbow- and Test Dependencies

diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml
@@ -41,7 +41,7 @@ jobs:
           # fetch the tags for version number generation
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
           python-version: 3.12
       - name: Install Archery and Crossbow dependencies

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -21,6 +21,7 @@ on:
   push:
     paths:
       - '.github/workflows/cpp.yml'
+      - 'ci/conda_env_*'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'
@@ -31,9 +32,11 @@ on:
       - 'cpp/**'
       - 'docker-compose.yml'
       - 'format/Flight.proto'
+      - 'testing'
   pull_request:
     paths:
       - '.github/workflows/cpp.yml'
+      - 'ci/conda_env_*'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'
@@ -44,6 +47,7 @@ on:
       - 'cpp/**'
       - 'docker-compose.yml'
       - 'format/Flight.proto'
+      - 'testing'
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
@@ -53,6 +57,7 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 
@@ -94,6 +99,7 @@ jobs:
             cat <<JSON >> "$GITHUB_OUTPUT"
           {
             "arch": "arm64v8",
+            "archery-use-docker-cli": "0",
             "clang-tools": "10",
             "image": "ubuntu-cpp",
             "llvm": "10",
@@ -118,6 +124,9 @@ jobs:
         include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
     env:
       ARCH: ${{ matrix.arch }}
+      # By default, use Docker CLI because docker-compose v1 is obsolete,
+      # except where the Docker client version is too old.
+      ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
       ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
       CLANG_TOOLS: ${{ matrix.clang-tools }}
       LLVM: ${{ matrix.llvm }}
@@ -237,7 +246,7 @@ jobs:
           $(brew --prefix bash)/bin/bash \
             ci/scripts/install_minio.sh latest ${ARROW_HOME}
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v5.1.0
         with:
           python-version: 3.12
       - name: Install Google Cloud Storage Testbench
@@ -458,7 +467,7 @@ jobs:
             https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z
           chmod +x /usr/local/bin/minio.exe
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v5.1.0
         with:
           python-version: 3.9
       - name: Install Google Cloud Storage Testbench

diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml
@@ -46,7 +46,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['7.0.x']
+        dotnet: ['8.0.x']
     steps:
       - name: Install C#
         uses: actions/setup-dotnet@v4
@@ -74,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['7.0.x']
+        dotnet: ['8.0.x']
     steps:
       - name: Install C#
         uses: actions/setup-dotnet@v4
@@ -94,19 +94,23 @@ jobs:
         run: ci/scripts/csharp_test.sh $(pwd)
 
   macos:
-    name: AMD64 macOS 11 C# ${{ matrix.dotnet }}
-    runs-on: macos-latest
+    name: AMD64 macOS 13 C# ${{ matrix.dotnet }}
+    runs-on: macos-13 # Pending https://github.com/pythonnet/pythonnet/issues/2396
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['7.0.x']
+        dotnet: ['8.0.x']
     steps:
       - name: Install C#
         uses: actions/setup-dotnet@v4
         with:
           dotnet-version: ${{ matrix.dotnet }}
+      - name: Setup Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: 3.12
       - name: Checkout Arrow
         uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -29,6 +29,10 @@ concurrency:
 permissions:
   contents: read
 
+env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
+
 jobs:
 
   lint:
@@ -42,7 +46,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup Python
-        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
           python-version: 3.12
       - name: Install pre-commit
@@ -101,22 +105,22 @@ jobs:
         with:
           fetch-depth: 0
       - name: Install Python
-        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
           python-version: '3.12'
       - name: Install Ruby
-        uses: ruby/setup-ruby@250fcd6a742febb1123a77a841497ccaa8b9e939 # v1.152.0
+        uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '2.7'
+          ruby-version: ruby
       - name: Install .NET
         uses: actions/setup-dotnet@4d6c8fcf3c8f7a60068d26b594648e99df24cee3 # v4.0.0
         with:
-          dotnet-version: '7.0.x'
+          dotnet-version: '8.0.x'
       - name: Install Dependencies
         shell: bash
         run: |
           gem install test-unit
-          pip install "cython>=0.29.31" setuptools six pytest jira
+          pip install "cython>=0.29.31" setuptools pytest jira setuptools-scm
       - name: Run Release Test
         env:
           ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -24,18 +24,20 @@ permissions:
   contents: read
 
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
 
   complete:
-    name: AMD64 Ubuntu 22.04 Complete Documentation
+    name: AMD64 Debian 12 Complete Documentation
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 150
     env:
-      UBUNTU: "22.04"
+      JDK: 17
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
@@ -48,10 +50,10 @@ jobs:
         uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
         with:
           path: .docker
-          key: ubuntu-docs-${{ hashFiles('cpp/**') }}
-          restore-keys: ubuntu-docs-
+          key: debian-docs-${{ hashFiles('cpp/**') }}
+          restore-keys: debian-docs-
       - name: Setup Python
-        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
           python-version: 3.12
       - name: Setup Archery
@@ -60,7 +62,8 @@ jobs:
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: archery docker run ubuntu-docs
+          JDK: 17
+        run: archery docker run debian-docs
       - name: Docker Push
         if: >-
           success() &&
@@ -71,4 +74,4 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push ubuntu-docs
+        run: archery docker push debian-docs
diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml
@@ -31,8 +31,10 @@ on:
 
 permissions:
   contents: read
-  
+
 env:
+  ARCHERY_DEBUG: 1
+  ARCHERY_USE_DOCKER_CLI: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 
@@ -57,7 +59,7 @@ jobs:
           key: conda-docs-${{ hashFiles('cpp/**') }}
           restore-keys: conda-docs-
       - name: Setup Python
-        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
           python-version: 3.12
       - name: Setup Archery