Merge branch 'main' into more-checks-for-read-levels

mapleFU · May 12, 2024 · 07c2386 · 07c2386
2 parents db35d8b + 1e3772c
commit 07c2386
Show file tree

Hide file tree

Showing 449 changed files with 11,828 additions and 4,680 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -21,10 +21,12 @@ github:
   collaborators:
     - anjakefala
     - benibus
-    - danepitkin
     - davisusanibar
+    - jbonofre
     - js8544
     - vibhatha
+    - zanmato1984
+    - ZhangHuiGui
 
 notifications:
   commits:      commits@arrow.apache.org

diff --git a/.env b/.env
@@ -56,7 +56,7 @@ UBUNTU=20.04
 CLANG_TOOLS=14
 CUDA=11.2.2
 DASK=latest
-DOTNET=7.0
+DOTNET=8.0
 GCC_VERSION=""
 GO=1.21.8
 STATICCHECK=v0.4.7
@@ -71,12 +71,12 @@ NUMBA=latest
 NUMPY=latest
 PANDAS=latest
 PYTHON=3.8
-R=4.2
+R=4.4
 SPARK=master
 TURBODBC=latest
 
-# These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-gcc-release:latest
-R_IMAGE=ubuntu-gcc-release
+# These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-release:latest
+R_IMAGE=ubuntu-release
 R_ORG=rhub
 R_TAG=latest
 

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -23,30 +23,35 @@ updates:
       interval: "weekly"
     commit-message:
       prefix: "MINOR: [CI] "
+    open-pull-requests-limit: 10
   - package-ecosystem: "gomod"
     directory: "/go/"
     schedule:
       interval: "weekly"
     commit-message:
       prefix: "MINOR: [Go] "
+    open-pull-requests-limit: 10
   - package-ecosystem: "maven"
     directory: "/java/"
     schedule:
       interval: "weekly"
     commit-message:
       prefix: "MINOR: [Java] "
+    open-pull-requests-limit: 10
   - package-ecosystem: "npm"
     directory: "/js/"
     schedule:
       interval: "monthly"
     commit-message:
       prefix: "MINOR: [JS] "
+    open-pull-requests-limit: 10
   - package-ecosystem: "nuget"
     directory: "/csharp/"
     schedule:
       interval: "weekly"
     commit-message:
       prefix: "MINOR: [C#] "
+    open-pull-requests-limit: 10
     ignore:
       - dependency-name: "Microsoft.Extensions.*"
         update-types:

diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml
@@ -46,7 +46,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['7.0.x']
+        dotnet: ['8.0.x']
     steps:
       - name: Install C#
         uses: actions/setup-dotnet@v4
@@ -74,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['7.0.x']
+        dotnet: ['8.0.x']
     steps:
       - name: Install C#
         uses: actions/setup-dotnet@v4
@@ -94,19 +94,23 @@ jobs:
         run: ci/scripts/csharp_test.sh $(pwd)
 
   macos:
-    name: AMD64 macOS 11 C# ${{ matrix.dotnet }}
+    name: ARM64 macOS 14 C# ${{ matrix.dotnet }}
     runs-on: macos-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['7.0.x']
+        dotnet: ['8.0.x']
     steps:
       - name: Install C#
         uses: actions/setup-dotnet@v4
         with:
           dotnet-version: ${{ matrix.dotnet }}
+      - name: Setup Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: 3.12
       - name: Checkout Arrow
         uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -109,13 +109,13 @@ jobs:
         with:
           python-version: '3.12'
       - name: Install Ruby
-        uses: ruby/setup-ruby@250fcd6a742febb1123a77a841497ccaa8b9e939 # v1.152.0
+        uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '2.7'
+          ruby-version: ruby
       - name: Install .NET
         uses: actions/setup-dotnet@4d6c8fcf3c8f7a60068d26b594648e99df24cee3 # v4.0.0
         with:
-          dotnet-version: '7.0.x'
+          dotnet-version: '8.0.x'
       - name: Install Dependencies
         shell: bash
         run: |

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -32,12 +32,12 @@ env:
 jobs:
 
   complete:
-    name: AMD64 Ubuntu 22.04 Complete Documentation
+    name: AMD64 Debian 12 Complete Documentation
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 150
     env:
-      UBUNTU: "22.04"
+      JDK: 17
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
@@ -50,8 +50,8 @@ jobs:
         uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
         with:
           path: .docker
-          key: ubuntu-docs-${{ hashFiles('cpp/**') }}
-          restore-keys: ubuntu-docs-
+          key: debian-docs-${{ hashFiles('cpp/**') }}
+          restore-keys: debian-docs-
       - name: Setup Python
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
@@ -62,7 +62,8 @@ jobs:
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: archery docker run ubuntu-docs
+          JDK: 17
+        run: archery docker run debian-docs
       - name: Docker Push
         if: >-
           success() &&
@@ -73,4 +74,4 @@ jobs:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
-        run: archery docker push ubuntu-docs
+        run: archery docker push debian-docs
diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml
@@ -31,7 +31,7 @@ on:
 
 permissions:
   contents: read
-  
+
 env:
   ARCHERY_DEBUG: 1
   ARCHERY_USE_DOCKER_CLI: 1

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -75,6 +75,11 @@ jobs:
         with:
           repository: apache/arrow-rs
           path: rust
+      - name: Checkout Arrow nanoarrow
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        with:
+          repository: apache/arrow-nanoarrow
+          path: nanoarrow
       - name: Free up disk space
         run: |
           ci/scripts/util_free_space.sh
@@ -97,6 +102,7 @@ jobs:
         run: >
           archery docker run \
             -e ARCHERY_DEFAULT_BRANCH=${{ github.event.repository.default_branch }} \
+            -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \
             -e ARCHERY_INTEGRATION_WITH_RUST=1 \
             conda-integration
       - name: Docker Push

diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml
@@ -70,7 +70,7 @@ jobs:
       - name: Install MATLAB
         uses: matlab-actions/setup-matlab@v2
         with:
-          release: R2023a
+          release: R2024a
       - name: Install ccache
         run: sudo apt-get install ccache
       - name: Setup ccache
@@ -96,10 +96,18 @@ jobs:
         uses: matlab-actions/run-tests@v2
         with:
           select-by-folder: matlab/test
+          strict: true
   macos:
-    name: AMD64 macOS 12 MATLAB
-    runs-on: macos-latest
+    name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} MATLAB
+    runs-on: macos-${{ matrix.macos-version }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    strategy:
+      matrix:
+        include:
+            - architecture: AMD64
+              macos-version: "12"
+            - architecture: ARM64
+              macos-version: "14"
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -110,7 +118,7 @@ jobs:
       - name: Install MATLAB
         uses: matlab-actions/setup-matlab@v2
         with:
-          release: R2023a
+          release: R2024a
       - name: Install ccache
         run: brew install ccache
       - name: Setup ccache
@@ -135,7 +143,8 @@ jobs:
           MATLABPATH: matlab/install/arrow_matlab
         uses: matlab-actions/run-tests@v2
         with:
-          select-by-folder: matlab/test 
+          select-by-folder: matlab/test
+          strict: true
   windows:
     name: AMD64 Windows 2022 MATLAB
     runs-on: windows-2022
@@ -148,7 +157,7 @@ jobs:
       - name: Install MATLAB
         uses: matlab-actions/setup-matlab@v2
         with:
-          release: R2023a
+          release: R2024a
       - name: Download Timezone Database
         shell: bash
         run: ci/scripts/download_tz_database.sh
@@ -181,4 +190,5 @@ jobs:
           MATLABPATH: matlab/install/arrow_matlab
         uses: matlab-actions/run-tests@v2
         with:
-          select-by-folder: matlab/test 
+          select-by-folder: matlab/test
+          strict: true
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -128,12 +128,20 @@ jobs:
         run: archery docker push ${{ matrix.image }}
 
   macos:
-    name: AMD64 macOS 12 Python 3
-    runs-on: macos-latest
+    name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} Python 3
+    runs-on: macos-${{ matrix.macos-version }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - architecture: AMD64
+            macos-version: "12"
+          - architecture: ARM64
+            macos-version: "14"
     env:
-      ARROW_HOME: /usr/local
+      ARROW_HOME: /tmp/local
       ARROW_AZURE: ON
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
@@ -174,6 +182,19 @@ jobs:
           python -m pip install \
             -r python/requirements-build.txt \
             -r python/requirements-test.txt
+      - name: Setup ccache
+        shell: bash
+        run: ci/scripts/ccache_setup.sh
+      - name: ccache info
+        id: ccache-info
+        shell: bash
+        run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
+      - name: Cache ccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.ccache-info.outputs.cache-dir }}
+          key: python-ccache-macos-${{ matrix.macos-version }}-${{ hashFiles('cpp/**', 'python/**') }}
+          restore-keys: python-ccache-macos-${{ matrix.macos-version }}-
       - name: Build
         shell: bash
         run: |

diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
@@ -121,7 +121,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        r: ["4.3"]
+        r: ["4.4"]
         ubuntu: [20.04]
         force-tests: ["true"]
     env:
@@ -192,7 +192,7 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - { org: "rhub", image: "debian-gcc-devel", tag: "latest", devtoolset: "" }
+          - { org: "rhub", image: "ubuntu-gcc12", tag: "latest", devtoolset: "" }
     env:
       R_ORG: ${{ matrix.config.org }}
       R_IMAGE: ${{ matrix.config.image }}

diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
@@ -117,7 +117,7 @@ jobs:
         run: archery docker push ubuntu-ruby
 
   macos:
-    name: AMD64 macOS 12 GLib & Ruby
+    name: AMD64 macOS 14 GLib & Ruby
     runs-on: macos-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60
@@ -132,7 +132,7 @@ jobs:
       ARROW_GCS: ON
       ARROW_GLIB_GTK_DOC: true
       ARROW_GLIB_WERROR: true
-      ARROW_HOME: /usr/local
+      ARROW_HOME: /tmp/local
       ARROW_JEMALLOC: OFF
       ARROW_ORC: OFF
       ARROW_PARQUET: ON
@@ -141,7 +141,6 @@ jobs:
       ARROW_WITH_SNAPPY: ON
       ARROW_WITH_ZLIB: ON
       ARROW_WITH_ZSTD: ON
-      XML_CATALOG_FILES: /usr/local/etc/xml/catalog
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v4

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -132,3 +132,9 @@ repos:
           ?^cpp/cmake_modules/UseCython\.cmake$|
           ?^cpp/src/arrow/util/config\.h\.cmake$|
           )
+  - repo: https://github.com/sphinx-contrib/sphinx-lint
+    rev: v0.9.1
+    hooks:
+      - id: sphinx-lint
+        files: ^docs/
+        args: ['--disable', 'all', '--enable', 'trailing-whitespace,missing-final-newline', 'docs']
diff --git a/c_glib/arrow-glib/scalar.cpp b/c_glib/arrow-glib/scalar.cpp
@@ -1063,7 +1063,8 @@ garrow_base_binary_scalar_get_value(GArrowBaseBinaryScalar *scalar)
   if (!priv->value) {
     const auto arrow_scalar = std::static_pointer_cast<arrow::BaseBinaryScalar>(
       garrow_scalar_get_raw(GARROW_SCALAR(scalar)));
-    priv->value = garrow_buffer_new_raw(&(arrow_scalar->value));
+    priv->value = garrow_buffer_new_raw(
+      const_cast<std::shared_ptr<arrow::Buffer> *>(&(arrow_scalar->value)));
   }
   return priv->value;
 }
@@ -1983,7 +1984,8 @@ garrow_base_list_scalar_get_value(GArrowBaseListScalar *scalar)
   if (!priv->value) {
     const auto arrow_scalar = std::static_pointer_cast<arrow::BaseListScalar>(
       garrow_scalar_get_raw(GARROW_SCALAR(scalar)));
-    priv->value = garrow_array_new_raw(&(arrow_scalar->value));
+    priv->value = garrow_array_new_raw(
+      const_cast<std::shared_ptr<arrow::Array> *>(&(arrow_scalar->value)));
   }
   return priv->value;
 }

diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt
@@ -26,7 +26,9 @@ pydata-sphinx-theme=0.14
 sphinx-autobuild
 sphinx-design
 sphinx-copybutton
+sphinx-lint
 sphinxcontrib-jquery
+sphinxcontrib-mermaid
 sphinx==6.2
 # Requirement for doctest-cython
 # Needs upper pin of 0.3.0, see: