Merge branch 'main' into add-more-comments-about-row-table

mapleFU · Aug 19, 2024 · e1fbb46 · e1fbb46
2 parents fa0d12a + a380d69
commit e1fbb46
Show file tree

Hide file tree

Showing 49 changed files with 2,295 additions and 575 deletions.
diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
@@ -133,6 +133,9 @@ jobs:
         with:
           fetch-depth: 0
           submodules: recursive
+      - name: Free up disk space
+        run: |
+          ci/scripts/util_free_space.sh
       - name: Cache Docker Volumes
         uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
         with:

diff --git a/c_glib/arrow-glib/file-system.cpp b/c_glib/arrow-glib/file-system.cpp
@@ -56,6 +56,8 @@ G_BEGIN_DECLS
  * #GArrowS3FileSystem is a class for S3-backed file system.
  *
  * #GArrowGCSFileSystem is a class for GCS-backed file system.
+ *
+ * #GArrowAzureFileSystem is a class for Azure-backed file system.
  */
 
 /* arrow::fs::FileInfo */
@@ -1561,6 +1563,18 @@ garrow_gcs_file_system_class_init(GArrowGCSFileSystemClass *klass)
 {
 }
 
+G_DEFINE_TYPE(GArrowAzureFileSystem, garrow_azure_file_system, GARROW_TYPE_FILE_SYSTEM)
+
+static void
+garrow_azure_file_system_init(GArrowAzureFileSystem *file_system)
+{
+}
+
+static void
+garrow_azure_file_system_class_init(GArrowAzureFileSystemClass *klass)
+{
+}
+
 G_END_DECLS
 
 GArrowFileInfo *
@@ -1592,6 +1606,8 @@ garrow_file_system_new_raw(std::shared_ptr<arrow::fs::FileSystem> *arrow_file_sy
     file_system_type = GARROW_TYPE_S3_FILE_SYSTEM;
   } else if (type_name == "gcs") {
     file_system_type = GARROW_TYPE_GCS_FILE_SYSTEM;
+  } else if (type_name == "abfs") {
+    file_system_type = GARROW_TYPE_AZURE_FILE_SYSTEM;
   } else if (type_name == "mock") {
     file_system_type = GARROW_TYPE_MOCK_FILE_SYSTEM;
   }

diff --git a/c_glib/arrow-glib/file-system.h b/c_glib/arrow-glib/file-system.h
@@ -337,4 +337,16 @@ struct _GArrowGCSFileSystemClass
   GArrowFileSystemClass parent_class;
 };
 
+#define GARROW_TYPE_AZURE_FILE_SYSTEM (garrow_azure_file_system_get_type())
+GARROW_AVAILABLE_IN_18_0
+G_DECLARE_DERIVABLE_TYPE(GArrowAzureFileSystem,
+                         garrow_azure_file_system,
+                         GARROW,
+                         AZURE_FILE_SYSTEM,
+                         GArrowFileSystem)
+struct _GArrowAzureFileSystemClass
+{
+  GArrowFileSystemClass parent_class;
+};
+
 G_END_DECLS
diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh
@@ -80,7 +80,7 @@ case "$(uname)" in
     ;;
 esac
 
-if [ "${ARROW_EMSCRIPTEN:-OFF}" = "ON" ]; then  
+if [ "${ARROW_EMSCRIPTEN:-OFF}" = "ON" ]; then
   n_jobs=1 # avoid spurious fails on emscripten due to loading too many big executables
 fi
 

diff --git a/ci/scripts/install_azurite.sh b/ci/scripts/install_azurite.sh
@@ -19,20 +19,32 @@
 
 set -e
 
-# Pin azurite to 3.29.0 due to https://github.com/apache/arrow/issues/41505
+node_version="$(node --version)"
+echo "node version = ${node_version}"
+
+case "${node_version}" in
+  v12*)
+    # Pin azurite to 3.29.0 due to https://github.com/apache/arrow/issues/41505
+    azurite_version=v3.29.0
+    ;;
+  *)
+    azurite_version=latest
+    ;;
+esac
+
 case "$(uname)" in
   Darwin)
-    npm install -g azurite@v3.29.0
+    npm install -g azurite@${azurite_version}
     which azurite
     ;;
   MINGW*)
     choco install nodejs.install
-    npm install -g azurite@v3.29.0
+    npm install -g azurite@${azurite_version}
     ;;
   Linux)
-    npm install -g azurite@v3.29.0
+    npm install -g azurite@${azurite_version}
     which azurite
     ;;
 esac
-echo "node version = $(node --version)"
-echo "azurite version = $(azurite --version)"
+
+echo "azurite version = $(azurite --version)"
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -4965,8 +4965,20 @@ macro(build_awssdk)
   set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install")
   set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include")
 
+  # The AWS SDK has a few warnings around shortening lengths
+  set(AWS_C_FLAGS "${EP_C_FLAGS}")
+  set(AWS_CXX_FLAGS "${EP_CXX_FLAGS}")
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL
+                                                    "Clang")
+    # Negate warnings that AWS SDK cannot build under
+    string(APPEND AWS_C_FLAGS " -Wno-error=shorten-64-to-32")
+    string(APPEND AWS_CXX_FLAGS " -Wno-error=shorten-64-to-32")
+  endif()
+
   set(AWSSDK_COMMON_CMAKE_ARGS
       ${EP_COMMON_CMAKE_ARGS}
+      -DCMAKE_C_FLAGS=${AWS_C_FLAGS}
+      -DCMAKE_CXX_FLAGS=${AWS_CXX_FLAGS}
       -DCPP_STANDARD=${CMAKE_CXX_STANDARD}
       -DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}
       -DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
@@ -723,7 +723,6 @@ set(ARROW_COMPUTE_SRCS
     compute/ordering.cc
     compute/registry.cc
     compute/kernels/codegen_internal.cc
-    compute/kernels/row_encoder.cc
     compute/kernels/ree_util_internal.cc
     compute/kernels/scalar_cast_boolean.cc
     compute/kernels/scalar_cast_dictionary.cc
@@ -742,6 +741,7 @@ set(ARROW_COMPUTE_SRCS
     compute/row/encode_internal.cc
     compute/row/compare_internal.cc
     compute/row/grouper.cc
+    compute/row/row_encoder_internal.cc
     compute/row/row_internal.cc
     compute/util.cc
     compute/util_internal.cc)

diff --git a/cpp/src/arrow/acero/asof_join_node_test.cc b/cpp/src/arrow/acero/asof_join_node_test.cc
@@ -41,8 +41,8 @@
 #include "arrow/acero/util.h"
 #include "arrow/api.h"
 #include "arrow/compute/api_scalar.h"
-#include "arrow/compute/kernels/row_encoder_internal.h"
 #include "arrow/compute/kernels/test_util.h"
+#include "arrow/compute/row/row_encoder_internal.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/matchers.h"
 #include "arrow/testing/random.h"

diff --git a/cpp/src/arrow/acero/hash_join.cc b/cpp/src/arrow/acero/hash_join.cc
@@ -27,8 +27,8 @@
 
 #include "arrow/acero/hash_join_dict.h"
 #include "arrow/acero/task_util.h"
-#include "arrow/compute/kernels/row_encoder_internal.h"
 #include "arrow/compute/row/encode_internal.h"
+#include "arrow/compute/row/row_encoder_internal.h"
 #include "arrow/util/tracing_internal.h"
 
 namespace arrow {

diff --git a/cpp/src/arrow/acero/hash_join_benchmark.cc b/cpp/src/arrow/acero/hash_join_benchmark.cc
@@ -23,7 +23,7 @@
 #include "arrow/acero/test_util_internal.h"
 #include "arrow/acero/util.h"
 #include "arrow/api.h"
-#include "arrow/compute/kernels/row_encoder_internal.h"
+#include "arrow/compute/row/row_encoder_internal.h"
 #include "arrow/testing/random.h"
 #include "arrow/util/thread_pool.h"
 

diff --git a/cpp/src/arrow/acero/hash_join_dict.h b/cpp/src/arrow/acero/hash_join_dict.h
@@ -22,7 +22,7 @@
 
 #include "arrow/acero/schema_util.h"
 #include "arrow/compute/exec.h"
-#include "arrow/compute/kernels/row_encoder_internal.h"
+#include "arrow/compute/row/row_encoder_internal.h"
 #include "arrow/result.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
-Original file line number
+Diff line change
@@ Expand Up / @@ -80,7 +80,7 @@ case "$(uname)" in @@
         ;;
     esac
-    if [ "${ARROW_EMSCRIPTEN:-OFF}" = "ON" ]; then
+    if [ "${ARROW_EMSCRIPTEN:-OFF}" = "ON" ]; then
       n_jobs=1 # avoid spurious fails on emscripten due to loading too many big executables
     fi
@@ Expand Down @@