[Native] Support linux aarch64

.github/workflows/consistency-ci.yml

@@ -84,7 +84,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           default: true
       - uses: Swatinem/rust-cache@v2
         with:
@@ -93,7 +93,7 @@ jobs:
         with:
           use-cross: true
           command: build
-          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release'
       - name: Build with Maven
         run: |
           mkdir -p rust/target/release

.github/workflows/deployment.yml

@@ -24,7 +24,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           default: true
       - uses: Swatinem/rust-cache@v2
         with:
@@ -37,7 +37,7 @@ jobs:
         with:
           use-cross: true
           command: build
-          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --features hdfs'
       - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu
@@ -65,15 +65,15 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           default: true
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: "./rust -> target"
       - uses: actions-rs/cargo@v1
         with:
           command: build
-          args: '--manifest-path rust/Cargo.toml --release --all-features'
+          args: '--manifest-path rust/Cargo.toml --release --features hdfs'
       - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-pc-windows-msvc
@@ -103,15 +103,15 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           default: true
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: "./rust -> target"
       - uses: actions-rs/cargo@v1
         with:
           command: build
-          args: '--manifest-path rust/Cargo.toml --release --all-features'
+          args: '--manifest-path rust/Cargo.toml --release --features hdfs'
       - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-apple-darwin

.github/workflows/flink-cdc-hdfs-test.yml

@@ -69,7 +69,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           default: true
       - uses: Swatinem/rust-cache@v2
         with:
@@ -85,7 +85,7 @@ jobs:
         with:
           use-cross: true
           command: build
-          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --features hdfs'
       - name: Build with Maven
         run: |
           mkdir -p rust/target/release

.github/workflows/flink-cdc-test.yml

@@ -71,7 +71,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           default: true
       - uses: Swatinem/rust-cache@v2
         with:
@@ -87,7 +87,7 @@ jobs:
         with:
           use-cross: true
           command: build
-          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release'
       - name: Build with Maven
         run: |
           mkdir -p rust/target/release

.github/workflows/maven-test.yml

@@ -43,7 +43,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           default: true
       - uses: Swatinem/rust-cache@v2
         with:
@@ -56,7 +56,7 @@ jobs:
         with:
           use-cross: true
           command: build
-          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --features hdfs'
       - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test

.github/workflows/native-build.yml

@@ -36,7 +36,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           default: true
       - uses: Swatinem/rust-cache@v2
         with:
@@ -50,7 +50,7 @@ jobs:
         with:
           use-cross: true
           command: build
-          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --features hdfs'
       - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu
@@ -78,7 +78,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           default: true
       - uses: Swatinem/rust-cache@v2
         with:
@@ -87,7 +87,7 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: build
-          args: '--manifest-path rust/Cargo.toml --release --all-features'
+          args: '--manifest-path rust/Cargo.toml --release --features hdfs'
       - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-pc-windows-msvc
@@ -117,7 +117,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           default: true
       - uses: Swatinem/rust-cache@v2
         with:
@@ -126,7 +126,7 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: build
-          args: '--manifest-path rust/Cargo.toml --release --all-features'
+          args: '--manifest-path rust/Cargo.toml --release --features hdfs'
       - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-apple-darwin

.github/workflows/presto-cdc-test.yml

@@ -71,7 +71,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           default: true
       - uses: Swatinem/rust-cache@v2
         with:
@@ -87,7 +87,7 @@ jobs:
         with:
           use-cross: true
           command: build
-          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+          args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release'
       - name: Build with Maven
         run: |
           mkdir -p rust/target/release

.github/workflows/rust-clippy.yml

@@ -30,7 +30,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-05-20
+          toolchain: stable
           components: clippy
           default: true
       - name: Install Protoc
@@ -42,5 +42,5 @@ jobs:
           workspaces: "./rust -> target"
           key: "ubuntu-latest-clippy"
       - name: Run Clippy
-        run: cd rust && cargo clippy --no-deps --all-features --package lakesoul-io-c --package lakesoul-metadata-c
+        run: cd rust && cargo clippy --no-deps --features hdfs --package lakesoul-io-c --package lakesoul-metadata-c
 

lakesoul-flink/pom.xml

@@ -448,6 +448,7 @@ SPDX-License-Identifier: Apache-2.0
                             <include>org.furyio:fury-core</include>
                             <include>com.google.guava:guava</include>
                             <include>com.google.guava:failureaccess</include>
+                            <include>org.eclipse.collections:*</include>
 
                             <!-- casbin & aspectj -->
                             <include>org.casbin:jdbc-adapter</include>
@@ -483,9 +484,12 @@ SPDX-License-Identifier: Apache-2.0
                             </excludes>
                         </filter>
                         <filter>
-                            <artifact>org.apache.arrow:arrow-c-data</artifact>
+                            <artifact>*:*</artifact>
                             <excludes>
-                                <exclude>org/apache/arrow/c/jni/JniLoader.class</exclude>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                                <exclude>META-INF/versions/**</exclude>
                             </excludes>
                         </filter>
                     </filters>

lakesoul-spark/pom.xml

@@ -478,6 +478,7 @@ SPDX-License-Identifier: Apache-2.0
                             <include>org.postgresql:postgresql</include>
                             <include>com.alibaba:fastjson</include>
                             <include>mysql:mysql-connector-java</include>
+                            <include>org.eclipse.collections:*</include>
 
                             <!-- casbin & aspectj -->
                             <include>org.casbin:jdbc-adapter</include>
@@ -497,9 +498,12 @@ SPDX-License-Identifier: Apache-2.0
                     </artifactSet>
                     <filters>
                         <filter>
-                            <artifact>org.apache.arrow:arrow-c-data</artifact>
+                            <artifact>*:*</artifact>
                             <excludes>
-                                <exclude>org/apache/arrow/c/jni/JniLoader.class</exclude>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                                <exclude>META-INF/versions/**</exclude>
                             </excludes>
                         </filter>
                     </filters>

native-io/lakesoul-io-java/pom.xml

@@ -25,7 +25,7 @@ SPDX-License-Identifier: Apache-2.0
     <properties>
         <maven.compiler.source>8</maven.compiler.source>
         <maven.compiler.target>8</maven.compiler.target>
-        <arrow.version>12.0.0</arrow.version>
+        <arrow.version>15.0.2</arrow.version>
         <scalatest.version>3.1.0</scalatest.version>
         <substrait.version>0.28.0</substrait.version>
         <protobuf.version>3.22.0</protobuf.version>
@@ -95,6 +95,14 @@ SPDX-License-Identifier: Apache-2.0
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-jdk14</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
             </exclusions>
             <version>${substrait.version}</version>
             <scope>compile</scope>
@@ -584,12 +592,6 @@ SPDX-License-Identifier: Apache-2.0
                                 <exclude>META-INF/versions/**</exclude>
                             </excludes>
                         </filter>
-                        <filter>
-                            <artifact>org.apache.arrow:arrow-c-data</artifact>
-                            <excludes>
-                                <exclude>org/apache/arrow/c/jni/JniLoader.class</exclude>
-                            </excludes>
-                        </filter>
                     </filters>
                     <relocations>
                         <relocation>