diff --git a/.env b/.env index bb9f63ab3..b50a16eb0 100644 --- a/.env +++ b/.env @@ -47,4 +47,4 @@ ULIMIT_CORE=-1 # Default versions for various dependencies JDK=11 -MAVEN=3.9.6 +MAVEN=3.9.9 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 473ce84ca..69c8db0f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ env: jobs: ubuntu: - name: AMD64 Ubuntu 22.04 JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + name: AMD64 ${{ matrix.name }} JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 @@ -46,8 +46,13 @@ jobs: fail-fast: false matrix: jdk: [11, 17, 21, 22] - maven: [3.9.6] - image: [java] + maven: [3.9.9] + image: [ubuntu, conda-jni-cdata] + include: + - image: ubuntu + name: "Ubuntu" + - image: conda-jni-cdata + name: "Conda JNI" env: JDK: ${{ matrix.jdk }} MAVEN: ${{ matrix.maven }} diff --git a/ci/docker/conda-jni.dockerfile b/ci/docker/conda-jni.dockerfile new file mode 100644 index 000000000..e14db7368 --- /dev/null +++ b/ci/docker/conda-jni.dockerfile @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ghcr.io/mamba-org/micromamba:ubuntu24.04 + +ARG jdk=11 +ARG maven=3.9.9 + +RUN micromamba install -y \ + -c conda-forge \ + cmake \ + compilers \ + maven=${maven} \ + ninja \ + openjdk=${jdk} && \ + micromamba clean --all diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh new file mode 100755 index 000000000..7b63dbeea --- /dev/null +++ b/ci/scripts/java_jni_build.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +arrow_dir=${1} +arrow_install_dir=${2} +build_dir=${3}/java_jni +# The directory where the final binaries will be stored when scripts finish +dist_dir=${4} +prefix_dir="${build_dir}/java-jni" + +echo "=== Clear output directories and leftovers ===" +# Clear output directories and leftovers +rm -rf ${build_dir} + +echo "=== Building Arrow Java C Data Interface native library ===" +mkdir -p "${build_dir}" +pushd "${build_dir}" + +case "$(uname)" in + Linux) + n_jobs=$(nproc) + ;; + Darwin) + n_jobs=$(sysctl -n hw.logicalcpu) + ;; + *) + n_jobs=${NPROC:-1} + ;; +esac + +: ${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-OFF}} +: ${CMAKE_BUILD_TYPE:=release} +cmake \ + -DARROW_JAVA_JNI_ENABLE_DATASET=${ARROW_DATASET:-OFF} \ + -DARROW_JAVA_JNI_ENABLE_GANDIVA=${ARROW_GANDIVA:-OFF} \ + -DARROW_JAVA_JNI_ENABLE_ORC=${ARROW_ORC:-OFF} \ + -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DCMAKE_PREFIX_PATH=${arrow_install_dir} \ + -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ + -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ + -DProtobuf_USE_STATIC_LIBS=ON \ + -GNinja \ + ${JAVA_JNI_CMAKE_ARGS:-} \ + ${arrow_dir} +export CMAKE_BUILD_PARALLEL_LEVEL=${n_jobs} +cmake --build . --config ${CMAKE_BUILD_TYPE} +if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then + ctest \ + --output-on-failure \ + --parallel ${n_jobs} \ + --timeout 300 +fi +cmake --build . --config ${CMAKE_BUILD_TYPE} --target install +popd + +mkdir -p ${dist_dir} +# For Windows. *.dll are installed into bin/ on Windows. +if [ -d "${prefix_dir}/bin" ]; then + mv ${prefix_dir}/bin/* ${dist_dir}/ +else + mv ${prefix_dir}/lib/* ${dist_dir}/ +fi diff --git a/docker-compose.yml b/docker-compose.yml index 103f2f3ad..ae378865b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,18 +30,48 @@ volumes: name: maven-cache services: - java: + ubuntu: # Usage: - # docker compose build java - # docker compose run java + # docker compose build ubuntu + # docker compose run ubuntu # Parameters: - # MAVEN: 3.9.6 + # MAVEN: 3.9.9 # JDK: 11, 17, 21 image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} - volumes: &java-volumes + volumes: - .:/arrow-java:delegated - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - command: &java-command > + command: /bin/bash -c " /arrow-java/ci/scripts/java_build.sh /arrow-java /build && /arrow-java/ci/scripts/java_test.sh /arrow-java /build" + + conda-jni-cdata: + # Usage: + # docker compose build conda-jni-cdata + # docker compose run conda-jni-cdata + # Parameters: + # MAVEN: 3.9.9 + # JDK: 11, 17, 21 + image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + build: + context: . + dockerfile: ci/docker/conda-jni.dockerfile + cache_from: + - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + args: + jdk: ${JDK} + maven: ${MAVEN} + # required to use micromamba with rootless docker + # https://github.com/mamba-org/micromamba-docker/issues/407#issuecomment-2088523507 + user: root + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + environment: + ARROW_JAVA_CDATA: "ON" + command: + /bin/bash -c " + /arrow-java/ci/scripts/java_jni_build.sh /arrow-java /build/jni /build /jni && + /arrow-java/ci/scripts/java_build.sh /arrow-java /build /jni && + /arrow-java/ci/scripts/java_test.sh /arrow-java /build /jni"