From bcfb6b8f81e0e2bc4841cfe97a92747dd7c47a23 Mon Sep 17 00:00:00 2001 From: Gary Shen Date: Sat, 13 Jan 2024 14:53:33 +0800 Subject: [PATCH 1/3] Enable ccache in UDF examples build Signed-off-by: Gary Shen --- .../RAPIDS-accelerated-UDFs/Dockerfile | 15 +++++++++++ .../RAPIDS-accelerated-UDFs/README.md | 25 ++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile index cf0ed8635..c4223d286 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile @@ -70,3 +70,18 @@ RUN cd /tmp \ && make install -j${PARALLEL_LEVEL} \ && cd /tmp && rm -rf /tmp/cmake-$CMAKE_VERSION* +# Install ccache +ARG CCACHE_VERSION=4.6 +RUN cd /tmp && wget --quiet https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz && \ + tar zxf ccache-${CCACHE_VERSION}.tar.gz && \ + rm ccache-${CCACHE_VERSION}.tar.gz && \ + cd ccache-${CCACHE_VERSION} && \ + mkdir build && \ + cd build && \ + cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DZSTD_FROM_INTERNET=ON \ + -DREDIS_STORAGE_BACKEND=OFF && \ + cmake --build . --parallel ${PARALLEL_LEVEL} --target install && \ + cd ../.. && \ + rm -rf ccache-${CCACHE_VERSION} diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md index 29caa2d9d..986360c31 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md @@ -133,11 +133,34 @@ In the Docker container, clone the code and compile. ```bash git clone https://github.com/NVIDIA/spark-rapids-examples.git cd spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs +export LOCAL_CCACHE_DIR="$HOME/.ccache" +mkdir -p $LOCAL_CCACHE_DIR +export CCACHE_DIR="$LOCAL_CCACHE_DIR" +export CMAKE_C_COMPILER_LAUNCHER="ccache" +export CMAKE_CXX_COMPILER_LAUNCHER="ccache" +export CMAKE_CUDA_COMPILER_LAUNCHER="ccache" +export CMAKE_CXX_LINKER_LAUNCHER="ccache mvn clean package -Pudf-native-examples ``` -The build could take a long time (e.g.: 1.5 hours). Then the rapids-4-spark-udf-examples*.jar is +The Docker container has installed ccache 4.6 to accelerate the incremental building. +You can change the LOCAL_CCACHE_DIR to a mounted folder so that the cache can be kept by next docker running. +If you don't want to use ccache, you can remove or unset the ccache environment variables. + +```bash +unset CCACHE_DIR +unset CMAKE_C_COMPILER_LAUNCHER +unset CMAKE_CXX_COMPILER_LAUNCHER +unset CMAKE_CUDA_COMPILER_LAUNCHER +unset CMAKE_CXX_LINKER_LAUNCHER +``` + +The first build could take a long time (e.g.: 1.5 hours). Then the rapids-4-spark-udf-examples*.jar is generated under RAPIDS-accelerated-UDFs/target directory. +The following build can benefit from ccache if you enable it. + +If you want to enable building with ccache on your own system, +please refer to the commands which build ccache from the source code in the Dockerfile. ### Run all the examples including native examples in the docker From e5f343763a805c51943b3f73bce66e3118be9c6c Mon Sep 17 00:00:00 2001 From: Gary Shen Date: Thu, 18 Jan 2024 06:23:01 +0800 Subject: [PATCH 2/3] Update examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md Co-authored-by: Jason Lowe --- examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md index 986360c31..26f510c67 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md @@ -144,7 +144,7 @@ mvn clean package -Pudf-native-examples ``` The Docker container has installed ccache 4.6 to accelerate the incremental building. -You can change the LOCAL_CCACHE_DIR to a mounted folder so that the cache can be kept by next docker running. +You can change the LOCAL_CCACHE_DIR to a mounted folder so that the cache can persist. If you don't want to use ccache, you can remove or unset the ccache environment variables. ```bash From 2dd2f1e19d381e03a103deeb85f0abc022df2bee Mon Sep 17 00:00:00 2001 From: Gary Shen Date: Thu, 18 Jan 2024 06:27:12 +0800 Subject: [PATCH 3/3] Update copyright Signed-off-by: Gary Shen --- examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile index c4223d286..b5ef1cc0c 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile @@ -1,5 +1,5 @@ # -# Copyright (c) 2021-2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.