From bc8d4ceba2dd5dd600b0058af8535e6090a80891 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey <41028320+ssheorey@users.noreply.github.com> Date: Wed, 3 Jan 2024 22:34:13 -0800 Subject: [PATCH] Build fixes for v0.18 (#6562) * Use launcher for open3d viewer on Linux to correctly locate resources. * Build TF ops in cxx11_abi packages * Update node.js install method * Fixes, update docs * Re-enable librealsense build --- cpp/apps/CMakeLists.txt | 1 + .../Open3DViewer/Debian/CMakeLists.in.txt | 17 ++++--- cpp/apps/Open3DViewer/Open3DViewerLauncher.sh | 7 +++ cpp/open3d/visualization/gui/Application.cpp | 16 ++++--- docker/Dockerfile.ci | 13 +++++- docker/docker_build.sh | 28 +++++++----- docker/docker_test.sh | 6 ++- docs/arm.rst | 9 ++-- docs/getting_started.in.rst | 45 ++++++++++++------- docs/release.md | 19 +++++--- docs/versions.js | 4 +- util/ci_utils.sh | 2 +- 12 files changed, 108 insertions(+), 59 deletions(-) create mode 100644 cpp/apps/Open3DViewer/Open3DViewerLauncher.sh diff --git a/cpp/apps/CMakeLists.txt b/cpp/apps/CMakeLists.txt index ac927bd147a..8b718c9968c 100644 --- a/cpp/apps/CMakeLists.txt +++ b/cpp/apps/CMakeLists.txt @@ -78,6 +78,7 @@ macro(open3d_add_app_gui SRC_DIR APP_NAME TARGET_NAME) COMMAND cp -a "${CMAKE_BINARY_DIR}/${APP_NAME}" . COMMAND cp "${SOURCE_DIR}/icon.svg" "${APP_NAME}/${APP_NAME}.svg" COMMAND cp "${SOURCE_DIR}/${TARGET_NAME}.xml" "${APP_NAME}/" + COMMAND cp "${SOURCE_DIR}/${TARGET_NAME}Launcher.sh" "${APP_NAME}/" COMMAND "${CMAKE_COMMAND}" -S . COMMAND "${CMAKE_COMMAND}" --build . -t package WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/package-${TARGET_NAME}-deb/" diff --git a/cpp/apps/Open3DViewer/Debian/CMakeLists.in.txt b/cpp/apps/Open3DViewer/Debian/CMakeLists.in.txt index c318c212612..9320f265d3e 100644 --- a/cpp/apps/Open3DViewer/Debian/CMakeLists.in.txt +++ b/cpp/apps/Open3DViewer/Debian/CMakeLists.in.txt @@ -4,6 +4,10 @@ project("Open3D-Debian") message(STATUS "Building package for Debian") +# How to set cpack prefix: https://stackoverflow.com/a/7363073/1255535 +set(CPACK_SET_DESTDIR true) +set(CPACK_INSTALL_PREFIX /usr/local) + # Install assets install(DIRECTORY "Open3D" DESTINATION share @@ -11,13 +15,16 @@ install(DIRECTORY "Open3D" PATTERN "Open3D/Open3D.svg" EXCLUDE PATTERN "Open3D/Open3D.desktop" EXCLUDE PATTERN "Open3D/Open3DViewer.xml" EXCLUDE - PATTERN "Open3D/Open3D" EXCLUDE PATTERN "Open3D/CMakeLists.txt" EXCLUDE + PATTERN "Open3D/Open3DViewerLauncher.sh" EXCLUDE ) -install(FILES "Open3D/Open3D.desktop" DESTINATION /usr/share/applications) +install(CODE "file(READ \"${CMAKE_BINARY_DIR}/Open3D/Open3D.desktop\" _DESKTOP_FILE) + string(REGEX REPLACE \"Exec=.*/bin/Open3D\" \"Exec=${CPACK_INSTALL_PREFIX}/bin/Open3D\" _DESKTOP_FILE \"\${_DESKTOP_FILE}\") + file(WRITE \"${CMAKE_BINARY_DIR}/Open3D/Open3D.deb.desktop\" \"\${_DESKTOP_FILE}\")") +install(FILES "Open3D/Open3D.deb.desktop" DESTINATION /usr/share/applications RENAME "Open3D.desktop") install(FILES "Open3D/Open3DViewer.xml" DESTINATION /usr/share/mime/packages) install(FILES "Open3D/Open3D.svg" DESTINATION /usr/share/icons/hicolor/scalable/apps) -install(PROGRAMS "Open3D/Open3D" DESTINATION bin) +install(PROGRAMS "Open3D/Open3DViewerLauncher.sh" DESTINATION bin RENAME "Open3D") # CPACK parameter set(CPACK_GENERATOR "DEB") @@ -29,8 +36,4 @@ set(CPACK_PACKAGE_VERSION "@OPEN3D_VERSION@") set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc++1, libgomp1, libpng16-16, libglfw3") set(CPACK_PACKAGE_HOMEPAGE_URL "@PROJECT_HOMEPAGE_URL@") -# How to set cpack prefix: https://stackoverflow.com/a/7363073/1255535 -set(CPACK_SET_DESTDIR true) -set(CPACK_INSTALL_PREFIX /usr/local) - include(CPack) diff --git a/cpp/apps/Open3DViewer/Open3DViewerLauncher.sh b/cpp/apps/Open3DViewer/Open3DViewerLauncher.sh new file mode 100644 index 00000000000..c917a06685e --- /dev/null +++ b/cpp/apps/Open3DViewer/Open3DViewerLauncher.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Linux launcher script for Open3DViewer, installed in the system PATH with name +# Open3D. The Open3D viewer binary is placed together with resources. +SCRIPT=$(readlink -f "$0") +INSTALL_DIRECTORY=$(readlink -f $(dirname "$SCRIPT")/..) +OPEN3D_PATH="$INSTALL_DIRECTORY/share/Open3D/Open3D" +"$OPEN3D_PATH" "$@" & \ No newline at end of file diff --git a/cpp/open3d/visualization/gui/Application.cpp b/cpp/open3d/visualization/gui/Application.cpp index 59290cfaba0..89cfa9746b2 100644 --- a/cpp/open3d/visualization/gui/Application.cpp +++ b/cpp/open3d/visualization/gui/Application.cpp @@ -45,6 +45,7 @@ namespace { const double RUNLOOP_DELAY_SEC = 0.010; std::string FindResourcePath(int argc, const char *argv[]) { + namespace o3dfs = open3d::utility::filesystem; std::string argv0; if (argc != 0 && argv) { argv0 = argv[0]; @@ -66,7 +67,7 @@ std::string FindResourcePath(int argc, const char *argv[]) { // is absolute path, we're done } else { // relative path: prepend working directory - auto cwd = open3d::utility::filesystem::GetWorkingDirectory(); + auto cwd = o3dfs::GetWorkingDirectory(); #ifdef __APPLE__ // When running an app from the command line with the full relative // path (e.g. `bin/Open3D.app/Contents/MacOS/Open3D`), the working @@ -85,14 +86,15 @@ std::string FindResourcePath(int argc, const char *argv[]) { } #endif // __APPLE__ - auto resource_path = path + "/resources"; - if (!open3d::utility::filesystem::DirectoryExists(resource_path)) { - resource_path = path + "/../resources"; // building with Xcode - if (!open3d::utility::filesystem::DirectoryExists(resource_path)) { - resource_path = path + "/share/resources"; // GNU + for (auto &subpath : + {"/resources", "/../resources" /*building with Xcode */, + "/share/resources" /* GNU */, "/share/Open3D/resources" /* GNU */}) { + if (o3dfs::DirectoryExists(path + subpath)) { + return path + subpath; } } - return resource_path; + open3d::utility::LogError("Could not find resource directory."); + return ""; } } // namespace diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 99e359df0e0..e446353744a 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -158,7 +158,12 @@ RUN source util/ci_utils.sh \ && pip install -r python/requirements_test.txt # Open3D Jupyter dependencies -RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \ + | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_16.x nodistro main" \ + | tee /etc/apt/sources.list.d/nodesource.list \ + && apt-get update \ && apt-get install -y nodejs \ && rm -rf /var/lib/apt/lists/* \ && node --version \ @@ -175,7 +180,11 @@ RUN \ export CMAKE_CXX_COMPILER=g++; \ export CMAKE_C_COMPILER=gcc; \ # TODO: PyTorch still use old CXX ABI, remove this line when PyTorch is updated - export GLIBCXX_USE_CXX11_ABI=OFF; \ + if [ "$BUILD_PYTORCH_OPS" = "ON" ]; then \ + export GLIBCXX_USE_CXX11_ABI=OFF; \ + else \ + export GLIBCXX_USE_CXX11_ABI=ON; \ + fi \ fi \ && mkdir build \ && cd build \ diff --git a/docker/docker_build.sh b/docker/docker_build.sh index 97f8cc86c5a..33ab6ad0115 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -44,10 +44,10 @@ OPTION: # Ubuntu CPU CI (Dockerfile.ci) cpu-static : Ubuntu CPU static - cpu-shared : Ubuntu CPU shared - cpu-shared-release : Ubuntu CPU shared, release mode - cpu-shared-ml : Ubuntu CPU shared with ML - cpu-shared-ml-release : Ubuntu CPU shared with ML, release mode + cpu-shared : Ubuntu CPU shared (cxx11_abi) + cpu-shared-release : Ubuntu CPU shared (cxx11_abi), release mode + cpu-shared-ml : Ubuntu CPU shared with ML (pre_cxx11_abi) + cpu-shared-ml-release : Ubuntu CPU shared with ML (pre_cxx11_abi), release mode # Sycl CPU CI (Dockerfile.ci) sycl-shared : SYCL (oneAPI) with shared lib @@ -55,10 +55,10 @@ OPTION: # ML CIs (Dockerfile.ci) 2-bionic : CUDA CI, 2-bionic, developer mode - 3-ml-shared-bionic-release : CUDA CI, 3-ml-shared-bionic, release mode - 3-ml-shared-bionic : CUDA CI, 3-ml-shared-bionic, developer mode - 4-shared-bionic : CUDA CI, 4-shared-bionic, developer mode - 4-shared-bionic-release : CUDA CI, 4-shared-bionic, release mode + 3-ml-shared-bionic-release : CUDA CI, 3-ml-shared-bionic (pre_cxx11_abi), release mode + 3-ml-shared-bionic : CUDA CI, 3-ml-shared-bionic (pre_cxx11_abi), developer mode + 4-shared-bionic : CUDA CI, 4-shared-bionic (cxx11_abi), developer mode + 4-shared-bionic-release : CUDA CI, 4-shared-bionic (cxx11_abi), release mode 5-ml-focal : CUDA CI, 5-ml-focal, developer mode # CUDA wheels (Dockerfile.wheel) @@ -314,7 +314,8 @@ ci_build() { export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON - export BUILD_TENSORFLOW_OPS=OFF + # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi + export BUILD_TENSORFLOW_OPS=ON export BUILD_PYTORCH_OPS=OFF export PACKAGE=ON export BUILD_SYCL_MODULE=OFF @@ -329,7 +330,8 @@ ci_build() { export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON - export BUILD_TENSORFLOW_OPS=OFF + # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi + export BUILD_TENSORFLOW_OPS=ON export BUILD_PYTORCH_OPS=OFF export PACKAGE=ON export BUILD_SYCL_MODULE=OFF @@ -375,7 +377,8 @@ cpu-shared_export_env() { export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF - export BUILD_TENSORFLOW_OPS=OFF + # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi + export BUILD_TENSORFLOW_OPS=ON export BUILD_PYTORCH_OPS=OFF export PACKAGE=ON export BUILD_SYCL_MODULE=OFF @@ -406,7 +409,8 @@ cpu-shared-release_export_env() { export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF - export BUILD_TENSORFLOW_OPS=OFF + # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi + export BUILD_TENSORFLOW_OPS=ON export BUILD_PYTORCH_OPS=OFF export PACKAGE=ON export BUILD_SYCL_MODULE=OFF diff --git a/docker/docker_test.sh b/docker/docker_test.sh index 877bbe1e5f8..544eeb755e4 100755 --- a/docker/docker_test.sh +++ b/docker/docker_test.sh @@ -106,18 +106,20 @@ cpp_python_linking_uninstall_test() { # - BUILD_PYTORCH_OPS # - BUILD_TENSORFLOW_OPS # - BUILD_SYCL_MODULE + # - NPROC (optional) echo "[cpp_python_linking_uninstall_test()] DOCKER_TAG=${DOCKER_TAG}" echo "[cpp_python_linking_uninstall_test()] BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}" echo "[cpp_python_linking_uninstall_test()] BUILD_CUDA_MODULE=${BUILD_CUDA_MODULE}" echo "[cpp_python_linking_uninstall_test()] BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS}" echo "[cpp_python_linking_uninstall_test()] BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS}" echo "[cpp_python_linking_uninstall_test()] BUILD_SYCL_MODULE=${BUILD_SYCL_MODULE}" + echo "[cpp_python_linking_uninstall_test()] NPROC=${NPROC:=$(nproc)}" # Config-dependent argument: gpu_run_args if [ "${BUILD_CUDA_MODULE}" == "ON" ]; then - docker_run="docker run --gpus all" + docker_run="docker run --cpus ${NPROC} --gpus all" else - docker_run="docker run" + docker_run="docker run --cpus ${NPROC}" fi # Config-dependent argument: pytest_args diff --git a/docs/arm.rst b/docs/arm.rst index 584d0d007ad..e9f9c942413 100644 --- a/docs/arm.rst +++ b/docs/arm.rst @@ -171,13 +171,14 @@ Compiling Open3D on ARM64 macOS # Optional: activate your virtualenv conda activate your-virtual-env - # Configure + # Configure and choose build options cd Open3D && mkdir build && cd build - cmake .. + cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TENSORFLOW_OPS=ON -DBUILD_PYTORCH_OPS=ON -DBUNDLE_OPEN3D_ML=ON .. # Build - make -j8 - make install-pip-package -j8 + make pip-package -j8 # Build Python wheel + make package -j8 # Build macOS devel binary package + make Open3DViewer -j8 # Build Open3D viewer app # Test C++ viewer app ./bin/Open3D/Open3D diff --git a/docs/getting_started.in.rst b/docs/getting_started.in.rst index d0d7388c189..b9f390ea835 100644 --- a/docs/getting_started.in.rst +++ b/docs/getting_started.in.rst @@ -153,10 +153,10 @@ C++ To get started with using Open3D in your C++ applications, you can download a binary package archive from `Github releases `__ (since `v0.15`). These binary -package archives contain the Open3D shared library built with all supported -features and are available for the main supported platforms. Also, the latest -development version (``HEAD`` of ``main`` branch) binary package archives are -provided here [#]_: +package archives contain the Open3D shared library, include headers and GUI / +rendering resources. These are built with all supported features and are +available for the main supported platforms. Also, the latest development version +(``HEAD`` of ``main`` branch) binary package archives are provided here [#]_: :Linux (Ubuntu 18.04+ or glibc 2.27+ [#]_): .. hlist:: @@ -203,19 +203,30 @@ Extract the archive and move the contents to a local folder (such as Linux / MacOSX: Windows: Open3D_install Open3D_install ├── include ├── bin - │ └── open3d │ └── Open3D.dll - │ ├── core ├── CMake - │ ├── ... │   ├── Open3DConfig.cmake - │   ├── Open3DConfig.h │ ├── ... - │   ├── Open3D.h ├── include - │   ├── ... │   └── open3d - └── lib │   ├── core - ├── cmake │   ├── ... - │   └── Open3D │   ├── Open3DConfig.h - │ ├── ... │   ├── Open3D.h - ├── libOpen3D.so │   ├── ... - ├── open3d_tf_ops.so └── lib - └── open3d_torch_ops.so └── Open3D.lib + │ └── open3d │ ├── Open3D.dll + │ ├── core │ └── resources + │ ├── ... │ ├── brightday_ibl.ktx + │   ├── Open3DConfig.h │ ├── ... + │   ├── Open3D.h │ + │   ├── ... ├── CMake + ├── lib │   ├── Open3DConfig.cmake + │ ├── cmake │ ├── ... + │ │   └── Open3D ├── include + │ │ ├── ... │   └── open3d + │ ├── pkgconfig │   ├── core + │ │   ├── Open3D.pc │   ├── ... + │ │ ├── ... │   ├── Open3DConfig.h + | | │   ├── Open3D.h + │ ├── libOpen3D.so │   ├── ... + │ ├── open3d_tf_ops.so └── lib + │ └── open3d_torch_ops.so └── Open3D.lib + └── share + └── resources +    ├── html + │ ├── ... + ├── brightday_ibl.ktx + ├── ... + Some files may be absent in the case of unsupported functionality. To use Open3D with your programs through `cmake`, add ``-D diff --git a/docs/release.md b/docs/release.md index 0364679d3e3..490c4fe85c5 100644 --- a/docs/release.md +++ b/docs/release.md @@ -12,17 +12,26 @@ Collect all release artifacts in the [Github draft release page](https://github. - [ ] Build Open3D C++ libraries (Linux, macOS, Windows x86_64) (CI) - [ ] Build docs (CI) and push to [Open3d_website repo](https://github.com/isl-org/Open3D_website) - [ ] Build arm64 wheels (with ML Ops): - - [ ] macOS 12. Python 3.8+ (desktop) + - [ ] macOS 12. Python 3.8+ (desktop): + + Configure: `cmake -DCMAKE_BUILD_TYPE=Release -DDEVELOPER_BUILD=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + -DBUILD_TENSORFLOW_OPS=ON -DBUILD_PYTORCH_OPS=ON -DBUNDLE_OPEN3D_ML=ON ..` + - [ ] Ubuntu 18.04 Python (all versions) (follow docs/arm.rst) (desktop) + + Build command: `cd docker; ./docker_build.sh openblas-arm64-py311; ...` + - [ ] Build Open3D app - [ ] Ubuntu 18.04, Windows 10, macOS 10.15 x86_64: (CI) - [ ] macOS 12 arm64 (desktop) - [ ] macOS (x86_64, arm64) sign (desktop) - - [ ] Windows app sign -- [ ] Upload wheels to PyPI + - [ ] (TBD) Windows app sign +- [ ] Testing: Run all (especially visualization) examples and Open3D viewer with + (Suzanne, Khronos helmet, large point cloud - apartment), on Linux x86-64, + macOS (x86-64, arm64), Windows x86-64. +- [ ] PyPI: Upload wheels - [ ] Github: Publish release with auto tag version -- [ ] Github: Create Open3D-ML release (auto tag version, auto-generate release - notes) +- [ ] Github: Create Open3D-ML release (auto tag version, auto-generate release notes) - [ ] `git pull` docs to website (Google cloud server `open3d:instance1`) - [ ] open3d.org (wordpress) : Update downloads table and post release notes - [ ] Upload release video to YouTube diff --git a/docs/versions.js b/docs/versions.js index ea58f05dabf..28a3b8d8839 100644 --- a/docs/versions.js +++ b/docs/versions.js @@ -22,8 +22,8 @@ document.write('\ 0.18.0 C++ (release)\ \ \ - 0.17.0 (release)\ - 0.17.0 C++ (release)\ + 0.17.0\ + 0.17.0 C++\ \ \ 0.16.0\ diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 1d48a8943aa..38ff654880e 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -185,7 +185,7 @@ build_pip_package() { "-DDEVELOPER_BUILD=$DEVELOPER_BUILD" "-DBUILD_COMMON_ISPC_ISAS=ON" "-DBUILD_AZURE_KINECT=$BUILD_AZURE_KINECT" - "-DBUILD_LIBREALSENSE=OFF" + "-DBUILD_LIBREALSENSE=ON" "-DGLIBCXX_USE_CXX11_ABI=$CXX11_ABI" "-DBUILD_TENSORFLOW_OPS=$BUILD_TENSORFLOW_OPS" "-DBUILD_PYTORCH_OPS=$BUILD_PYTORCH_OPS"