Build fixes for v0.18 (#6562)

* Use launcher for open3d viewer on Linux to correctly locate resources.
* Build TF ops in cxx11_abi packages
* Update node.js install method
* Fixes, update docs
* Re-enable librealsense build

cpp/apps/CMakeLists.txt

@@ -78,6 +78,7 @@ macro(open3d_add_app_gui SRC_DIR APP_NAME TARGET_NAME)
                 COMMAND cp -a "${CMAKE_BINARY_DIR}/${APP_NAME}" .
                 COMMAND cp "${SOURCE_DIR}/icon.svg" "${APP_NAME}/${APP_NAME}.svg"
                 COMMAND cp "${SOURCE_DIR}/${TARGET_NAME}.xml" "${APP_NAME}/"
+                COMMAND cp "${SOURCE_DIR}/${TARGET_NAME}Launcher.sh" "${APP_NAME}/"
                 COMMAND "${CMAKE_COMMAND}" -S .
                 COMMAND "${CMAKE_COMMAND}" --build . -t package
                 WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/package-${TARGET_NAME}-deb/"

cpp/apps/Open3DViewer/Debian/CMakeLists.in.txt

@@ -4,20 +4,27 @@ project("Open3D-Debian")
 
 message(STATUS "Building package for Debian")
 
+# How to set cpack prefix: https://stackoverflow.com/a/7363073/1255535
+set(CPACK_SET_DESTDIR true)
+set(CPACK_INSTALL_PREFIX /usr/local)
+
 # Install assets
 install(DIRECTORY   "Open3D"
         DESTINATION share
         USE_SOURCE_PERMISSIONS
         PATTERN     "Open3D/Open3D.svg" EXCLUDE
         PATTERN     "Open3D/Open3D.desktop" EXCLUDE
         PATTERN     "Open3D/Open3DViewer.xml" EXCLUDE
-        PATTERN     "Open3D/Open3D" EXCLUDE
         PATTERN     "Open3D/CMakeLists.txt" EXCLUDE
+        PATTERN     "Open3D/Open3DViewerLauncher.sh" EXCLUDE
 )
-install(FILES "Open3D/Open3D.desktop" DESTINATION /usr/share/applications)
+install(CODE "file(READ \"${CMAKE_BINARY_DIR}/Open3D/Open3D.desktop\" _DESKTOP_FILE)
+              string(REGEX REPLACE \"Exec=.*/bin/Open3D\" \"Exec=${CPACK_INSTALL_PREFIX}/bin/Open3D\" _DESKTOP_FILE \"\${_DESKTOP_FILE}\")
+              file(WRITE \"${CMAKE_BINARY_DIR}/Open3D/Open3D.deb.desktop\" \"\${_DESKTOP_FILE}\")")
+install(FILES "Open3D/Open3D.deb.desktop" DESTINATION /usr/share/applications RENAME "Open3D.desktop")
 install(FILES "Open3D/Open3DViewer.xml" DESTINATION /usr/share/mime/packages)
 install(FILES "Open3D/Open3D.svg" DESTINATION /usr/share/icons/hicolor/scalable/apps)
-install(PROGRAMS "Open3D/Open3D" DESTINATION bin)
+install(PROGRAMS "Open3D/Open3DViewerLauncher.sh" DESTINATION bin RENAME "Open3D")
 
 # CPACK parameter
 set(CPACK_GENERATOR "DEB")
@@ -29,8 +36,4 @@ set(CPACK_PACKAGE_VERSION "@OPEN3D_VERSION@")
 set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc++1, libgomp1, libpng16-16, libglfw3")
 set(CPACK_PACKAGE_HOMEPAGE_URL "@PROJECT_HOMEPAGE_URL@")
 
-# How to set cpack prefix: https://stackoverflow.com/a/7363073/1255535
-set(CPACK_SET_DESTDIR true)
-set(CPACK_INSTALL_PREFIX /usr/local)
-
 include(CPack)

cpp/apps/Open3DViewer/Open3DViewerLauncher.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+# Linux launcher script for Open3DViewer, installed in the system PATH with name
+# Open3D. The Open3D viewer binary is placed together with resources.
+SCRIPT=$(readlink -f "$0")
+INSTALL_DIRECTORY=$(readlink -f $(dirname "$SCRIPT")/..)
+OPEN3D_PATH="$INSTALL_DIRECTORY/share/Open3D/Open3D"
+"$OPEN3D_PATH" "$@" &

cpp/open3d/visualization/gui/Application.cpp

@@ -45,6 +45,7 @@ namespace {
 const double RUNLOOP_DELAY_SEC = 0.010;
 
 std::string FindResourcePath(int argc, const char *argv[]) {
+    namespace o3dfs = open3d::utility::filesystem;
     std::string argv0;
     if (argc != 0 && argv) {
         argv0 = argv[0];
@@ -66,7 +67,7 @@ std::string FindResourcePath(int argc, const char *argv[]) {
         // is absolute path, we're done
     } else {
         // relative path:  prepend working directory
-        auto cwd = open3d::utility::filesystem::GetWorkingDirectory();
+        auto cwd = o3dfs::GetWorkingDirectory();
 #ifdef __APPLE__
         // When running an app from the command line with the full relative
         // path (e.g. `bin/Open3D.app/Contents/MacOS/Open3D`), the working
@@ -85,14 +86,15 @@ std::string FindResourcePath(int argc, const char *argv[]) {
     }
 #endif  // __APPLE__
 
-    auto resource_path = path + "/resources";
-    if (!open3d::utility::filesystem::DirectoryExists(resource_path)) {
-        resource_path = path + "/../resources";  // building with Xcode
-        if (!open3d::utility::filesystem::DirectoryExists(resource_path)) {
-            resource_path = path + "/share/resources";  // GNU
+    for (auto &subpath :
+         {"/resources", "/../resources" /*building with Xcode */,
+          "/share/resources" /* GNU */, "/share/Open3D/resources" /* GNU */}) {
+        if (o3dfs::DirectoryExists(path + subpath)) {
+            return path + subpath;
         }
     }
-    return resource_path;
+    open3d::utility::LogError("Could not find resource directory.");
+    return "";
 }
 
 }  // namespace

docker/Dockerfile.ci

@@ -158,7 +158,12 @@ RUN source util/ci_utils.sh \
  && pip install -r python/requirements_test.txt
 
 # Open3D Jupyter dependencies
-RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \
+RUN mkdir -p /etc/apt/keyrings \
+ && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+ | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+ && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_16.x nodistro main" \
+ | tee /etc/apt/sources.list.d/nodesource.list \
+ && apt-get update \
  && apt-get install -y nodejs \
  && rm -rf /var/lib/apt/lists/* \
  && node --version \
@@ -175,7 +180,11 @@ RUN \
         export CMAKE_CXX_COMPILER=g++; \
         export CMAKE_C_COMPILER=gcc; \
         # TODO: PyTorch still use old CXX ABI, remove this line when PyTorch is updated
-        export GLIBCXX_USE_CXX11_ABI=OFF; \
+        if [ "$BUILD_PYTORCH_OPS" = "ON" ]; then \
+            export GLIBCXX_USE_CXX11_ABI=OFF; \
+        else \
+            export GLIBCXX_USE_CXX11_ABI=ON; \
+        fi \
     fi \
  && mkdir build \
  && cd build \

docker/docker_build.sh

@@ -44,21 +44,21 @@ OPTION:
 
     # Ubuntu CPU CI (Dockerfile.ci)
     cpu-static                  : Ubuntu CPU static
-    cpu-shared                  : Ubuntu CPU shared
-    cpu-shared-release          : Ubuntu CPU shared, release mode
-    cpu-shared-ml               : Ubuntu CPU shared with ML
-    cpu-shared-ml-release       : Ubuntu CPU shared with ML, release mode
+    cpu-shared                  : Ubuntu CPU shared (cxx11_abi)
+    cpu-shared-release          : Ubuntu CPU shared (cxx11_abi), release mode
+    cpu-shared-ml               : Ubuntu CPU shared with ML (pre_cxx11_abi)
+    cpu-shared-ml-release       : Ubuntu CPU shared with ML (pre_cxx11_abi), release mode
 
     # Sycl CPU CI (Dockerfile.ci)
     sycl-shared                : SYCL (oneAPI) with shared lib
     sycl-static                : SYCL (oneAPI) with static lib
 
     # ML CIs (Dockerfile.ci)
     2-bionic                   : CUDA CI, 2-bionic, developer mode
-    3-ml-shared-bionic-release : CUDA CI, 3-ml-shared-bionic, release mode
-    3-ml-shared-bionic         : CUDA CI, 3-ml-shared-bionic, developer mode
-    4-shared-bionic            : CUDA CI, 4-shared-bionic, developer mode
-    4-shared-bionic-release    : CUDA CI, 4-shared-bionic, release mode
+    3-ml-shared-bionic-release : CUDA CI, 3-ml-shared-bionic (pre_cxx11_abi), release mode
+    3-ml-shared-bionic         : CUDA CI, 3-ml-shared-bionic (pre_cxx11_abi), developer mode
+    4-shared-bionic            : CUDA CI, 4-shared-bionic (cxx11_abi), developer mode
+    4-shared-bionic-release    : CUDA CI, 4-shared-bionic (cxx11_abi), release mode
     5-ml-focal                 : CUDA CI, 5-ml-focal, developer mode
 
     # CUDA wheels (Dockerfile.wheel)
@@ -314,7 +314,8 @@ ci_build() {
     export PYTHON_VERSION=3.8
     export BUILD_SHARED_LIBS=ON
     export BUILD_CUDA_MODULE=ON
-    export BUILD_TENSORFLOW_OPS=OFF
+     # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi
+    export BUILD_TENSORFLOW_OPS=ON
     export BUILD_PYTORCH_OPS=OFF
     export PACKAGE=ON
     export BUILD_SYCL_MODULE=OFF
@@ -329,7 +330,8 @@ ci_build() {
     export PYTHON_VERSION=3.8
     export BUILD_SHARED_LIBS=ON
     export BUILD_CUDA_MODULE=ON
-    export BUILD_TENSORFLOW_OPS=OFF
+     # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi
+    export BUILD_TENSORFLOW_OPS=ON
     export BUILD_PYTORCH_OPS=OFF
     export PACKAGE=ON
     export BUILD_SYCL_MODULE=OFF
@@ -375,7 +377,8 @@ cpu-shared_export_env() {
     export PYTHON_VERSION=3.8
     export BUILD_SHARED_LIBS=ON
     export BUILD_CUDA_MODULE=OFF
-    export BUILD_TENSORFLOW_OPS=OFF
+     # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi
+    export BUILD_TENSORFLOW_OPS=ON
     export BUILD_PYTORCH_OPS=OFF
     export PACKAGE=ON
     export BUILD_SYCL_MODULE=OFF
@@ -406,7 +409,8 @@ cpu-shared-release_export_env() {
     export PYTHON_VERSION=3.8
     export BUILD_SHARED_LIBS=ON
     export BUILD_CUDA_MODULE=OFF
-    export BUILD_TENSORFLOW_OPS=OFF
+     # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi
+    export BUILD_TENSORFLOW_OPS=ON
     export BUILD_PYTORCH_OPS=OFF
     export PACKAGE=ON
     export BUILD_SYCL_MODULE=OFF

docker/docker_test.sh

@@ -106,18 +106,20 @@ cpp_python_linking_uninstall_test() {
     # - BUILD_PYTORCH_OPS
     # - BUILD_TENSORFLOW_OPS
     # - BUILD_SYCL_MODULE
+    # - NPROC (optional)
     echo "[cpp_python_linking_uninstall_test()] DOCKER_TAG=${DOCKER_TAG}"
     echo "[cpp_python_linking_uninstall_test()] BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}"
     echo "[cpp_python_linking_uninstall_test()] BUILD_CUDA_MODULE=${BUILD_CUDA_MODULE}"
     echo "[cpp_python_linking_uninstall_test()] BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS}"
     echo "[cpp_python_linking_uninstall_test()] BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS}"
     echo "[cpp_python_linking_uninstall_test()] BUILD_SYCL_MODULE=${BUILD_SYCL_MODULE}"
+    echo "[cpp_python_linking_uninstall_test()] NPROC=${NPROC:=$(nproc)}"
 
     # Config-dependent argument: gpu_run_args
     if [ "${BUILD_CUDA_MODULE}" == "ON" ]; then
-        docker_run="docker run --gpus all"
+        docker_run="docker run --cpus ${NPROC} --gpus all"
     else
-        docker_run="docker run"
+        docker_run="docker run --cpus ${NPROC}"
     fi
 
     # Config-dependent argument: pytest_args

docs/arm.rst

@@ -171,13 +171,14 @@ Compiling Open3D on ARM64 macOS
     # Optional: activate your virtualenv
     conda activate your-virtual-env
 
-    # Configure
+    # Configure and choose build options
     cd Open3D && mkdir build && cd build
-    cmake ..
+    cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TENSORFLOW_OPS=ON -DBUILD_PYTORCH_OPS=ON -DBUNDLE_OPEN3D_ML=ON ..
 
     # Build
-    make -j8
-    make install-pip-package -j8
+    make pip-package -j8    # Build Python wheel
+    make package -j8        # Build macOS devel binary package
+    make Open3DViewer -j8   # Build Open3D viewer app
 
     # Test C++ viewer app
     ./bin/Open3D/Open3D

docs/getting_started.in.rst

@@ -153,10 +153,10 @@ C++
 To get started with using Open3D in your C++ applications, you can download a
 binary package archive from `Github releases
 <https://github.com/isl-org/Open3D/releases>`__ (since `v0.15`). These binary
-package archives contain the Open3D shared library built with all supported
-features and are available for the main supported platforms. Also, the latest
-development version (``HEAD`` of ``main`` branch) binary package archives are
-provided here [#]_:
+package archives contain the Open3D shared library, include headers and GUI /
+rendering resources. These are built with all supported features and are
+available for the main supported platforms. Also, the latest development version
+(``HEAD`` of ``main`` branch) binary package archives are provided here [#]_:
 
 :Linux (Ubuntu 18.04+ or glibc 2.27+ [#]_):
     .. hlist::
@@ -203,19 +203,30 @@ Extract the archive and move the contents to a local folder (such as
     Linux / MacOSX:                       Windows:
     Open3D_install                        Open3D_install
     ├── include                           ├── bin
-    │   └── open3d                        │   └── Open3D.dll
-    │       ├── core                      ├── CMake
-    │       ├── ...                       │   ├── Open3DConfig.cmake
-    │       ├── Open3DConfig.h            │   ├── ...
-    │       ├── Open3D.h                  ├── include
-    │       ├── ...                       │   └── open3d
-    └── lib                               │       ├── core
-        ├── cmake                         │       ├── ...
-        │   └── Open3D                    │       ├── Open3DConfig.h
-        │        ├── ...                  │       ├── Open3D.h
-        ├── libOpen3D.so                  │       ├── ...
-        ├── open3d_tf_ops.so              └── lib
-        └── open3d_torch_ops.so               └── Open3D.lib
+    │   └── open3d                        │   ├── Open3D.dll
+    │       ├── core                      │   └── resources
+    │       ├── ...                       │       ├── brightday_ibl.ktx
+    │       ├── Open3DConfig.h            │       ├── ...
+    │       ├── Open3D.h                  │
+    │       ├── ...                       ├── CMake
+    ├── lib                               │   ├── Open3DConfig.cmake
+    │   ├── cmake                         │   ├── ...
+    │   │   └── Open3D                    ├── include
+    │   │        ├── ...                  │   └── open3d
+    │   ├── pkgconfig                     │       ├── core
+    │   │   ├── Open3D.pc                 │       ├── ...
+    │   │   ├── ...                       │       ├── Open3DConfig.h
+    |   |                                 │       ├── Open3D.h
+    │   ├── libOpen3D.so                  │       ├── ...
+    │   ├── open3d_tf_ops.so              └── lib
+    │   └── open3d_torch_ops.so               └── Open3D.lib
+    └── share
+        └── resources
+            ├── html
+            │    ├── ...
+            ├── brightday_ibl.ktx
+            ├── ...
+
 
 Some files may be absent in the case of unsupported functionality. To use Open3D
 with your programs through `cmake`, add ``-D

docs/release.md

@@ -12,17 +12,26 @@ Collect all release artifacts in the [Github draft release page](https://github.
 -   [ ] Build Open3D C++ libraries (Linux, macOS, Windows x86_64)  (CI)
 -   [ ] Build docs (CI) and push to [Open3d_website repo](https://github.com/isl-org/Open3D_website)
 -   [ ] Build arm64 wheels (with ML Ops):
-    -   [ ] macOS 12. Python 3.8+ (desktop)
+    -   [ ] macOS 12. Python 3.8+ (desktop):
+
+        Configure: `cmake -DCMAKE_BUILD_TYPE=Release -DDEVELOPER_BUILD=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON 
+                    -DBUILD_TENSORFLOW_OPS=ON -DBUILD_PYTORCH_OPS=ON -DBUNDLE_OPEN3D_ML=ON  ..`
+
     -   [ ] Ubuntu 18.04 Python (all versions) (follow docs/arm.rst) (desktop)
+
+        Build command: `cd docker; ./docker_build.sh openblas-arm64-py311; ...`
+
 -   [ ] Build Open3D app
     -   [ ] Ubuntu 18.04, Windows 10, macOS 10.15 x86_64: (CI)
     -   [ ] macOS 12 arm64 (desktop)
     -   [ ] macOS (x86_64, arm64) sign (desktop)
-    -   [ ] Windows app sign
--   [ ] Upload wheels to PyPI
+    -   [ ] (TBD) Windows app sign
+-   [ ] Testing: Run all (especially visualization) examples and Open3D viewer with
+        (Suzanne, Khronos helmet, large point cloud - apartment), on Linux x86-64,
+        macOS (x86-64, arm64), Windows x86-64.
+-   [ ] PyPI: Upload wheels
 -   [ ] Github: Publish release with auto tag version
--   [ ] Github: Create Open3D-ML release (auto tag version, auto-generate release
-    notes)
+-   [ ] Github: Create Open3D-ML release (auto tag version, auto-generate release notes)
 -   [ ] `git pull` docs to website (Google cloud server `open3d:instance1`)
 -   [ ] open3d.org (wordpress) : Update downloads table and post release notes
 -   [ ] Upload release video to YouTube

docs/versions.js

@@ -22,8 +22,8 @@ document.write('\
             <td><a href="https://www.open3d.org/docs/release/cpp_api">0.18.0 C++ (release)</a></td>\
         </tr>\
         <tr>\
-            <td><a href="https://www.open3d.org/docs/0.17.0">0.17.0 (release)</a></td>\
-            <td><a href="https://www.open3d.org/docs/0.17.0/cpp_api">0.17.0 C++ (release)</a></td>\
+            <td><a href="https://www.open3d.org/docs/0.17.0">0.17.0</a></td>\
+            <td><a href="https://www.open3d.org/docs/0.17.0/cpp_api">0.17.0 C++</a></td>\
         </tr>\
         <tr>\
             <td><a href="https://www.open3d.org/docs/0.16.0">0.16.0</a></td>\

util/ci_utils.sh

@@ -185,7 +185,7 @@ build_pip_package() {
         "-DDEVELOPER_BUILD=$DEVELOPER_BUILD"
         "-DBUILD_COMMON_ISPC_ISAS=ON"
         "-DBUILD_AZURE_KINECT=$BUILD_AZURE_KINECT"
-        "-DBUILD_LIBREALSENSE=OFF"
+        "-DBUILD_LIBREALSENSE=ON"
         "-DGLIBCXX_USE_CXX11_ABI=$CXX11_ABI"
         "-DBUILD_TENSORFLOW_OPS=$BUILD_TENSORFLOW_OPS"
         "-DBUILD_PYTORCH_OPS=$BUILD_PYTORCH_OPS"