From beaf1b45f4c5e92d26f2b42e6ad41d9403bbd3d1 Mon Sep 17 00:00:00 2001 From: Benjamin Ummenhofer Date: Wed, 12 Aug 2020 02:09:40 +0200 Subject: [PATCH] rpc visualization interface (#2090) --- .codacy.yml | 3 + 3rdparty/README.txt | 12 + 3rdparty/boost/boost.cmake | 38 ++ 3rdparty/find_dependencies.cmake | 36 ++ 3rdparty/msgpack/msgpack_build.cmake | 16 + 3rdparty/zeromq/zeromq_build.cmake | 41 ++ CHANGELOG.md | 2 + CMakeLists.txt | 9 + cpp/open3d/io/CMakeLists.txt | 5 + cpp/open3d/io/rpc/BufferConnection.cpp | 61 +++ cpp/open3d/io/rpc/BufferConnection.h | 57 +++ cpp/open3d/io/rpc/Connection.cpp | 99 +++++ cpp/open3d/io/rpc/Connection.h | 69 +++ cpp/open3d/io/rpc/ConnectionBase.h | 53 +++ cpp/open3d/io/rpc/DummyReceiver.cpp | 49 +++ cpp/open3d/io/rpc/DummyReceiver.h | 84 ++++ cpp/open3d/io/rpc/MessageUtils.cpp | 102 +++++ cpp/open3d/io/rpc/MessageUtils.h | 77 ++++ cpp/open3d/io/rpc/Messages.h | 345 +++++++++++++++ cpp/open3d/io/rpc/ReceiverBase.cpp | 269 ++++++++++++ cpp/open3d/io/rpc/ReceiverBase.h | 129 ++++++ cpp/open3d/io/rpc/RemoteFunctions.cpp | 414 ++++++++++++++++++ cpp/open3d/io/rpc/RemoteFunctions.h | 187 ++++++++ cpp/open3d/io/rpc/ZMQContext.cpp | 42 ++ cpp/open3d/io/rpc/ZMQContext.h | 42 ++ cpp/open3d/utility/CMakeLists.txt | 2 +- cpp/pybind/CMakeLists.txt | 6 + cpp/pybind/_build_config.py.in | 1 + cpp/pybind/io/io.cpp | 3 + cpp/pybind/io/io.h | 3 + cpp/pybind/io/rpc.cpp | 173 ++++++++ cpp/tests/CMakeLists.txt | 5 + cpp/tests/io/rpc/RemoteFunctions.cpp | 241 ++++++++++ python/MANIFEST.in | 3 + python/open3d/__init__.py | 2 +- python/open3d/visualization/__init__.py | 3 + .../visualization/_external_visualizer.py | 131 ++++++ python/test/test_remote_functions.py | 74 ++++ util/run_ci.sh | 9 + 39 files changed, 2895 insertions(+), 2 deletions(-) create mode 100644 .codacy.yml create mode 100644 3rdparty/boost/boost.cmake create mode 100644 3rdparty/msgpack/msgpack_build.cmake create mode 100644 3rdparty/zeromq/zeromq_build.cmake create mode 100644 cpp/open3d/io/rpc/BufferConnection.cpp create mode 100644 cpp/open3d/io/rpc/BufferConnection.h create mode 100644 cpp/open3d/io/rpc/Connection.cpp create mode 100644 cpp/open3d/io/rpc/Connection.h create mode 100644 cpp/open3d/io/rpc/ConnectionBase.h create mode 100644 cpp/open3d/io/rpc/DummyReceiver.cpp create mode 100644 cpp/open3d/io/rpc/DummyReceiver.h create mode 100644 cpp/open3d/io/rpc/MessageUtils.cpp create mode 100644 cpp/open3d/io/rpc/MessageUtils.h create mode 100644 cpp/open3d/io/rpc/Messages.h create mode 100644 cpp/open3d/io/rpc/ReceiverBase.cpp create mode 100644 cpp/open3d/io/rpc/ReceiverBase.h create mode 100644 cpp/open3d/io/rpc/RemoteFunctions.cpp create mode 100644 cpp/open3d/io/rpc/RemoteFunctions.h create mode 100644 cpp/open3d/io/rpc/ZMQContext.cpp create mode 100644 cpp/open3d/io/rpc/ZMQContext.h create mode 100644 cpp/pybind/io/rpc.cpp create mode 100644 cpp/tests/io/rpc/RemoteFunctions.cpp create mode 100644 python/open3d/visualization/_external_visualizer.py create mode 100644 python/test/test_remote_functions.py diff --git a/.codacy.yml b/.codacy.yml new file mode 100644 index 00000000000..17cb843980e --- /dev/null +++ b/.codacy.yml @@ -0,0 +1,3 @@ +--- +exclude_paths: + - CHANGELOG.md diff --git a/3rdparty/README.txt b/3rdparty/README.txt index 9410ac20636..9987d719edf 100644 --- a/3rdparty/README.txt +++ b/3rdparty/README.txt @@ -98,3 +98,15 @@ https://github.com/NVIDIA/cutlass benchmark 1.5.0 Apache-2 license A microbenchmark support library https://github.com/google/benchmark +-------------------------------------------------------------------------------- +msgpack-c da2fc25f8 Boost Software License 1.0 +MessagePack implementation for C and C++ +https://github.com/msgpack/msgpack-c/tree/cpp_master +-------------------------------------------------------------------------------- +libzmq 4.3.2 LGPLv3 + static link exception license +ZeroMQ is a high-performance asynchronous messaging library +https://github.com/zeromq/libzmq +-------------------------------------------------------------------------------- +cppzmq 4.6.0 MIT license +Header-only C++ binding for libzmq +https://github.com/zeromq/cppzmq diff --git a/3rdparty/boost/boost.cmake b/3rdparty/boost/boost.cmake new file mode 100644 index 00000000000..26723dbea88 --- /dev/null +++ b/3rdparty/boost/boost.cmake @@ -0,0 +1,38 @@ +# Build system for header-only boost libraries. +# +# In general, we prefer avoiding boost or use header-only boost libraries. +# Compiling boost libraries can addup to the build time. +# +# Current boost libraries: +# - predef (header-only) + +include(ExternalProject) + +if(WIN32) + message(FATAL_ERROR "Win32 not supported.") +endif() + +ExternalProject_Add( + ext_boost + PREFIX boost + GIT_REPOSITORY https://github.com/boostorg/boost.git + GIT_TAG boost-1.73.0 + GIT_SUBMODULES tools/boostdep libs/predef # Only need a subset of boost + GIT_SHALLOW ON # git clone --depth 1 + GIT_SUBMODULES_RECURSE OFF + BUILD_IN_SOURCE ON + CONFIGURE_COMMAND "" + BUILD_COMMAND echo "Running Boost build..." + COMMAND python tools/boostdep/depinst/depinst.py predef + COMMAND ./bootstrap.sh + COMMAND ./b2 headers + UPDATE_COMMAND "" + INSTALL_COMMAND "" +) + +ExternalProject_Get_Property(ext_boost SOURCE_DIR) +message(STATUS "Boost source dir: ${SOURCE_DIR}") + +# By default, BOOST_INCLUDE_DIRS should not have trailing "/". +# The actual headers files are located in `${SOURCE_DIR}/boost`. +set(BOOST_INCLUDE_DIRS ${SOURCE_DIR}/ext_boost) diff --git a/3rdparty/find_dependencies.cmake b/3rdparty/find_dependencies.cmake index de26fbea740..fe547fe27ae 100644 --- a/3rdparty/find_dependencies.cmake +++ b/3rdparty/find_dependencies.cmake @@ -204,6 +204,7 @@ endfunction() function(import_3rdparty_library name) cmake_parse_arguments(arg "PUBLIC;HEADER" "LIB_DIR" "INCLUDE_DIRS;LIBRARIES" ${ARGN}) if(arg_UNPARSED_ARGUMENTS) + message(STATUS "Unparsed: ${arg_UNPARSED_ARGUMENTS}") message(FATAL_ERROR "Invalid syntax: import_3rdparty_library(${name} ${ARGN})") endif() if(NOT arg_LIB_DIR) @@ -865,6 +866,41 @@ if(BUILD_GUI) list(APPEND Open3D_3RDPARTY_PRIVATE_TARGETS "${FILAMENT_TARGET}") endif() +# RPC interface +# - boost: predef +# - zeromq +# - msgpack +if(BUILD_RPC_INTERFACE) + # boost: predef + include(${Open3D_3RDPARTY_DIR}/boost/boost.cmake) + import_3rdparty_library(3rdparty_boost + INCLUDE_DIRS ${BOOST_INCLUDE_DIRS} + ) + set(BOOST_TARGET "3rdparty_boost") + add_dependencies(3rdparty_boost ext_boost) + list(APPEND Open3D_3RDPARTY_PRIVATE_TARGETS "${BOOST_TARGET}") + + # zeromq + include(${Open3D_3RDPARTY_DIR}/zeromq/zeromq_build.cmake) + import_3rdparty_library(3rdparty_zeromq + INCLUDE_DIRS ${ZEROMQ_INCLUDE_DIRS} + LIB_DIR ${ZEROMQ_LIB_DIR} + LIBRARIES ${ZEROMQ_LIBRARIES} + ) + set(ZEROMQ_TARGET "3rdparty_zeromq") + add_dependencies(${ZEROMQ_TARGET} ext_zeromq) + list(APPEND Open3D_3RDPARTY_PRIVATE_TARGETS "${ZEROMQ_TARGET}") + + # msgpack + include(${Open3D_3RDPARTY_DIR}/msgpack/msgpack_build.cmake) + import_3rdparty_library(3rdparty_msgpack + INCLUDE_DIRS ${MSGPACK_INCLUDE_DIRS} + ) + set(MSGPACK_TARGET "3rdparty_msgpack") + add_dependencies(3rdparty_msgpack ext_msgpack-c) + list(APPEND Open3D_3RDPARTY_PRIVATE_TARGETS "${MSGPACK_TARGET}") +endif() + # MKL, cuSOLVER, cuBLAS # We link MKL statically. For MKL link flags, refer to: # https://software.intel.com/content/www/us/en/develop/articles/intel-mkl-link-line-advisor.html diff --git a/3rdparty/msgpack/msgpack_build.cmake b/3rdparty/msgpack/msgpack_build.cmake new file mode 100644 index 00000000000..d817d46ad14 --- /dev/null +++ b/3rdparty/msgpack/msgpack_build.cmake @@ -0,0 +1,16 @@ +include(ExternalProject) + +ExternalProject_Add( + ext_msgpack-c + PREFIX msgpack-c + URL https://github.com/msgpack/msgpack-c/releases/download/cpp-3.3.0/msgpack-3.3.0.tar.gz + URL_HASH MD5=e676575d52caae974e579c3d5f0ba6a2 + # do not configure + CONFIGURE_COMMAND "" + # do not build + BUILD_COMMAND "" + # do not install + INSTALL_COMMAND "" + ) +ExternalProject_Get_Property( ext_msgpack-c SOURCE_DIR ) +set( MSGPACK_INCLUDE_DIRS "${SOURCE_DIR}/include/" ) diff --git a/3rdparty/zeromq/zeromq_build.cmake b/3rdparty/zeromq/zeromq_build.cmake new file mode 100644 index 00000000000..8152f7f1e2c --- /dev/null +++ b/3rdparty/zeromq/zeromq_build.cmake @@ -0,0 +1,41 @@ +include(ExternalProject) +include(FetchContent) + +# ExternalProject seems to be the best solution for including zeromq. +# The projects defines options which clash with and pollute our CMake cache. + +ExternalProject_Add( + ext_zeromq + PREFIX "${CMAKE_BINARY_DIR}/zeromq" + URL "https://github.com/zeromq/libzmq/releases/download/v4.3.2/zeromq-4.3.2.tar.gz" + URL_HASH MD5=2047e917c2cc93505e2579bcba67a573 + # do not update + UPDATE_COMMAND "" + CMAKE_ARGS + -DBUILD_STATIC=ON + -DBUILD_SHARED=OFF + -DBUILD_TESTS=OFF + -DENABLE_CPACK=OFF + -DENABLE_CURVE=OFF + -DZMQ_BUILD_TESTS=OFF + -DWITH_DOCS=OFF + -DWITH_PERF_TOOL=OFF + -DCMAKE_INSTALL_PREFIX= +) + +# cppzmq is header only. we just need to download +FetchContent_Declare( + ext_cppzmq + URL "https://github.com/zeromq/cppzmq/archive/v4.6.0.tar.gz" + URL_HASH MD5=7cae1b3fbfeaddb9cf1f70e99a98add2 +) +FetchContent_GetProperties(ext_cppzmq) +if(NOT ext_cppzmq_POPULATED) + FetchContent_Populate(ext_cppzmq) + # do not add subdirectory here +endif() + +ExternalProject_Get_Property( ext_zeromq INSTALL_DIR ) +set(ZEROMQ_LIBRARIES zmq) +set(ZEROMQ_LIB_DIR ${INSTALL_DIR}/lib) +set(ZEROMQ_INCLUDE_DIRS "${INSTALL_DIR}/include/;${ext_cppzmq_SOURCE_DIR}/") diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d2f2fb5ae4..16ed8499eea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ * Added option BUILD_BENCHMARKS for building microbenchmarks * Extend Python API of UniformTSDFVolume to allow setting the origin * Corrected documentation of PointCloud.h +* Added an RPC interface for external visualizers running in a separate process + ## 0.9.0 * Version bump to 0.9.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index b81a5667243..1651e1be740 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,7 @@ option(BUILD_AZURE_KINECT "Build support for Azure Kinect sensor" OFF # ML library options option(BUILD_TENSORFLOW_OPS "Build ops for Tensorflow" OFF) option(BUILD_PYTORCH_OPS "Build ops for Pytorch" OFF) +option(BUILD_RPC_INTERFACE "Build the RPC interface" OFF) set(FILAMENT_PRECOMPILED_ROOT "" CACHE PATH "Path to precompiled Filament library (used if BUILD_FILAMENT_FROM_SOURCE=OFF)") set(FILAMENT_SOURCE_ROOT "" CACHE PATH "Path to Filament library sources (used if BUILD_FILAMENT_FROM_SOURCE=ON)") @@ -104,6 +105,10 @@ if(BUILD_GUI AND WIN32) message(STATUS "New GUI is currently unsupported on Windows") set(BUILD_GUI OFF) endif() +if(BUILD_RPC_INTERFACE AND WIN32) + message(STATUS "The RPC interface is currently unsupported on Windows.") + set(BUILD_RPC_INTERFACE OFF) +endif() # Parse Open3D version number file(STRINGS "cpp/open3d/version.txt" OPEN3D_VERSION_READ) @@ -301,6 +306,9 @@ function(open3d_set_global_properties target) if(BUILD_CUDA_MODULE) target_compile_definitions(${target} PRIVATE BUILD_CUDA_MODULE) endif() + if(BUILD_RPC_INTERFACE) + target_compile_definitions(${target} PRIVATE BUILD_RPC_INTERFACE) + endif() if(GLIBCXX_USE_CXX11_ABI) target_compile_definitions(${target} PUBLIC _GLIBCXX_USE_CXX11_ABI=1) else() @@ -426,6 +434,7 @@ endif() open3d_aligned_print("Build Tensorflow Ops" "${BUILD_TENSORFLOW_OPS}") open3d_aligned_print("Build Pytorch Ops" "${BUILD_PYTORCH_OPS}") open3d_aligned_print("Build Benchmarks" "${BUILD_BENCHMARKS}") +open3d_aligned_print("Build RPC interface" "${BUILD_RPC_INTERFACE}") if(GLIBCXX_USE_CXX11_ABI) set(usage "1") else() diff --git a/cpp/open3d/io/CMakeLists.txt b/cpp/open3d/io/CMakeLists.txt index 323a84ced2f..483aee29044 100644 --- a/cpp/open3d/io/CMakeLists.txt +++ b/cpp/open3d/io/CMakeLists.txt @@ -8,6 +8,11 @@ if (BUILD_AZURE_KINECT) set(IO_ALL_SOURCE_FILES ${IO_ALL_SOURCE_FILES} ${SENSOR_SOURCE_FILES}) endif () +if (BUILD_RPC_INTERFACE) + file(GLOB RPC_SOURCE_FILES "rpc/*.cpp") + set(IO_ALL_SOURCE_FILES ${IO_ALL_SOURCE_FILES} ${RPC_SOURCE_FILES}) +endif() + # Create object library add_library(io OBJECT ${IO_ALL_SOURCE_FILES} diff --git a/cpp/open3d/io/rpc/BufferConnection.cpp b/cpp/open3d/io/rpc/BufferConnection.cpp new file mode 100644 index 00000000000..0951c417210 --- /dev/null +++ b/cpp/open3d/io/rpc/BufferConnection.cpp @@ -0,0 +1,61 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#include "open3d/io/rpc/BufferConnection.h" + +#include + +#include "open3d/io/rpc/Messages.h" +#include "open3d/utility/Console.h" + +using namespace open3d::utility; + +namespace open3d { +namespace io { +namespace rpc { + +std::shared_ptr BufferConnection::Send( + zmq::message_t& send_msg) { + buffer_.write((char*)send_msg.data(), send_msg.size()); + + auto OK = messages::Status::OK(); + msgpack::sbuffer sbuf; + messages::Reply reply{OK.MsgId()}; + msgpack::pack(sbuf, reply); + msgpack::pack(sbuf, OK); + return std::shared_ptr( + new zmq::message_t(sbuf.data(), sbuf.size())); +} + +std::shared_ptr BufferConnection::Send(const void* data, + size_t size) { + zmq::message_t send_msg(data, size); + return Send(send_msg); +} + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/BufferConnection.h b/cpp/open3d/io/rpc/BufferConnection.h new file mode 100644 index 00000000000..62c7616a803 --- /dev/null +++ b/cpp/open3d/io/rpc/BufferConnection.h @@ -0,0 +1,57 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#pragma once + +#include +#include + +#include "open3d/io/rpc/ConnectionBase.h" + +namespace open3d { +namespace io { +namespace rpc { + +/// Implements a connection writing to a buffer +class BufferConnection : public ConnectionBase { +public: + BufferConnection() {} + + /// Function for sending data wrapped in a zmq message object. + std::shared_ptr Send(zmq::message_t& send_msg); + + /// Function for sending raw data. Meant for testing purposes + std::shared_ptr Send(const void* data, size_t size); + + std::stringstream& buffer() { return buffer_; } + const std::stringstream& buffer() const { return buffer_; } + +private: + std::stringstream buffer_; +}; +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/Connection.cpp b/cpp/open3d/io/rpc/Connection.cpp new file mode 100644 index 00000000000..31d9d86fe74 --- /dev/null +++ b/cpp/open3d/io/rpc/Connection.cpp @@ -0,0 +1,99 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#include "open3d/io/rpc/Connection.h" + +#include + +#include "open3d/io/rpc/ZMQContext.h" +#include "open3d/utility/Console.h" + +using namespace open3d::utility; + +namespace { + +struct ConnectionDefaults { + std::string address = "tcp://localhost:51454"; + int connect_timeout = 5000; + int timeout = 10000; +} defaults; + +} // namespace + +namespace open3d { +namespace io { +namespace rpc { + +Connection::Connection() + : Connection(defaults.address, defaults.connect_timeout, defaults.timeout) { +} + +Connection::Connection(const std::string& address, + int connect_timeout, + int timeout) + : socket_(new zmq::socket_t(GetZMQContext(), ZMQ_REQ)), + address_(address), + connect_timeout_(connect_timeout), + timeout_(timeout) { + socket_->setsockopt(ZMQ_LINGER, timeout); + socket_->setsockopt(ZMQ_CONNECT_TIMEOUT, connect_timeout); + socket_->setsockopt(ZMQ_RCVTIMEO, timeout); + socket_->setsockopt(ZMQ_SNDTIMEO, timeout); + socket_->connect(address.c_str()); +} + +Connection::~Connection() { socket_->close(); } + +std::shared_ptr Connection::Send(zmq::message_t& send_msg) { + if (!socket_->send(send_msg)) { + zmq::error_t err; + if (err.num()) { + LogInfo("Connection::send() send failed with: {}", err.what()); + } + } + + std::shared_ptr msg(new zmq::message_t()); + if (socket_->recv(*msg)) { + LogDebug("Connection::send() received answer with {} bytes", + msg->size()); + } else { + zmq::error_t err; + if (err.num()) { + LogInfo("Connection::send() recv failed with: {}", err.what()); + } + } + return msg; +} + +std::shared_ptr Connection::Send(const void* data, + size_t size) { + zmq::message_t send_msg(data, size); + return Send(send_msg); +} + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/Connection.h b/cpp/open3d/io/rpc/Connection.h new file mode 100644 index 00000000000..dfd344f65f5 --- /dev/null +++ b/cpp/open3d/io/rpc/Connection.h @@ -0,0 +1,69 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#pragma once + +#include +#include + +#include "open3d/io/rpc/ConnectionBase.h" + +namespace open3d { +namespace io { +namespace rpc { + +/// This class implements the Connection which is used as default in all +/// functions. +class Connection : public ConnectionBase { +public: + Connection(); + + /// Creates a Connection object used for sending data. + /// \param address The address of the receiving end. + /// + /// \param connect_timeout The timeout for the connect operation of the + /// socket. + /// + /// \param timeout The timeout for sending data. + /// + Connection(const std::string& address, int connect_timeout, int timeout); + ~Connection(); + + /// Function for sending data wrapped in a zmq message object. + std::shared_ptr Send(zmq::message_t& send_msg); + + /// Function for sending raw data. Meant for testing purposes + std::shared_ptr Send(const void* data, size_t size); + +private: + std::unique_ptr socket_; + const std::string address_; + const int connect_timeout_; + const int timeout_; +}; +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/ConnectionBase.h b/cpp/open3d/io/rpc/ConnectionBase.h new file mode 100644 index 00000000000..be6d4554553 --- /dev/null +++ b/cpp/open3d/io/rpc/ConnectionBase.h @@ -0,0 +1,53 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#pragma once + +#include + +namespace zmq { +class message_t; +class socket_t; +} // namespace zmq + +namespace open3d { +namespace io { +namespace rpc { + +/// Base class for all connections +class ConnectionBase { +public: + ConnectionBase(){}; + virtual ~ConnectionBase(){}; + + /// Function for sending data wrapped in a zmq message object. + virtual std::shared_ptr Send(zmq::message_t& send_msg) = 0; + virtual std::shared_ptr Send(const void* data, + size_t size) = 0; +}; +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/DummyReceiver.cpp b/cpp/open3d/io/rpc/DummyReceiver.cpp new file mode 100644 index 00000000000..2d276f1a27c --- /dev/null +++ b/cpp/open3d/io/rpc/DummyReceiver.cpp @@ -0,0 +1,49 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#include "open3d/io/rpc/DummyReceiver.h" + +#include + +#include "open3d/io/rpc/Messages.h" + +namespace open3d { +namespace io { +namespace rpc { + +std::shared_ptr DummyReceiver::CreateStatusOKMsg() { + auto OK = messages::Status::OK(); + msgpack::sbuffer sbuf; + messages::Reply reply{OK.MsgId()}; + msgpack::pack(sbuf, reply); + msgpack::pack(sbuf, OK); + return std::shared_ptr( + new zmq::message_t(sbuf.data(), sbuf.size())); +} + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/DummyReceiver.h b/cpp/open3d/io/rpc/DummyReceiver.h new file mode 100644 index 00000000000..bb5b05b5bc4 --- /dev/null +++ b/cpp/open3d/io/rpc/DummyReceiver.h @@ -0,0 +1,84 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#pragma once + +#include "open3d/io/rpc/ReceiverBase.h" + +namespace open3d { +namespace io { +namespace rpc { + +/// Receiver implementation which always returns a successful status. +/// This class is meant for testing puproses. +class DummyReceiver : public ReceiverBase { +public: + DummyReceiver(const std::string& address, int timeout) + : ReceiverBase(address, timeout) {} + + std::shared_ptr CreateStatusOKMsg(); + + std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::SetMeshData& msg, + const MsgpackObject& obj) override { + return CreateStatusOKMsg(); + } + std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::GetMeshData& msg, + const MsgpackObject& obj) override { + return CreateStatusOKMsg(); + } + std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::SetCameraData& msg, + const MsgpackObject& obj) override { + return CreateStatusOKMsg(); + } + std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::SetProperties& msg, + const MsgpackObject& obj) override { + return CreateStatusOKMsg(); + } + std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::SetActiveCamera& msg, + const MsgpackObject& obj) override { + return CreateStatusOKMsg(); + } + std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::SetTime& msg, + const MsgpackObject& obj) override { + return CreateStatusOKMsg(); + } +}; + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/MessageUtils.cpp b/cpp/open3d/io/rpc/MessageUtils.cpp new file mode 100644 index 00000000000..f82c6bf3d84 --- /dev/null +++ b/cpp/open3d/io/rpc/MessageUtils.cpp @@ -0,0 +1,102 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#include "open3d/io/rpc/MessageUtils.h" + +#include + +#include "open3d/io/rpc/Messages.h" +#include "open3d/utility/Console.h" + +using namespace open3d::utility; + +namespace open3d { +namespace io { +namespace rpc { + +std::shared_ptr UnpackStatusFromReply( + const zmq::message_t& msg, size_t& offset, bool& ok) { + ok = false; + if (msg.size() <= offset) { + return std::shared_ptr(); + }; + + messages::Reply reply; + messages::Status status; + try { + auto obj_handle = + msgpack::unpack((char*)msg.data(), msg.size(), offset); + obj_handle.get().convert(reply); + if (reply.msg_id != status.MsgId()) { + LogDebug("Expected msg with id {} but got {}", status.MsgId(), + reply.msg_id); + } else { + auto status_obj_handle = + msgpack::unpack((char*)msg.data(), msg.size(), offset); + status_obj_handle.get().convert(status); + ok = true; + } + } catch (std::exception& e) { + LogDebug("Failed to parse message: {}", e.what()); + offset = msg.size(); + } + return std::make_shared(status); +} + +bool ReplyIsOKStatus(const zmq::message_t& msg) { + size_t offset = 0; + return ReplyIsOKStatus(msg, offset); +} + +bool ReplyIsOKStatus(const zmq::message_t& msg, size_t& offset) { + bool ok; + auto status = UnpackStatusFromReply(msg, offset, ok); + if (ok && status && 0 == status->code) { + return true; + } + return false; +} + +std::string CreateSerializedRequestMessage(const std::string& msg_id) { + messages::Request request{msg_id}; + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, request); + return std::string(sbuf.data(), sbuf.size()); +} + +std::tuple GetZMQMessageDataAndSize( + const zmq::message_t& msg) { + return std::make_tuple(msg.data(), msg.size()); +} + +std::tuple GetStatusCodeAndStr( + const messages::Status& status) { + return std::make_tuple(status.code, status.str); +} + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/MessageUtils.h b/cpp/open3d/io/rpc/MessageUtils.h new file mode 100644 index 00000000000..d57134e750e --- /dev/null +++ b/cpp/open3d/io/rpc/MessageUtils.h @@ -0,0 +1,77 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#pragma once + +#include "open3d/io/rpc/ReceiverBase.h" + +namespace zmq { +class message_t; +} + +namespace open3d { +namespace io { +namespace rpc { + +namespace messages { +struct Status; +} + +/// Helper function for unpacking the Status message from a reply. +/// \param msg The message that contains the Reply and the Status messages. +/// +/// \param offset Byte offset into the message. Defines where to start parsing +/// the message. The offset will be updated and will point to the first byte +/// after the parse messages. If unpacking fails offset will be set to the end +/// of the message. +/// +/// \param ok Output variable which will be set to true if the unpacking +/// was successful. +/// +/// \return The extracted Status message object. Check \param ok to see if the +/// returned object is valid. +std::shared_ptr UnpackStatusFromReply( + const zmq::message_t& msg, size_t& offset, bool& ok); + +/// Convenience function for checking if the message is an OK. +bool ReplyIsOKStatus(const zmq::message_t& msg); + +/// Convenience function for checking if the message is an OK. +/// \param offset \see UnpackStatusFromReply +bool ReplyIsOKStatus(const zmq::message_t& msg, size_t& offset); + +/// Creates a serialized Request message for testing purposes. +std::string CreateSerializedRequestMessage(const std::string& msg_id); + +std::tuple GetZMQMessageDataAndSize( + const zmq::message_t& msg); + +std::tuple GetStatusCodeAndStr( + const messages::Status& status); + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/Messages.h b/cpp/open3d/io/rpc/Messages.h new file mode 100644 index 00000000000..3b93237119b --- /dev/null +++ b/cpp/open3d/io/rpc/Messages.h @@ -0,0 +1,345 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#pragma once +#include + +#include +#include +#include +#include + +#if BOOST_ENDIAN_LITTLE_BYTE +#define ENDIANNESS_STR "<" +#elif BOOST_ENDIAN_BIG_BYTE +#define ENDIANNESS_STR ">" +#else +#error Cannot determine endianness +#endif + +namespace open3d { +namespace io { +namespace rpc { +namespace messages { + +/// Template function for converting types to their string representation. +/// E.g. TypeStr() -> " +inline std::string TypeStr() { + return ""; +} +template <> +inline std::string TypeStr() { + return ENDIANNESS_STR "f4"; +} +template <> +inline std::string TypeStr() { + return ENDIANNESS_STR "f8"; +} +template <> +inline std::string TypeStr() { + return "|i1"; +} +template <> +inline std::string TypeStr() { + return ENDIANNESS_STR "i2"; +} +template <> +inline std::string TypeStr() { + return ENDIANNESS_STR "i4"; +} +template <> +inline std::string TypeStr() { + return ENDIANNESS_STR "i8"; +} +template <> +inline std::string TypeStr() { + return "|u1"; +} +template <> +inline std::string TypeStr() { + return ENDIANNESS_STR "u2"; +} +template <> +inline std::string TypeStr() { + return ENDIANNESS_STR "u4"; +} +template <> +inline std::string TypeStr() { + return ENDIANNESS_STR "u8"; +} + +#undef ENDIANNESS_STR + +/// Array structure inspired by msgpack_numpy but not directly compatible +/// because they use bin-type for the map keys and we must use string. +/// This structure does not have ownership of the data. +/// +/// The following code can be used in python to create a compatible dict +/// +/// def numpy_to_Array(arr): +/// if isinstance(arr, np.ndarray): +/// return {'type': arr.dtype.str, +/// 'shape': arr.shape, +/// 'data': arr.tobytes()} +/// raise Exception('object is not a numpy array') +/// +/// +/// This codes converts the dict back to numpy.ndarray +/// +/// def Array_to_numpy(dic): +/// return np.frombuffer(dic['data'], +/// dtype=np.dtype(dic['type'])).reshape(dic['shape']) +/// +struct Array { + static std::string MsgId() { return "array"; } + + template + static Array FromPtr(const T* const ptr, + const std::vector& shape) { + Array arr; + arr.type = TypeStr(); + arr.shape = shape; + arr.data.ptr = (const char*)ptr; + int64_t num = 1; + for (int64_t n : shape) num *= n; + arr.data.size = sizeof(T) * num; + return arr; + } + std::string type; + std::vector shape; + msgpack::type::raw_ref data; + + template + const T* Ptr() { + return (T*)data.ptr; + } + + // macro for creating the serialization/deserialization code + MSGPACK_DEFINE_MAP(type, shape, data); +}; + +/// struct for storing MeshData, e.g., PointClouds, TriangleMesh, .. +struct MeshData { + static std::string MsgId() { return "mesh_data"; } + + /// shape must be [num_verts,3] + Array vertices; + /// stores arbitrary attributes for each vertex, hence the first dim must + /// be num_verts + std::map vertex_attributes; + + /// This array stores vertex indices to define faces. + /// The array can be of rank 1 or 2. + /// An array of rank 2 with shape [num_faces,n] defines num_faces n-gons. + /// If the rank of the array is 1 then polys of different lengths are stored + /// sequentially. Each polygon is stored as a sequence 'n i1 i2 ... in' with + /// n>=3. The type of the array must be int32_t or int64_t + Array faces; + /// stores arbitrary attributes for each face + std::map face_attributes; + + /// This array stores vertex indices to define lines. + /// The array can be of rank 1 or 2. + /// An array of rank 2 with shape [num_lines,n] defines num_lines linestrips + /// with n vertices. If the rank of the array is 1 then linestrips with + /// different number of veertices are stored sequentially. Each linestrip is + /// stored as a sequence 'n i1 i2 ... in' with n>=2. The type of the array + /// must be int32_t or int64_t + Array lines; + /// stores arbitrary attributes for each line + std::map line_attributes; + + /// map of arrays that can be interpreted as textures + std::map textures; + + MSGPACK_DEFINE_MAP(vertices, + vertex_attributes, + faces, + face_attributes, + lines, + line_attributes, + textures); +}; + +/// struct for defining a "set_mesh_data" message, which adds or replaces mesh +/// data. +struct SetMeshData { + static std::string MsgId() { return "set_mesh_data"; } + + SetMeshData() : time(0) {} + + /// Path defining the location in the scene tree. + std::string path; + /// The time associated with this data + int32_t time; + /// The layer for this data + std::string layer; + + /// The data to be set + MeshData data; + + MSGPACK_DEFINE_MAP(path, time, layer, data); +}; + +/// struct for defining a "get_mesh_data" message, which requests mesh data. +struct GetMeshData { + static std::string MsgId() { return "get_mesh_data"; } + + GetMeshData() : time(0) {} + + /// Path defining the location in the scene tree. + std::string path; + /// The time for which to return the data + int32_t time; + /// The layer for which to return the data + std::string layer; + + MSGPACK_DEFINE_MAP(path, time, layer); +}; + +/// struct for storing camera data +struct CameraData { + static std::string MsgId() { return "camera_data"; } + + CameraData() : width(0), height(0) {} + + // extrinsic parameters defining the world to camera transform, i.e., + // X_cam = X_world * R + t + + /// rotation R as quaternion [x,y,z,w] + std::array R; + /// translation + std::array t; + + /// intrinsic parameters following colmap's convention, e.g. + /// intrinsic_model = "SIMPLE_RADIAL"; + /// intrinsic_parameters = {f, cx, cy, k}; + std::string intrinsic_model; + std::vector intrinsic_parameters; + + /// image dimensions in pixels + int width; + int height; + + /// map of arrays that can be interpreted as camera images + std::map images; + + MSGPACK_DEFINE_MAP( + R, t, intrinsic_model, intrinsic_parameters, width, height, images); +}; + +/// struct for defining a "set_camera_data" message, which adds or replaces a +/// camera in the scene tree. +struct SetCameraData { + static std::string MsgId() { return "set_camera_data"; } + + SetCameraData() : time(0) {} + + /// Path defining the location in the scene tree. + std::string path; + /// The time for which to return the data + int32_t time; + /// The layer for which to return the data + std::string layer; + + /// The data to be set + CameraData data; + + MSGPACK_DEFINE_MAP(path, time, layer, data); +}; + +/// struct for defining a "set_time" message, which sets the current time +/// to the specified value. +struct SetTime { + static std::string MsgId() { return "set_time"; } + SetTime() : time(0) {} + int32_t time; + + MSGPACK_DEFINE_MAP(time); +}; + +/// struct for defining a "set_active_camera" message, which sets the active +/// camera as the object with the specified path in the scene tree. +struct SetActiveCamera { + static std::string MsgId() { return "set_active_camera"; } + std::string path; + + MSGPACK_DEFINE_MAP(path); +}; + +/// struct for defining a "set_properties" message, which sets properties for +/// the object in the scene tree +struct SetProperties { + static std::string MsgId() { return "set_properties"; } + std::string path; + + // application specific members go here. + + MSGPACK_DEFINE_MAP(path); +}; + +/// struct for defining a "request" message, which describes the subsequent +/// message by storing the msg_id. +struct Request { + std::string msg_id; + MSGPACK_DEFINE_MAP(msg_id); +}; + +/// struct for defining a "reply" message, which describes the subsequent +/// message by storing the msg_id. +struct Reply { + std::string msg_id; + MSGPACK_DEFINE_MAP(msg_id); +}; + +/// struct for defining a "status" message, which will be used for returning +/// error codes or returning code 0 if the call does not return something else. +struct Status { + static std::string MsgId() { return "status"; } + + Status() : code(0) {} + Status(int code, const std::string& str) : code(code), str(str) {} + static Status OK() { return Status(); } + static Status ErrorUnsupportedMsgId() { + return Status(1, "unsupported msg_id"); + } + static Status ErrorUnpackingFailed() { + return Status(2, "error during unpacking"); + } + + /// return code. 0 means everything is OK. + int32_t code; + /// string representation of the code + std::string str; + + MSGPACK_DEFINE_MAP(code, str); +}; + +} // namespace messages +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/ReceiverBase.cpp b/cpp/open3d/io/rpc/ReceiverBase.cpp new file mode 100644 index 00000000000..465eec0bbdd --- /dev/null +++ b/cpp/open3d/io/rpc/ReceiverBase.cpp @@ -0,0 +1,269 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#include "open3d/io/rpc/ReceiverBase.h" + +#include + +#include "open3d/io/rpc/Messages.h" +#include "open3d/io/rpc/ZMQContext.h" + +using namespace open3d::utility; + +namespace { +std::shared_ptr CreateStatusMessage( + const open3d::io::rpc::messages::Status& status) { + msgpack::sbuffer sbuf; + open3d::io::rpc::messages::Reply reply{status.MsgId()}; + msgpack::pack(sbuf, reply); + msgpack::pack(sbuf, status); + std::shared_ptr msg = + std::make_shared(sbuf.data(), sbuf.size()); + + return msg; +} +} // namespace + +namespace open3d { +namespace io { +namespace rpc { + +struct ReceiverBase::MsgpackObject { + explicit MsgpackObject(msgpack::object& obj) : obj_(obj) {} + msgpack::object& obj_; +}; + +ReceiverBase::ReceiverBase(const std::string& address, int timeout) + : address_(address), + timeout_(timeout), + keep_running_(false), + loop_running_(false) {} + +ReceiverBase::~ReceiverBase() { Stop(); } + +void ReceiverBase::Start() { + const std::lock_guard lock(mutex_); + if (!keep_running_) { + keep_running_ = true; + thread_ = std::thread(&ReceiverBase::Mainloop, this); + // wait for the loop to start running + while (!loop_running_.load()) { + std::this_thread::yield(); + }; + + LogDebug("ReceiverBase: started"); + } else { + LogDebug("ReceiverBase: already running"); + } +} + +void ReceiverBase::Stop() { + bool keep_running_old; + { + const std::lock_guard lock(mutex_); + keep_running_old = keep_running_; + if (keep_running_old) { + keep_running_ = false; + } + } + if (keep_running_old) { + thread_.join(); + LogDebug("ReceiverBase: stopped"); + } else { + LogDebug("ReceiverBase: already stopped"); + } +} + +void ReceiverBase::Mainloop() { + socket_ = std::unique_ptr( + new zmq::socket_t(GetZMQContext(), ZMQ_REP)); + + socket_->setsockopt(ZMQ_LINGER, 1000); + socket_->setsockopt(ZMQ_RCVTIMEO, 1000); + socket_->setsockopt(ZMQ_SNDTIMEO, timeout_); + + auto limits = msgpack::unpack_limit(0xffffffff, // array + 0xffffffff, // map + 65536, // str + 0xffffffff, // bin + 0xffffffff, // ext + 100 // depth + ); + try { + socket_->bind(address_.c_str()); + } catch (const zmq::error_t& err) { + LogError("ReceiverBase::Mainloop: Failed to bind address, {}", + err.what()); + } + + loop_running_.store(true); + while (true) { + { + const std::lock_guard lock(mutex_); + if (!keep_running_) break; + } + try { + zmq::message_t message; + if (!socket_->recv(message)) { + continue; + } + + const char* buffer = (char*)message.data(); + size_t buffer_size = message.size(); + + std::vector> replies; + + size_t offset = 0; + while (offset < buffer_size) { + messages::Request req; + try { + auto obj_handle = + msgpack::unpack(buffer, buffer_size, offset, + nullptr, nullptr, limits); + auto obj = obj_handle.get(); + req = obj.as(); + + if (false) { + } +#define PROCESS_MESSAGE(MSGTYPE) \ + else if (MSGTYPE::MsgId() == req.msg_id) { \ + auto oh = msgpack::unpack(buffer, buffer_size, offset, nullptr, \ + nullptr, limits); \ + auto obj = oh.get(); \ + MSGTYPE msg; \ + msg = obj.as(); \ + auto reply = ProcessMessage(req, msg, MsgpackObject(obj)); \ + replies.push_back(reply); \ + } + PROCESS_MESSAGE(messages::SetMeshData) + PROCESS_MESSAGE(messages::GetMeshData) + PROCESS_MESSAGE(messages::SetCameraData) + PROCESS_MESSAGE(messages::SetProperties) + PROCESS_MESSAGE(messages::SetActiveCamera) + PROCESS_MESSAGE(messages::SetTime) + else { + LogInfo("ReceiverBase::Mainloop: unsupported msg " + "id '{}'", + req.msg_id); + auto status = messages::Status::ErrorUnsupportedMsgId(); + replies.push_back(CreateStatusMessage(status)); + break; + } + } catch (std::exception& err) { + LogInfo("ReceiverBase::Mainloop: {}", err.what()); + auto status = messages::Status::ErrorUnpackingFailed(); + status.str += std::string(" with ") + err.what(); + replies.push_back(CreateStatusMessage(status)); + break; + } + } + if (replies.size() == 1) { + socket_->send(*replies[0]); + } else { + size_t size = 0; + for (auto r : replies) { + size += r->size(); + } + zmq::message_t reply(size); + size_t offset = 0; + for (auto r : replies) { + memcpy((char*)reply.data() + offset, r->data(), r->size()); + offset += r->size(); + } + socket_->send(reply); + } + } catch (const zmq::error_t& err) { + LogInfo("ReceiverBase::Mainloop: {}", err.what()); + } + } + socket_->close(); + loop_running_.store(false); +} + +std::shared_ptr ReceiverBase::ProcessMessage( + const messages::Request& req, + const messages::SetMeshData& msg, + const MsgpackObject& obj) { + utility::LogInfo( + "ReceiverBase::ProcessMessage: messages with id {} will be " + "ignored", + msg.MsgId()); + return std::shared_ptr(); +} +std::shared_ptr ReceiverBase::ProcessMessage( + const messages::Request& req, + const messages::GetMeshData& msg, + const MsgpackObject& obj) { + utility::LogInfo( + "ReceiverBase::ProcessMessage: messages with id {} will be " + "ignored", + msg.MsgId()); + return std::shared_ptr(); +} +std::shared_ptr ReceiverBase::ProcessMessage( + const messages::Request& req, + const messages::SetCameraData& msg, + const MsgpackObject& obj) { + utility::LogInfo( + "ReceiverBase::ProcessMessage: messages with id {} will be " + "ignored", + msg.MsgId()); + return std::shared_ptr(); +} +std::shared_ptr ReceiverBase::ProcessMessage( + const messages::Request& req, + const messages::SetProperties& msg, + const MsgpackObject& obj) { + utility::LogInfo( + "ReceiverBase::ProcessMessage: messages with id {} will be " + "ignored", + msg.MsgId()); + return std::shared_ptr(); +} +std::shared_ptr ReceiverBase::ProcessMessage( + const messages::Request& req, + const messages::SetActiveCamera& msg, + const MsgpackObject& obj) { + utility::LogInfo( + "ReceiverBase::ProcessMessage: messages with id {} will be " + "ignored", + msg.MsgId()); + return std::shared_ptr(); +} +std::shared_ptr ReceiverBase::ProcessMessage( + const messages::Request& req, + const messages::SetTime& msg, + const MsgpackObject& obj) { + utility::LogInfo( + "ReceiverBase::ProcessMessage: messages with id {} will be " + "ignored", + msg.MsgId()); + return std::shared_ptr(); +} + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/ReceiverBase.h b/cpp/open3d/io/rpc/ReceiverBase.h new file mode 100644 index 00000000000..27f36a0059e --- /dev/null +++ b/cpp/open3d/io/rpc/ReceiverBase.h @@ -0,0 +1,129 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#pragma once + +#include +#include +#include + +#include "open3d/utility/Console.h" + +namespace zmq { +class message_t; +class socket_t; +} // namespace zmq + +namespace open3d { +namespace io { +namespace rpc { + +namespace messages { +struct Request; +struct SetMeshData; +struct GetMeshData; +struct SetCameraData; +struct SetProperties; +struct SetActiveCamera; +struct SetTime; +} // namespace messages + +/// Base class for the server side receiving requests from a client. +/// Subclass from this and implement the overloaded ProcessMessage functions as +/// needed. +class ReceiverBase { +public: + /// Constructs a receiver listening on the specified address. + /// \param address Address to listen on. + /// \param timeout Timeout in milliseconds for sending the repy. + ReceiverBase(const std::string& address = "tcp://127.0.0.1:51454", + int timeout = 10000); + + ReceiverBase(const ReceiverBase&) = delete; + ReceiverBase& operator=(const ReceiverBase&) = delete; + + ~ReceiverBase(); + + /// Starts the receiver mainloop in a new thread. + void Start(); + + /// Stops the receiver mainloop and joins the thread. + /// This function blocks until the mainloop is done with processing + /// messages that have already been received. + void Stop(); + +protected: + // Opaque type for providing the original msgpack::object to the + // ProcessMessage functions + struct MsgpackObject; + + /// Function for processing a msg. + /// \param req The Request object that accompanies the \param msg object. + /// + /// \param msg The message to be processed + /// + /// \param obj The object from which the \param msg was unpacked. Can be + /// used for custom unpacking. + virtual std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::SetMeshData& msg, + const MsgpackObject& obj); + virtual std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::GetMeshData& msg, + const MsgpackObject& obj); + virtual std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::SetCameraData& msg, + const MsgpackObject& obj); + virtual std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::SetProperties& msg, + const MsgpackObject& obj); + virtual std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::SetActiveCamera& msg, + const MsgpackObject& obj); + virtual std::shared_ptr ProcessMessage( + const messages::Request& req, + const messages::SetTime& msg, + const MsgpackObject& obj); + +private: + void Mainloop(); + + const std::string address_; + const int timeout_; + std::unique_ptr socket_; + std::thread thread_; + std::mutex mutex_; + bool keep_running_; + std::atomic loop_running_; +}; + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/RemoteFunctions.cpp b/cpp/open3d/io/rpc/RemoteFunctions.cpp new file mode 100644 index 00000000000..61b5a08e83c --- /dev/null +++ b/cpp/open3d/io/rpc/RemoteFunctions.cpp @@ -0,0 +1,414 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#include "open3d/io/rpc/RemoteFunctions.h" + +#include +#include + +#include "open3d/core/Dispatch.h" +#include "open3d/io/rpc/Connection.h" +#include "open3d/io/rpc/MessageUtils.h" +#include "open3d/io/rpc/Messages.h" +#include "open3d/utility/Console.h" + +using namespace open3d::utility; + +namespace open3d { +namespace io { +namespace rpc { + +bool SetPointCloud(const geometry::PointCloud& pcd, + const std::string& path, + int time, + const std::string& layer, + std::shared_ptr connection) { + // TODO use SetMeshData here after switching to the new PointCloud class. + if (!pcd.HasPoints()) { + LogInfo("SetMeshData: point cloud is empty"); + return false; + } + + messages::SetMeshData msg; + msg.path = path; + msg.time = time; + msg.layer = layer; + + msg.data.vertices = messages::Array::FromPtr( + (double*)pcd.points_.data(), {int64_t(pcd.points_.size()), 3}); + if (pcd.HasNormals()) { + msg.data.vertex_attributes["normals"] = + messages::Array::FromPtr((double*)pcd.normals_.data(), + {int64_t(pcd.normals_.size()), 3}); + } + if (pcd.HasColors()) { + msg.data.vertex_attributes["colors"] = messages::Array::FromPtr( + (double*)pcd.colors_.data(), {int64_t(pcd.colors_.size()), 3}); + } + + msgpack::sbuffer sbuf; + messages::Request request{msg.MsgId()}; + msgpack::pack(sbuf, request); + msgpack::pack(sbuf, msg); + + zmq::message_t send_msg(sbuf.data(), sbuf.size()); + if (!connection) { + connection = std::shared_ptr(new Connection()); + } + auto reply = connection->Send(send_msg); + return ReplyIsOKStatus(*reply); +} + +bool SetTriangleMesh(const geometry::TriangleMesh& mesh, + const std::string& path, + int time, + const std::string& layer, + std::shared_ptr connection) { + // TODO use SetMeshData here after switching to the new TriangleMesh class. + if (!mesh.HasTriangles()) { + LogInfo("SetMeshData: triangle mesh is empty"); + return false; + } + + messages::SetMeshData msg; + msg.path = path; + msg.time = time; + msg.layer = layer; + + msg.data.vertices = + messages::Array::FromPtr((double*)mesh.vertices_.data(), + {int64_t(mesh.vertices_.size()), 3}); + msg.data.faces = messages::Array::FromPtr( + (int*)mesh.triangles_.data(), {int64_t(mesh.triangles_.size()), 3}); + if (mesh.HasVertexNormals()) { + msg.data.vertex_attributes["normals"] = messages::Array::FromPtr( + (double*)mesh.vertex_normals_.data(), + {int64_t(mesh.vertex_normals_.size()), 3}); + } + if (mesh.HasVertexColors()) { + msg.data.vertex_attributes["colors"] = messages::Array::FromPtr( + (double*)mesh.vertex_colors_.data(), + {int64_t(mesh.vertex_colors_.size()), 3}); + } + if (mesh.HasTriangleNormals()) { + msg.data.face_attributes["normals"] = messages::Array::FromPtr( + (double*)mesh.triangle_normals_.data(), + {int64_t(mesh.triangle_normals_.size()), 3}); + } + if (mesh.HasTriangleUvs()) { + msg.data.face_attributes["uvs"] = messages::Array::FromPtr( + (double*)mesh.triangle_uvs_.data(), + {int64_t(mesh.triangle_uvs_.size()), 2}); + } + if (mesh.HasTextures()) { + int tex_id = 0; + for (const auto& image : mesh.textures_) { + if (!image.IsEmpty()) { + std::vector shape( + {image.height_, image.width_, image.num_of_channels_}); + if (image.bytes_per_channel_ == sizeof(uint8_t)) { + msg.data.textures[std::to_string(tex_id)] = + messages::Array::FromPtr( + (uint8_t*)image.data_.data(), shape); + } else if (image.bytes_per_channel_ == sizeof(float)) { + msg.data.textures[std::to_string(tex_id)] = + messages::Array::FromPtr((float*)image.data_.data(), + shape); + } else if (image.bytes_per_channel_ == sizeof(double)) { + msg.data.textures[std::to_string(tex_id)] = + messages::Array::FromPtr( + (double*)image.data_.data(), shape); + } + } + ++tex_id; + } + } + + msgpack::sbuffer sbuf; + messages::Request request{msg.MsgId()}; + msgpack::pack(sbuf, request); + msgpack::pack(sbuf, msg); + + zmq::message_t send_msg(sbuf.data(), sbuf.size()); + if (!connection) { + connection = std::shared_ptr(new Connection()); + } + auto reply = connection->Send(send_msg); + return ReplyIsOKStatus(*reply); +} + +bool SetMeshData(const core::Tensor& vertices, + const std::string& path, + int time, + const std::string& layer, + const std::map& vertex_attributes, + const core::Tensor& faces, + const std::map& face_attributes, + const core::Tensor& lines, + const std::map& line_attributes, + const std::map& textures, + std::shared_ptr connection) { + if (vertices.NumElements() == 0) { + LogInfo("SetMeshData: vertices Tensor is empty"); + return false; + } + if (vertices.NumDims() != 2) { + LogInfo("SetMeshData: vertices ndim must be 2 but is {}", + vertices.NumDims()); + return false; + } + if (vertices.GetDtype() != core::Dtype::Float32 && + vertices.GetDtype() != core::Dtype::Float64) { + LogError( + "SetMeshData: vertices must have dtype Float32 or Float64 but " + "is {}", + core::DtypeUtil::ToString(vertices.GetDtype())); + } + + auto PrepareTensor = [](const core::Tensor& a) { + if (a.GetDevice().GetType() != core::Device::DeviceType::CPU) { + return a.Copy(core::Device("CPU:0")); + } + return a.Contiguous(); + }; + + auto CreateArray = [](const core::Tensor& a) { + return DISPATCH_DTYPE_TO_TEMPLATE(a.GetDtype(), [&]() { + return messages::Array::FromPtr( + (scalar_t*)a.GetDataPtr(), + static_cast>(a.GetShape())); + }); + }; + + messages::SetMeshData msg; + msg.path = path; + msg.time = time; + msg.layer = layer; + + const core::Tensor vertices_ok = PrepareTensor(vertices); + msg.data.vertices = CreateArray(vertices_ok); + + // store tensors in this vector to make sure the memory blob is alive + // for tensors where a deep copy was necessary. + std::vector tensor_cache; + for (const auto& item : vertex_attributes) { + tensor_cache.push_back(PrepareTensor(item.second)); + const core::Tensor& tensor = tensor_cache.back(); + if (tensor.NumDims() >= 1 && + tensor.GetShape()[0] == vertices.GetShape()[0]) { + msg.data.vertex_attributes[item.first] = CreateArray(tensor); + } else { + LogError("SetMeshData: Attribute {} has incompatible shape {}", + item.first, tensor.GetShape().ToString()); + } + } + + if (faces.NumElements()) { + if (faces.GetDtype() != core::Dtype::Int32 && + faces.GetDtype() != core::Dtype::Int64) { + LogError( + "SetMeshData: faces must have dtype Int32 or Int64 but " + "is {}", + core::DtypeUtil::ToString(vertices.GetDtype())); + } else if (faces.NumDims() != 2) { + LogError("SetMeshData: faces must have rank 2 but is {}", + faces.NumDims()); + } else if (faces.GetShape()[1] >= 3) { + LogError("SetMeshData: last dim of faces must be >3 but is {}", + faces.GetShape()[1]); + } else { + tensor_cache.push_back(PrepareTensor(faces)); + const core::Tensor& faces_ok = tensor_cache.back(); + msg.data.faces = CreateArray(faces_ok); + + for (const auto& item : face_attributes) { + tensor_cache.push_back(PrepareTensor(item.second)); + const core::Tensor& tensor = tensor_cache.back(); + if (tensor.NumDims() >= 1 && + tensor.GetShape()[0] == faces.GetShape()[0]) { + msg.data.face_attributes[item.first] = CreateArray(tensor); + } else { + LogError( + "SetMeshData: Attribute {} has incompatible shape " + "{}", + item.first, tensor.GetShape().ToString()); + } + } + } + } + + if (lines.NumElements()) { + if (lines.GetDtype() != core::Dtype::Int32 && + lines.GetDtype() != core::Dtype::Int64) { + LogError( + "SetMeshData: lines must have dtype Int32 or Int64 but " + "is {}", + core::DtypeUtil::ToString(vertices.GetDtype())); + } else if (lines.NumDims() != 2) { + LogError("SetMeshData: lines must have rank 2 but is {}", + lines.NumDims()); + } else if (lines.GetShape()[1] >= 2) { + LogError("SetMeshData: last dim of lines must be >2 but is {}", + lines.GetShape()[1]); + } else { + tensor_cache.push_back(PrepareTensor(lines)); + const core::Tensor& lines_ok = tensor_cache.back(); + msg.data.lines = CreateArray(lines_ok); + + for (const auto& item : line_attributes) { + tensor_cache.push_back(PrepareTensor(item.second)); + const core::Tensor& tensor = tensor_cache.back(); + if (tensor.NumDims() >= 1 && + tensor.GetShape()[0] == lines.GetShape()[0]) { + msg.data.line_attributes[item.first] = CreateArray(tensor); + } else { + LogError( + "SetMeshData: Attribute {} has incompatible shape " + "{}", + item.first, tensor.GetShape().ToString()); + } + } + } + } + + for (const auto& item : textures) { + tensor_cache.push_back(PrepareTensor(item.second)); + const core::Tensor& tensor = tensor_cache.back(); + if (tensor.NumElements()) { + msg.data.textures[item.first] = CreateArray(tensor); + } else { + LogError("SetMeshData: Texture {} is empty", item.first); + } + } + + msgpack::sbuffer sbuf; + messages::Request request{msg.MsgId()}; + msgpack::pack(sbuf, request); + msgpack::pack(sbuf, msg); + + zmq::message_t send_msg(sbuf.data(), sbuf.size()); + if (!connection) { + connection = std::shared_ptr(new Connection()); + } + auto reply = connection->Send(send_msg); + return ReplyIsOKStatus(*reply); +} + +bool SetLegacyCamera(const camera::PinholeCameraParameters& camera, + const std::string& path, + int time, + const std::string& layer, + std::shared_ptr connection) { + messages::SetCameraData msg; + msg.path = path; + msg.time = time; + msg.layer = layer; + + // convert extrinsics + Eigen::Matrix3d R = camera.extrinsic_.block<3, 3>(0, 0); + Eigen::Vector3d t = camera.extrinsic_.block<3, 1>(0, 3); + Eigen::Quaterniond q(R); + msg.data.R[0] = q.x(); + msg.data.R[1] = q.y(); + msg.data.R[2] = q.z(); + msg.data.R[3] = q.w(); + + msg.data.t[0] = t[0]; + msg.data.t[1] = t[1]; + msg.data.t[2] = t[2]; + + // convert intrinsics + if (camera.intrinsic_.IsValid()) { + msg.data.width = camera.intrinsic_.width_; + msg.data.height = camera.intrinsic_.height_; + msg.data.intrinsic_model = "PINHOLE"; + msg.data.intrinsic_parameters.resize(4); + msg.data.intrinsic_parameters[0] = + camera.intrinsic_.intrinsic_matrix_(0, 0); + msg.data.intrinsic_parameters[1] = + camera.intrinsic_.intrinsic_matrix_(1, 1); + msg.data.intrinsic_parameters[2] = + camera.intrinsic_.intrinsic_matrix_(0, 2); + msg.data.intrinsic_parameters[3] = + camera.intrinsic_.intrinsic_matrix_(1, 2); + if (camera.intrinsic_.GetSkew() != 0.0) { + LogWarning( + "SetLegacyCamera: Nonzero skew parameteer in " + "PinholeCameraParameters will be ignored!"); + } + } + + msgpack::sbuffer sbuf; + messages::Request request{msg.MsgId()}; + msgpack::pack(sbuf, request); + msgpack::pack(sbuf, msg); + + zmq::message_t send_msg(sbuf.data(), sbuf.size()); + if (!connection) { + connection = std::shared_ptr(new Connection()); + } + auto reply = connection->Send(send_msg); + return ReplyIsOKStatus(*reply); +} + +bool SetTime(int time, std::shared_ptr connection) { + messages::SetTime msg; + msg.time = time; + + msgpack::sbuffer sbuf; + messages::Request request{msg.MsgId()}; + msgpack::pack(sbuf, request); + msgpack::pack(sbuf, msg); + + zmq::message_t send_msg(sbuf.data(), sbuf.size()); + if (!connection) { + connection = std::shared_ptr(new Connection()); + } + auto reply = connection->Send(send_msg); + return ReplyIsOKStatus(*reply); +} + +bool SetActiveCamera(const std::string& path, + std::shared_ptr connection) { + messages::SetActiveCamera msg; + msg.path = path; + + msgpack::sbuffer sbuf; + messages::Request request{msg.MsgId()}; + msgpack::pack(sbuf, request); + msgpack::pack(sbuf, msg); + + zmq::message_t send_msg(sbuf.data(), sbuf.size()); + if (!connection) { + connection = std::shared_ptr(new Connection()); + } + auto reply = connection->Send(send_msg); + return ReplyIsOKStatus(*reply); +} + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/RemoteFunctions.h b/cpp/open3d/io/rpc/RemoteFunctions.h new file mode 100644 index 00000000000..5b89085bd8c --- /dev/null +++ b/cpp/open3d/io/rpc/RemoteFunctions.h @@ -0,0 +1,187 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#pragma once + +#include + +#include "open3d/camera/PinholeCameraParameters.h" +#include "open3d/core/Tensor.h" +#include "open3d/geometry/PointCloud.h" +#include "open3d/geometry/TriangleMesh.h" +#include "open3d/io/rpc/ConnectionBase.h" + +namespace zmq { +class message_t; +} + +namespace open3d { +namespace io { +namespace rpc { + +namespace messages { +struct Status; +} + +/// Function for sending a PointCloud. +/// \param pcd The PointCloud object. +/// +/// \param path Path descriptor defining a location in the scene tree. +/// E.g., 'mygroup/mypoints'. +/// +/// \param time The time point associated with the object. +/// +/// \param layer The layer for this object. +/// +/// \param connection The connection object used for sending the data. +/// If nullptr a default connection object will be used. +/// +bool SetPointCloud(const geometry::PointCloud& pcd, + const std::string& path = "", + int time = 0, + const std::string& layer = "", + std::shared_ptr connection = + std::shared_ptr()); + +/// Function for sending a TriangleMesh. +/// \param pcd The TriangleMesh object. +/// +/// \param path Path descriptor defining a location in the scene tree. +/// E.g., 'mygroup/mypoints'. +/// +/// \param time The time point associated with the object. +/// +/// \param layer The layer for this object. +/// +/// \param connection The connection object used for sending the data. +/// If nullptr a default connection object will be used. +/// +bool SetTriangleMesh(const geometry::TriangleMesh& mesh, + const std::string& path = "", + int time = 0, + const std::string& layer = "", + std::shared_ptr connection = + std::shared_ptr()); + +/// Function for sending general mesh data. +/// \param vertices Tensor with vertices of shape [N,3] +/// +/// \param path Path descriptor defining a location in the scene +/// tree. E.g., 'mygroup/mypoints'. +/// +/// \param time The time point associated with the object. +/// +/// \param layer The layer for this object. +/// +/// \param vertex_attributes Map with Tensors storing vertex attributes. The +/// first dim of each attribute must match the number of vertices. +/// +/// \param faces Tensor with vertex indices defining the faces. The +/// Tensor is of rank 1 or 2. A rank 2 Tensor with shape [num_faces,n] defines +/// num_faces n-gons. If the rank is 1 then polys of different lengths are +/// stored sequentially. Each polygon is stored as a sequence 'n i1 i2 ... in' +/// with n>=3. The type of the array must be int32_t or int64_t +/// +/// \param face_attributes Map with Tensors storing face attributes. The +/// first dim of each attribute must match the number of faces. +/// +/// \param lines Tensor with vertex indices defining the lines. The +/// Tensor is of rank 1 or 2. A rank 2 Tensor with shape [num_lines,n] defines +/// num_lines linestrips. If the rank is 1 then linestrips of different lengths +/// are stored sequentially. Each linestrips is stored as a sequence 'n i1 i2 +/// ... in' with n>=2. The type of the array must be int32_t or int64_t +/// +/// \param line_attributes Map with Tensors storing line attributes. The +/// first dim of each attribute must match the number of lines. +/// +/// \param textures Map of Tensors for storing textures. +/// +/// \param connection The connection object used for sending the data. +/// If nullptr a default connection object will be used. +/// +bool SetMeshData(const core::Tensor& vertices, + const std::string& path = "", + int time = 0, + const std::string& layer = "", + const std::map& vertex_attributes = + std::map(), + const core::Tensor& faces = core::Tensor({0}, + core::Dtype::Int32), + const std::map& face_attributes = + std::map(), + const core::Tensor& lines = core::Tensor({0}, + core::Dtype::Int32), + const std::map& line_attributes = + std::map(), + const std::map& textures = + std::map(), + std::shared_ptr connection = + std::shared_ptr()); + +/// Function for sending Camera data. +/// \param camera The PinholeCameraParameters object. +/// +/// \param path Path descriptor defining a location in the scene tree. +/// E.g., 'mygroup/mycam'. +/// +/// \param time The time point associated with the object. +/// +/// \param layer The layer for this object. +/// +/// \param connection The connection object used for sending the data. +/// If nullptr a default connection object will be used. +/// +bool SetLegacyCamera(const camera::PinholeCameraParameters& camera, + const std::string& path = "", + int time = 0, + const std::string& layer = "", + std::shared_ptr connection = + std::shared_ptr()); + +/// Sets the time in the external visualizer. +/// \param time The time value +/// +/// \param connection The connection object used for sending the data. +/// If nullptr a default connection object will be used. +/// +bool SetTime(int time, + std::shared_ptr connection = + std::shared_ptr()); + +/// Sets the object with the specified path as the active camera. +/// \param path Path descriptor defining a location in the scene tree. +/// E.g., 'mygroup/mycam'. +/// +/// \param connection The connection object used for sending the data. +/// If nullptr a default connection object will be used. +/// +bool SetActiveCamera(const std::string& path, + std::shared_ptr connection = + std::shared_ptr()); + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/ZMQContext.cpp b/cpp/open3d/io/rpc/ZMQContext.cpp new file mode 100644 index 00000000000..1a1fed8b19f --- /dev/null +++ b/cpp/open3d/io/rpc/ZMQContext.cpp @@ -0,0 +1,42 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#include "open3d/io/rpc/ZMQContext.h" + +#include + +namespace open3d { +namespace io { +namespace rpc { + +zmq::context_t& GetZMQContext() { + static zmq::context_t context; + return context; +} + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/io/rpc/ZMQContext.h b/cpp/open3d/io/rpc/ZMQContext.h new file mode 100644 index 00000000000..0819af25a65 --- /dev/null +++ b/cpp/open3d/io/rpc/ZMQContext.h @@ -0,0 +1,42 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#pragma once + +namespace zmq { +class context_t; +} + +namespace open3d { +namespace io { +namespace rpc { + +/// Returns the zeromq context for this process. +zmq::context_t& GetZMQContext(); + +} // namespace rpc +} // namespace io +} // namespace open3d diff --git a/cpp/open3d/utility/CMakeLists.txt b/cpp/open3d/utility/CMakeLists.txt index 679c56a332c..86c1e7e0128 100644 --- a/cpp/open3d/utility/CMakeLists.txt +++ b/cpp/open3d/utility/CMakeLists.txt @@ -1,5 +1,5 @@ # build -file(GLOB_RECURSE ALL_SOURCE_FILES "*.cpp") +file(GLOB ALL_SOURCE_FILES "*.cpp") # create object library add_library(utility OBJECT ${ALL_SOURCE_FILES}) diff --git a/cpp/pybind/CMakeLists.txt b/cpp/pybind/CMakeLists.txt index 87ee16fdb46..7c677d50599 100644 --- a/cpp/pybind/CMakeLists.txt +++ b/cpp/pybind/CMakeLists.txt @@ -46,6 +46,12 @@ if (NOT BUILD_AZURE_KINECT) "${CMAKE_CURRENT_SOURCE_DIR}/io/sensor.cpp") endif() +if (NOT BUILD_RPC_INTERFACE) + list(REMOVE_ITEM PY_ALL_SOURCE_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/io/rpc.cpp" + ) +endif() + # NO_EXTRAS disables LTO which causes problems during link with nvcc 9.x if ( CMAKE_CUDA_COMPILER_VERSION VERSION_LESS "10.0.0" ) pybind11_add_module(${PACKAGE_NAME} NO_EXTRAS diff --git a/cpp/pybind/_build_config.py.in b/cpp/pybind/_build_config.py.in index 68f87f1313f..5e9a7613f0f 100644 --- a/cpp/pybind/_build_config.py.in +++ b/cpp/pybind/_build_config.py.in @@ -6,6 +6,7 @@ _build_config = { "BUILD_LIBREALSENSE" : "@BUILD_LIBREALSENSE@" == "ON", "BUILD_SHARED_LIBS" : "@BUILD_SHARED_LIBS@" == "ON", "BUILD_GUI" : "@BUILD_GUI@" == "ON", + "BUILD_RPC_INTERFACE": "@BUILD_RPC_INTERFACE@" == "ON", "ENABLE_HEADLESS_RENDERING" : "@ENABLE_HEADLESS_RENDERING@" == "ON", "BUILD_JUPYTER_EXTENSION" : "@BUILD_JUPYTER_EXTENSION@" == "ON", "GLIBCXX_USE_CXX11_ABI" : "@GLIBCXX_USE_CXX11_ABI@" == "ON", diff --git a/cpp/pybind/io/io.cpp b/cpp/pybind/io/io.cpp index aa2f792eddf..74463625bf9 100644 --- a/cpp/pybind/io/io.cpp +++ b/cpp/pybind/io/io.cpp @@ -36,6 +36,9 @@ void pybind_io(py::module &m) { #ifdef BUILD_AZURE_KINECT pybind_sensor(m_io); #endif +#ifdef BUILD_RPC_INTERFACE + pybind_rpc(m_io); +#endif } } // namespace open3d diff --git a/cpp/pybind/io/io.h b/cpp/pybind/io/io.h index 6580d03bad0..dc30d8328f8 100644 --- a/cpp/pybind/io/io.h +++ b/cpp/pybind/io/io.h @@ -38,4 +38,7 @@ void pybind_class_io(py::module& m); void pybind_sensor(py::module& m); #endif +#ifdef BUILD_RPC_INTERFACE +void pybind_rpc(py::module& m); +#endif } // namespace open3d diff --git a/cpp/pybind/io/rpc.cpp b/cpp/pybind/io/rpc.cpp new file mode 100644 index 00000000000..efc8a54a581 --- /dev/null +++ b/cpp/pybind/io/rpc.cpp @@ -0,0 +1,173 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#include "open3d/io/rpc/Connection.h" +#include "open3d/io/rpc/DummyReceiver.h" +#include "open3d/io/rpc/RemoteFunctions.h" +#include "pybind/docstring.h" +#include "pybind/open3d_pybind.h" + +namespace open3d { + +void pybind_rpc(py::module& m_io) { + py::module m = m_io.def_submodule("rpc"); + py::class_>(m, "_ConnectionBase"); + + py::class_, + io::rpc::ConnectionBase>(m, "Connection") + .def(py::init([](std::string address, int connect_timeout, + int timeout) { + return std::shared_ptr( + new io::rpc::Connection(address, connect_timeout, + timeout)); + }), + "Creates a connection object", + "address"_a = "tcp://127.0.0.1:51454", + "connect_timeout"_a = 5000, "timeout"_a = 10000); + + py::class_>( + m, "_DummyReceiver", + "Dummy receiver for the server side receiving requests from a " + "client.") + .def(py::init([](std::string address, int timeout) { + return std::shared_ptr( + new io::rpc::DummyReceiver(address, timeout)); + }), + "Creates the receiver object which can be used for testing " + "connections.", + "address"_a = "tcp://127.0.0.1:51454", "timeout"_a = 10000) + .def("start", &io::rpc::DummyReceiver::Start, + "Starts the receiver mainloop in a new thread.") + .def("stop", &io::rpc::DummyReceiver::Stop, + "Stops the receiver mainloop and joins the thread. This " + "function blocks until the mainloop is done with processing " + "messages that have already been received."); + + m.def("set_point_cloud", &io::rpc::SetPointCloud, "pcd"_a, "path"_a = "", + "time"_a = 0, "layer"_a = "", + "connection"_a = std::shared_ptr(), + "Sends a point cloud message to a viewer."); + docstring::FunctionDocInject( + m, "set_point_cloud", + { + {"pcd", "Point cloud object."}, + {"path", "A path descriptor, e.g., 'mygroup/points'."}, + {"time", "The time associated with this data."}, + {"layer", "The layer associated with this data."}, + {"connection", + "A Connection object. Use None to automatically create " + "the connection."}, + }); + + m.def("set_triangle_mesh", &io::rpc::SetTriangleMesh, "mesh"_a, + "path"_a = "", "time"_a = 0, "layer"_a = "", + "connection"_a = std::shared_ptr(), + "Sends a point cloud message to a viewer."); + docstring::FunctionDocInject( + m, "set_triangle_mesh", + { + {"mesh", "The TriangleMesh object."}, + {"path", "A path descriptor, e.g., 'mygroup/mesh'."}, + {"time", "The time associated with this data."}, + {"layer", "The layer associated with this data."}, + {"connection", + "A Connection object. Use None to automatically create " + "the connection."}, + }); + + m.def("set_mesh_data", &io::rpc::SetMeshData, "vertices"_a, "path"_a = "", + "time"_a = 0, "layer"_a = "", + "vertex_attributes"_a = std::map(), + "faces"_a = core::Tensor({0}, core::Dtype::Int32), + "face_attributes"_a = std::map(), + "lines"_a = core::Tensor({0}, core::Dtype::Int32), + "line_attributes"_a = std::map(), + "textures"_a = std::map(), + "connection"_a = std::shared_ptr(), + "Sends a set_mesh_data message."); + docstring::FunctionDocInject( + m, "set_mesh_data", + { + {"vertices", "Tensor defining the vertices."}, + {"path", "A path descriptor, e.g., 'mygroup/points'."}, + {"time", "The time associated with this data."}, + {"layer", "The layer associated with this data."}, + {"vertex_attributes", + "dict of Tensors with vertex attributes."}, + {"faces", "Tensor defining the faces with vertex indices."}, + {"face_attributes", + "dict of Tensors with face attributes."}, + {"lines", "Tensor defining lines with vertex indices."}, + {"line_attributes", + "dict of Tensors with line attributes."}, + {"textures", "dict of Tensors with textures."}, + {"connection", + "A Connection object. Use None to automatically create " + "the connection."}, + }); + + m.def("set_legacy_camera", &io::rpc::SetLegacyCamera, "camera"_a, + "path"_a = "", "time"_a = 0, "layer"_a = "", + "connection"_a = std::shared_ptr(), + "Sends a PinholeCameraParameters object."); + docstring::FunctionDocInject( + m, "set_legacy_camera", + { + {"path", "A path descriptor, e.g., 'mygroup/camera'."}, + {"time", "The time associated with this data."}, + {"layer", "The layer associated with this data."}, + {"connection", + "A Connection object. Use None to automatically create " + "the connection."}, + }); + + m.def("set_time", &io::rpc::SetTime, "time"_a, + "connection"_a = std::shared_ptr(), + "Sets the time in the external visualizer."); + docstring::FunctionDocInject( + m, "set_time", + { + {"time", "The time value to set."}, + {"connection", + "A Connection object. Use None to automatically create " + "the connection."}, + }); + + m.def("set_active_camera", &io::rpc::SetActiveCamera, "path"_a, + "connection"_a = std::shared_ptr(), + "Sets the object with the specified path as the active camera."); + docstring::FunctionDocInject( + m, "set_active_camera", + { + {"path", "A path descriptor, e.g., 'mygroup/camera'."}, + {"connection", + "A Connection object. Use None to automatically create " + "the connection."}, + }); +} + +} // namespace open3d diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index cbb900a20b3..d051cb3407a 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -25,6 +25,10 @@ if (NOT BUILD_GUI) list(FILTER UNIT_TEST_SOURCE_FILES EXCLUDE REGEX .*/visualization/rendering/.*cpp) endif() +if (NOT BUILD_RPC_INTERFACE) + list(FILTER UNIT_TEST_SOURCE_FILES EXCLUDE REGEX .*/io/rpc/RemoteFunctions.cpp) +endif() + add_executable(tests ${UNIT_TEST_SOURCE_FILES}) add_definitions(-DTEST_DATA_DIR="${PROJECT_SOURCE_DIR}/examples/test_data") @@ -47,3 +51,4 @@ if (BUILD_CUDA_MODULE) target_link_libraries(tests PRIVATE ${CUDA_LIBRARIES}) target_include_directories(tests SYSTEM PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) endif() + diff --git a/cpp/tests/io/rpc/RemoteFunctions.cpp b/cpp/tests/io/rpc/RemoteFunctions.cpp new file mode 100644 index 00000000000..8ed2ba2d85b --- /dev/null +++ b/cpp/tests/io/rpc/RemoteFunctions.cpp @@ -0,0 +1,241 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2020 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#include "open3d/io/rpc/RemoteFunctions.h" + +#include + +#include "open3d/geometry/PointCloud.h" +#include "open3d/geometry/TriangleMesh.h" +#include "open3d/io/rpc/BufferConnection.h" +#include "open3d/io/rpc/Connection.h" +#include "open3d/io/rpc/DummyReceiver.h" +#include "open3d/io/rpc/MessageUtils.h" +#include "tests/UnitTest.h" + +using namespace open3d::io::rpc; + +namespace open3d { +namespace tests { + +#ifdef _WIN32 +const std::string connection_address = "tcp://127.0.0.1:51454"; +#else +const std::string connection_address = "ipc:///tmp/open3d_ipc"; +#endif + +TEST(RemoteFunctions, SendReceiveUnpackMessages) { + { + // start receiver + DummyReceiver receiver(connection_address, 500); + receiver.Start(); + + geometry::PointCloud pcd; + pcd.points_.push_back(Eigen::Vector3d(1, 2, 3)); + auto connection = + std::make_shared(connection_address, 500, 500); + ASSERT_TRUE(SetPointCloud(pcd, "", 0, "", connection)); + receiver.Stop(); + } + { + // start receiver + DummyReceiver receiver(connection_address, 500); + receiver.Start(); + + geometry::TriangleMesh mesh; + mesh.vertices_.push_back(Eigen::Vector3d(1, 2, 3)); + mesh.vertices_.push_back(Eigen::Vector3d(1, 2, 3)); + mesh.vertices_.push_back(Eigen::Vector3d(1, 2, 3)); + mesh.triangles_.push_back(Eigen::Vector3i(0, 1, 2)); + auto connection = + std::make_shared(connection_address, 500, 500); + ASSERT_TRUE(SetTriangleMesh(mesh, "", 0, "", connection)); + receiver.Stop(); + } + { + // start receiver + DummyReceiver receiver(connection_address, 500); + receiver.Start(); + + camera::PinholeCameraParameters cam; + auto connection = + std::make_shared(connection_address, 500, 500); + ASSERT_TRUE(SetLegacyCamera(cam, "", 0, "", connection)); + receiver.Stop(); + } + { + // start receiver + DummyReceiver receiver(connection_address, 500); + receiver.Start(); + + auto connection = + std::make_shared(connection_address, 500, 500); + ASSERT_TRUE(SetTime(0, connection)); + receiver.Stop(); + } + { + // start receiver + DummyReceiver receiver(connection_address, 500); + receiver.Start(); + + auto connection = + std::make_shared(connection_address, 500, 500); + ASSERT_TRUE(SetActiveCamera("group/mycam", connection)); + receiver.Stop(); + } + + // chain multiple messages to test if the receiver can handle this + { + // start receiver + DummyReceiver receiver(connection_address, 500); + receiver.Start(); + + geometry::PointCloud pcd; + pcd.points_.push_back(Eigen::Vector3d(1, 2, 3)); + auto buf_connection = std::make_shared(); + ASSERT_TRUE(SetPointCloud(pcd, "", 0, "", buf_connection)); + + camera::PinholeCameraParameters cam; + ASSERT_TRUE(SetLegacyCamera(cam, "", 0, "", buf_connection)); + + ASSERT_TRUE(SetTime(0, buf_connection)); + + auto connection = + std::make_shared(connection_address, 500, 500); + std::string buf = buf_connection->buffer().str(); + auto reply = connection->Send(buf.data(), buf.size()); + + // check reply and stop listening + const void* reply_data; + size_t reply_size; + std::tie(reply_data, reply_size) = GetZMQMessageDataAndSize(*reply); + size_t offset = 0; + int count = 0; + while (offset < reply_size) { + ASSERT_TRUE(ReplyIsOKStatus(*reply, offset)); + ++count; + } + ASSERT_EQ(offset, reply_size); + ASSERT_EQ(count, 3); + + // Since we reached the end this must now return false. + ASSERT_FALSE(ReplyIsOKStatus(*reply, offset)); + receiver.Stop(); + } +} + +TEST(RemoteFunctions, SendGarbage) { + std::mt19937 rng; + rng.seed(123); + + // start receiver + DummyReceiver receiver(connection_address, 500); + receiver.Start(); + + // send invalid msg id + { + std::string data = CreateSerializedRequestMessage("bla123"); + + // send to receiver + Connection connection(connection_address, 500, 500); + auto reply = connection.Send(data.data(), data.size()); + const void* reply_data; + size_t reply_size; + std::tie(reply_data, reply_size) = GetZMQMessageDataAndSize(*reply); + + size_t offset = 0; + bool ok; + auto status = UnpackStatusFromReply(*reply, offset, ok); + int32_t code; + std::string str; + std::tie(code, str) = GetStatusCodeAndStr(*status); + ASSERT_EQ(code, 1); + ASSERT_EQ(offset, reply_size); + } + + // send valid request message followed by garbage + { + std::string req = CreateSerializedRequestMessage("set_mesh_data"); + + std::vector data; + for (int i = 0; i < 123; ++i) { + data.push_back(rng() % 256); + } + BufferConnection buf_connection; + buf_connection.Send(req.data(), req.size()); + buf_connection.Send(data.data(), data.size()); + + // send to receiver + Connection connection(connection_address, 500, 500); + std::string buf = buf_connection.buffer().str(); + auto reply = connection.Send(buf.data(), buf.size()); + const void* reply_data; + size_t reply_size; + std::tie(reply_data, reply_size) = GetZMQMessageDataAndSize(*reply); + + size_t offset = 0; + bool ok; + auto status = UnpackStatusFromReply(*reply, offset, ok); + int32_t code; + std::string str; + std::tie(code, str) = GetStatusCodeAndStr(*status); + ASSERT_NE(code, 0); + ASSERT_EQ(offset, reply_size); + } + + // send only garbage + { + std::vector data; + for (int i = 0; i < 1234; ++i) { + data.push_back(rng() % 256); + } + + BufferConnection buf_connection; + buf_connection.Send(data.data(), data.size()); + + // send to receiver + Connection connection(connection_address, 500, 500); + std::string buf = buf_connection.buffer().str(); + auto reply = connection.Send(buf.data(), buf.size()); + const void* reply_data; + size_t reply_size; + std::tie(reply_data, reply_size) = GetZMQMessageDataAndSize(*reply); + + size_t offset = 0; + bool ok; + auto status = UnpackStatusFromReply(*reply, offset, ok); + int32_t code; + std::string str; + std::tie(code, str) = GetStatusCodeAndStr(*status); + ASSERT_NE(code, 0); + ASSERT_EQ(offset, reply_size); + } + + receiver.Stop(); +} + +} // namespace tests +} // namespace open3d diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 3b07ce17cd9..8f77b67325d 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -48,5 +48,8 @@ global-exclude *.py[co] # ml module recursive-include open3d/ml *.py +# visualizer module +recursive-include open3d/visualization *.py + # gui include open3d/resources/* diff --git a/python/open3d/__init__.py b/python/open3d/__init__.py index ddc98460cb7..e1f14672c18 100644 --- a/python/open3d/__init__.py +++ b/python/open3d/__init__.py @@ -50,9 +50,9 @@ from open3d.pybind import io from open3d.pybind import pipelines from open3d.pybind import utility -from open3d.pybind import visualization import open3d.core +import open3d.visualization __version__ = "@PROJECT_VERSION@" diff --git a/python/open3d/visualization/__init__.py b/python/open3d/visualization/__init__.py index 37ac9412d9a..034f4a9ba38 100644 --- a/python/open3d/visualization/__init__.py +++ b/python/open3d/visualization/__init__.py @@ -25,3 +25,6 @@ if "@BUILD_GUI@" == "ON": from open3d.pybind.visualization import gui + +from open3d.pybind.visualization import * +from ._external_visualizer import * diff --git a/python/open3d/visualization/_external_visualizer.py b/python/open3d/visualization/_external_visualizer.py new file mode 100644 index 00000000000..58ce2bb9ac5 --- /dev/null +++ b/python/open3d/visualization/_external_visualizer.py @@ -0,0 +1,131 @@ +import open3d as o3d + +if o3d._build_config['BUILD_RPC_INTERFACE']: + __all__ = ['ExternalVisualizer', 'EV'] +else: + __all__ = [] + + +class ExternalVisualizer: + """This class allows to send data to an external Visualizer + + Example: + This example sends a point cloud to the visualizer:: + + import open3d as o3d + import numpy as np + ev = o3d.visualizer.ExternalVisualizer() + pcd = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(np.random.rand(100,3))) + ev.set(pcd) + + Args: + address: The address where the visualizer is running. + The default is localhost. + timeout: The timeout for sending data in milliseconds. + """ + + def __init__(self, address='tcp://127.0.0.1:51454', timeout=10000): + self.address = address + self.timeout = timeout + + def set(self, obj=None, path='', time=0, layer='', connection=None): + """Send Open3D objects for visualization to the visualizer. + + Example: + To quickly send a single object just write:: + ev.set(point_cloud) + + To place the object at a specific location in the scene tree do:: + ev.set(point_cloud, path='group/mypoints', time=42, layer='') + Note that depending on the visualizer some arguments like time or + layer may not be supported and will be ignored. + + To set multiple objects use a list to pass multiple objects:: + ev.set([point_cloud, mesh, camera]) + Each entry in the list can be a tuple specifying all or some of the + location parameters:: + ev.set(objs=[(point_cloud,'group/mypoints', 1, 'layer1'), + (mesh, 'group/mymesh'), + camera + ] + + Args: + obj: A geometry or camera object or a list of objects. See the + example seection for usage instructions. + + path: A path describing a location in the scene tree. + + time: An integer time value associated with the object. + + layer: The layer associated with the object. + + connection: A connection object to use for sending data. This + parameter can be used to override the default object. + """ + if connection is None: + connection = o3d.io.rpc.Connection(address=self.address, + timeout=self.timeout) + result = [] + if isinstance(obj, (tuple, list)): + # item can be just an object or a tuple with path, time, layer, e.g., + # set(obj=[point_cloud, mesh, camera]) + # set(obj=[(point_cloud,'group/mypoints', 1, 'layer1'), + # (mesh, 'group/mymesh'), + # camera + # ] + for item in obj: + if isinstance(item, (tuple, list)): + if len(item) in range(1, 5): + result.append(self.set(*item, connection=connection)) + else: + result.append(self.set(item, connection=connection)) + elif isinstance(obj, o3d.geometry.PointCloud): + status = o3d.io.rpc.set_point_cloud(obj, + path=path, + time=time, + layer=layer, + connection=connection) + result.append(status) + elif isinstance(obj, o3d.geometry.TriangleMesh): + status = o3d.io.rpc.set_triangle_mesh(obj, + path=path, + time=time, + layer=layer, + connection=connection) + result.append(status) + elif isinstance(obj, o3d.camera.PinholeCameraParameters): + status = o3d.io.rpc.set_legacy_camera(obj, + path=path, + time=time, + layer=layer, + connection=connection) + result.append(status) + else: + raise Exception("Unsupported object type '{}'".format(str( + type(obj)))) + + return all(result) + + def set_time(self, time): + """Sets the time in the external visualizer + + Args: + time: The time value + """ + connection = o3d.io.rpc.Connection(address=self.address, + timeout=self.timeout) + return o3d.io.rpc.set_time(time, connection) + + def set_active_camera(self, path): + """Sets the time in the external visualizer + + Args: + path: A path describing a location in the scene tree. + """ + connection = o3d.io.rpc.Connection(address=self.address, + timeout=self.timeout) + return o3d.io.rpc.set_active_camera(path, connection) + + +# convenience default external visualizer +EV = ExternalVisualizer() diff --git a/python/test/test_remote_functions.py b/python/test/test_remote_functions.py new file mode 100644 index 00000000000..6db4f8e3b79 --- /dev/null +++ b/python/test/test_remote_functions.py @@ -0,0 +1,74 @@ +# ---------------------------------------------------------------------------- +# - Open3D: www.open3d.org - +# ---------------------------------------------------------------------------- +# The MIT License (MIT) +# +# Copyright (c) 2020 www.open3d.org +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# ---------------------------------------------------------------------------- + +import os +import open3d as o3d +import numpy as np +import pytest + +# skip all tests if the ml ops were not built +pytestmark = pytest.mark.skipif(not o3d._build_config['BUILD_RPC_INTERFACE'], + reason='rpc interface not built.') +if os.name == 'nt': + address = 'tcp://127.0.0.1:51455' +else: + address = 'ipc:///tmp/open3d_ipc' + + +def test_external_visualizer(): + o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Debug) + + # create dummy receiver which will receive all data + receiver = o3d.io.rpc._DummyReceiver(address=address) + receiver.start() + + # create ev with the same address + ev = o3d.visualization.ExternalVisualizer(address=address) + + # create some objects + mesh = o3d.geometry.TriangleMesh.create_torus() + pcd = o3d.geometry.PointCloud( + o3d.utility.Vector3dVector(np.random.rand(100, 3))) + camera = o3d.camera.PinholeCameraParameters() + camera.extrinsic = np.eye(4) + + # send single objects + assert ev.set(pcd, path='bla/pcd', time=42) + assert ev.set(mesh, path='bla/mesh', time=42) + assert ev.set(camera, path='bla/camera', time=42) + + # send multiple objects + assert ev.set(obj=[pcd, mesh, camera]) + + # send multiple objects with args + assert ev.set(obj=[(pcd, 'pcd', 1), (mesh, 'mesh', 2), (camera, 'camera', + 3)]) + + # test other commands + ev.set_time(10) + ev.set_active_camera('camera') + + receiver.stop() diff --git a/util/run_ci.sh b/util/run_ci.sh index bac4a65d8b5..b73aa01ef89 100755 --- a/util/run_ci.sh +++ b/util/run_ci.sh @@ -87,6 +87,14 @@ fi if [ "$BUILD_TENSORFLOW_OPS" == "ON" -o "$BUILD_PYTORCH_OPS" == "ON" ]; then reportRun pip install -U yapf==0.30.0 fi + +# build the rpc interface only if we do not build the cuda module and the +# ml module to keep build times short +if [ "$BUILD_CUDA_MODULE" == "OFF" -a "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then + BUILD_RPC_INTERFACE="ON" +else + BUILD_RPC_INTERFACE="OFF" +fi mkdir build cd build @@ -97,6 +105,7 @@ cmakeOptions="-DBUILD_SHARED_LIBS=${SHARED} \ -DCUDA_ARCH=BasicPTX \ -DBUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS} \ -DBUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS} \ + -DBUILD_RPC_INTERFACE=${BUILD_RPC_INTERFACE} \ -DBUILD_UNIT_TESTS=ON \ -DBUILD_BENCHMARKS=ON \ -DCMAKE_INSTALL_PREFIX=${OPEN3D_INSTALL_DIR} \