[Experimental] Add a RuntimeOption to set inter and intra op threadpool sizes

build/install_python_deps.sh

@@ -5,7 +5,7 @@ set -e
 NEUROPOD_PYTHON_BINARY="python${NEUROPOD_PYTHON_VERSION}"
 
 # Install pip
-wget https://bootstrap.pypa.io/2.7/get-pip.py -O /tmp/get-pip.py
+wget https://bootstrap.pypa.io/pip/2.7/get-pip.py -O /tmp/get-pip.py
 ${NEUROPOD_PYTHON_BINARY} /tmp/get-pip.py
 
 # Setup a virtualenv

source/neuropod/backends/tensorflow/tf_backend.cc

@@ -93,7 +93,7 @@
 }
 
 // Get TF session options given Neuropod RuntimeOptions
-tensorflow::SessionOptions get_tf_opts(const RuntimeOptions & /*unused*/)
+tensorflow::SessionOptions get_tf_opts(const RuntimeOptions &runtime_opts)
 {
     tensorflow::SessionOptions opts;
 
@@ -103,6 +103,26 @@
     opts.config.set_allow_soft_placement(true);
     opts.config.set_log_device_placement(false);
 
+    // Set intra and inter op parallelism
+    // See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto
+    if (runtime_opts.experimental_intra_op_parallelism_threads != 0)
+    {
+        opts.config.set_intra_op_parallelism_threads(
+            static_cast<int32_t>(runtime_opts.experimental_intra_op_parallelism_threads));
+    }
+
+    if (runtime_opts.experimental_inter_op_parallelism_threads == 1)
+    {
+        // Only use the caller thread
+        opts.config.set_inter_op_parallelism_threads(-1);
+    }
+    else if (runtime_opts.experimental_inter_op_parallelism_threads > 1)
+    {
+        // The number in runtime_opts includes the caller thread
+        opts.config.set_inter_op_parallelism_threads(
+            static_cast<int32_t>(runtime_opts.experimental_inter_op_parallelism_threads) - 1);
+    }
+
     // Note: we can't use GPUOptions::visible_device_list as it is a per process setting
     //
     // From: https://github.com/tensorflow/tensorflow/issues/18861#issuecomment-385610497

source/neuropod/backends/torchscript/torch_backend.cc

@@ -225,6 +225,21 @@
 TorchNeuropodBackend::TorchNeuropodBackend(const std::string &neuropod_path, const RuntimeOptions &options)
     : NeuropodBackendWithDefaultAllocator<TorchNeuropodTensor>(neuropod_path, options)
 {
+// inter and intra op parallelism settings only supported in Torch >= 1.2.0
+#if CAFFE2_NIGHTLY_VERSION >= 20190808
+    // Set intra and inter op parallelism
+    // See https://pytorch.org/docs/stable/notes/cpu_threading_torchscript_inference.html#runtime-api
+    if (options.experimental_inter_op_parallelism_threads != 0)
+    {
+        at::set_num_interop_threads(static_cast<int32_t>(options.experimental_inter_op_parallelism_threads));
+    }
+
+    if (options.experimental_intra_op_parallelism_threads != 0)
+    {
+        at::set_num_threads(static_cast<int32_t>(options.experimental_intra_op_parallelism_threads));
+    }
+#endif
+
     if (options.load_model_at_construction)
     {
         load_model();

source/neuropod/options.hh

@@ -75,6 +75,23 @@ struct RuntimeOptions
 
     // Whether or not to disable shape and type checking when running inference
     bool disable_shape_and_type_checking = false;
+
+    // EXPERIMENTAL
+    // Set the intra and inter op parallelism for the underlying framework
+    // Within a given process, only the first usage of the below configuration is used
+    // See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto
+    // and https://pytorch.org/docs/stable/notes/cpu_threading_torchscript_inference.html#runtime-api
+    // for more details
+    // For true per-model control of these values, use out-of-process execution (see above)
+    // A value of 0 means system defined
+    // Note: for TorchScript, requires at least Torch 1.2.0
+    uint32_t experimental_intra_op_parallelism_threads = 0;
+
+    // EXPERIMENTAL
+    // A value of 0 means system defined
+    // Note: this count includes the caller thread
+    // Note: for TorchScript, requires at least Torch 1.2.0
+    uint32_t experimental_inter_op_parallelism_threads = 0;
 };
 
 } // namespace neuropod