diff --git a/3rdparty/cmake/FindPytorch.cmake b/3rdparty/cmake/FindPytorch.cmake
index 28da7ec71ba..31babdca149 100644
--- a/3rdparty/cmake/FindPytorch.cmake
+++ b/3rdparty/cmake/FindPytorch.cmake
@@ -55,15 +55,17 @@ if(NOT Pytorch_FOUND)
     unset(PyTorch_FETCH_PROPERTIES)
     unset(PyTorch_PROPERTIES)
 
+    if(BUILD_CUDA_MODULE)
     # Using CUDA 12.x and Pytorch <2.4 gives the error "Unknown CUDA Architecture Name 9.0a in CUDA_SELECT_NVCC_ARCH_FLAGS".
     # As a workaround we explicitly set TORCH_CUDA_ARCH_LIST
-    set(TORCH_CUDA_ARCH_LIST "")
-    foreach(arch IN LISTS CMAKE_CUDA_ARCHITECTURES)
-        translate_arch_string("${arch}" ptarch)
-        list(APPEND TORCH_CUDA_ARCH_LIST "${ptarch}")
-    endforeach()
+        set(TORCH_CUDA_ARCH_LIST "")
+        foreach(arch IN LISTS CMAKE_CUDA_ARCHITECTURES)
+            translate_arch_string("${arch}" ptarch)
+            list(APPEND TORCH_CUDA_ARCH_LIST "${ptarch}")
+        endforeach()
+        message(STATUS "Using top level CMAKE_CUDA_ARCHITECTURES for TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST}")
+    endif()
     
-    message(STATUS "Using top level CMAKE_CUDA_ARCHITECTURES for TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST}")
     # Use the cmake config provided by torch
     find_package(Torch REQUIRED PATHS "${Pytorch_ROOT}"
                  NO_DEFAULT_PATH)
diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel
index 9d86856c177..71a16a2ac13 100644
--- a/docker/Dockerfile.wheel
+++ b/docker/Dockerfile.wheel
@@ -113,7 +113,7 @@ COPY ./python/requirements.txt /root/Open3D/python/
 COPY ./python/requirements_jupyter_build.txt /root/Open3D/python/
 COPY ./python/requirements_jupyter_install.txt /root/Open3D/python/
 RUN source /root/Open3D/util/ci_utils.sh \
- && install_python_dependencies with-cuda with-jupyter
+ && install_python_dependencies with-jupyter
 
 # Open3D Jupyter dependencies
 RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \