(feat) Add support for Vive Facial Tracker

.gitignore

@@ -6,19 +6,21 @@ BabbleApp/babble_settings.json
 BabbleApp/babble_settings.backup
 BabbleApp/build
 BabbleApp/dist
+/BabbleApp/Models/dev
+/BabbleApp/venv
 /vivetest
 /training
 .vscode
 /testing
 /old_models
 /Lib
 /share
-/BabbleApp/Models/dev
 
 scripts/example_build_app_and_installer.bat
 scripts/example_build_app_and_installer.bat
 scripts/installer.iss
 Glia_cap.py
 lazyass_minecraft_script.py
 scripts/example_build_app_and_installer.bat
-/BabbleApp/venv
+*.kdev*
+*.code-workspace

BabbleApp/.gitignore

@@ -0,0 +1 @@
+*.log

BabbleApp/babbleapp.py

@@ -22,6 +22,7 @@
 import requests
 import threading
 import asyncio
+import logging
 from ctypes import c_int
 from babble_model_loader import *
 from camera_widget import CameraWidget
@@ -111,6 +112,9 @@ async def async_main():
 
     # Run the update check
     await check_for_updates(config, notification_manager)
+
+    # Uncomment for low-level Vive Facial Tracker logging
+    # logging.basicConfig(filename='BabbleApp.log', filemode='w', encoding='utf-8', level=logging.INFO)
 
     cancellation_event = threading.Event()
     ROSC = False

BabbleApp/camera.py

@@ -3,16 +3,20 @@
 import numpy as np
 import queue
 import serial
-import serial.tools.list_ports
-import threading
+import sys
 import time
+import traceback
+import threading
+from enum import Enum
+import serial.tools.list_ports
 from lang_manager import LocaleStringManager as lang
 
 from colorama import Fore
 from config import BabbleConfig, BabbleSettingsConfig
-from utils.misc_utils import get_camera_index_by_name, list_camera_names, is_nt
-from enum import Enum
-import sys
+from utils.misc_utils import get_camera_index_by_name, list_camera_names
+
+from vivefacialtracker.vivetracker import ViveTracker
+from vivefacialtracker.camera_controller import FTCameraController
 
 WAIT_TIME = 0.1
 BUFFER_SIZE = 32768
@@ -24,6 +28,7 @@
 #  packet (packet-size bytes)
 ETVR_HEADER = b"\xff\xa0\xff\xa1"
 ETVR_HEADER_LEN = 6
+PORTS = ("COM", "/dev/ttyACM")
 
 
 class CameraState(Enum):
@@ -54,13 +59,15 @@ def __init__(
         self.cancellation_event = cancellation_event
         self.current_capture_source = config.capture_source
         self.cv2_camera: "cv2.VideoCapture" = None
+        self.vft_camera: FTCameraController = None
 
         self.serial_connection = None
         self.last_frame_time = time.time()
         self.fps = 0
         self.bps = 0
         self.start = True
         self.buffer = b""
+        self.frame_number = 0
         self.FRAME_SIZE = [0, 0]
 
         self.error_message = f'{Fore.YELLOW}[{lang._instance.get_string("log.warn")}] {lang._instance.get_string("info.enterCaptureOne")} {{}} {lang._instance.get_string("info.enterCaptureTwo")}{Fore.RESET}'
@@ -78,6 +85,8 @@ def run(self):
                 print(
                     f'{Fore.CYAN}[{lang._instance.get_string("log.info")}] {lang._instance.get_string("info.exitCaptureThread")}{Fore.RESET}'
                 )
+                if self.vft_camera is not None:
+                    self.vft_camera.close()
                 return
             should_push = True
             # If things aren't open, retry until they are. Don't let read requests come in any earlier
@@ -86,8 +95,15 @@ def run(self):
                 self.config.capture_source is not None
                 and self.config.capture_source != ""
             ):
-                ports = ("COM", "/dev/ttyACM")
-                if any(x in str(self.config.capture_source) for x in ports):
+                isSerial = any(x in str(self.config.capture_source) for x in PORTS)
+
+                if isSerial:
+                    if self.cv2_camera is not None:
+                        self.cv2_camera.release()
+                        self.cv2_camera = None
+                    if self.vft_camera is not None:
+                        self.vft_camera.close()
+                    self.device_is_vft = False;
                     if (
                         self.serial_connection is None
                         or self.camera_status == CameraState.DISCONNECTED
@@ -96,25 +112,51 @@ def run(self):
                         port = self.config.capture_source
                         self.current_capture_source = port
                         self.start_serial_connection(port)
-                else:
-                    if (
+                elif ViveTracker.is_device_vive_tracker(self.config.capture_source):
+                    if self.cv2_camera is not None:
+                        self.cv2_camera.release()
+                        self.cv2_camera = None
+                    self.device_is_vft = True;
+
+                    if self.vft_camera is None:
+                        print(self.error_message.format(self.config.capture_source))
+                        # capture_source is an index into a list of devices, so it should be treated as such
+                        if self.cancellation_event.wait(WAIT_TIME):
+                            return
+                        try:
+                            # Only create the camera once, reuse it
+                            self.vft_camera = FTCameraController(get_camera_index_by_name(self.config.capture_source))
+                            self.vft_camera.open()
+                            should_push = False
+                        except Exception:
+                            print(traceback.format_exc())
+                            if self.vft_camera is not None:
+                                self.vft_camera.close()
+                    else:
+                        # If the camera is already open it don't spam it!!
+                        if (not self.vft_camera.is_open):
+                            self.vft_camera.open()
+                            should_push = False
+                elif (
                         self.cv2_camera is None
                         or not self.cv2_camera.isOpened()
                         or self.camera_status == CameraState.DISCONNECTED
-                        or self.config.capture_source != self.current_capture_source
+                        or get_camera_index_by_name(self.config.capture_source) != self.current_capture_source 
                     ):
+                        if self.vft_camera is not None:
+                            self.vft_camera.close()
+                        self.device_is_vft = False;
+
                         print(self.error_message.format(self.config.capture_source))
                         # This requires a wait, otherwise we can error and possible screw up the camera
                         # firmware. Fickle things.
                         if self.cancellation_event.wait(WAIT_TIME):
                             return
-
+                        
                         if self.config.capture_source not in self.camera_list:
                             self.current_capture_source = self.config.capture_source
                         else:
-                            self.current_capture_source = get_camera_index_by_name(
-                                self.config.capture_source
-                            )
+                            self.current_capture_source = get_camera_index_by_name(self.config.capture_source)
 
                         if self.config.use_ffmpeg:
                             self.cv2_camera = cv2.VideoCapture(
@@ -139,9 +181,12 @@ def run(self):
                             self.cv2_camera.set(
                                 cv2.CAP_PROP_FPS, self.settings.gui_cam_framerate
                             )
+
                         should_push = False
             else:
                 # We don't have a capture source to try yet, wait for one to show up in the GUI.
+                if self.vft_camera is not None:
+                    self.vft_camera.close()
                 if self.cancellation_event.wait(WAIT_TIME):
                     self.camera_status = CameraState.DISCONNECTED
                     return
@@ -151,24 +196,35 @@ def run(self):
             if should_push and not self.capture_event.wait(timeout=0.02):
                 continue
             if self.config.capture_source is not None:
-                ports = ("COM", "/dev/ttyACM")
-                if any(x in str(self.config.capture_source) for x in ports):
+                if isSerial:
                     self.get_serial_camera_picture(should_push)
                 else:
                     self.__del__()
-                    self.get_cv2_camera_picture(should_push)
+                    self.get_camera_picture(should_push)
                 if not should_push:
                     # if we get all the way down here, consider ourselves connected
                     self.camera_status = CameraState.CONNECTED
 
-    def get_cv2_camera_picture(self, should_push):
+    def get_camera_picture(self, should_push):
         try:
-            ret, image = self.cv2_camera.read()
-            if not ret:
-                self.cv2_camera.set(cv2.CAP_PROP_POS_FRAMES, 0)
-                raise RuntimeError(lang._instance.get_string("error.frame"))
+            image = None
+            # Is the current camera a Vive Facial Tracker and have we opened a connection to it before?
+            if self.vft_camera is not None and self.device_is_vft:
+                image = self.vft_camera.get_image()
+                if image is None:
+                    return
+                self.frame_number = self.frame_number + 1
+            elif self.cv2_camera is not None and self.cv2_camera.isOpened():
+                ret, image = self.cv2_camera.read()
+                if not ret:
+                    self.cv2_camera.set(cv2.CAP_PROP_POS_FRAMES, 0)
+                    raise RuntimeError(lang._instance.get_string("error.frame"))
+                self.frame_number = self.cv2_camera.get(cv2.CAP_PROP_POS_FRAMES) + 1
+            else:
+                # Switching from a Vive Facial Tracker to a CV2 camera
+                return
+
             self.FRAME_SIZE = image.shape
-            frame_number = self.cv2_camera.get(cv2.CAP_PROP_POS_FRAMES)
             # Calculate FPS
             current_frame_time = time.time()    # Should be using "time.perf_counter()", not worth ~3x cycles?
             delta_time = current_frame_time - self.last_frame_time
@@ -179,8 +235,9 @@ def get_cv2_camera_picture(self, should_push):
             self.bps = image.nbytes * self.fps
 
             if should_push:
-                self.push_image_to_queue(image, frame_number + 1, self.fps)
+                self.push_image_to_queue(image, self.frame_number, self.fps)
         except Exception:
+            FTCameraController._logger.exception("get_image")
             print(
                 f'{Fore.YELLOW}[{lang._instance.get_string("log.warn")}] {lang._instance.get_string("warn.captureProblem")}{Fore.RESET}'
             )

BabbleApp/camera_widget.py

@@ -323,10 +323,7 @@ def render(self, window, event, values):
                 if any(x in str(value) for x in ports):
                     self.config.capture_source = value
                 else:
-                    cam = get_camera_index_by_name(value)   # Set capture_source to the UVC index. Otherwise treat value like an ipcam if we return none
-                    if cam != None:
-                        self.config.capture_source = cam
-                    elif is_valid_int_input(value): 
+                    if is_valid_int_input(value): 
                         self.config.capture_source = int(value)
                     else:
                         self.config.capture_source = value

BabbleApp/utils/misc_utils.py

@@ -71,20 +71,16 @@ def is_uvc_device(device):
 def list_linux_uvc_devices():
     """List UVC video devices on Linux (excluding metadata devices)"""
     try:
-        result = subprocess.run(["v4l2-ctl", "--list-devices"], stdout=subprocess.PIPE)
-        output = result.stdout.decode("utf-8")
-
-        lines = output.splitlines()
+        # v4l2-ctl --list-devices breaks if video devices are non-sequential.
+        # So this might be better?
+        result = glob.glob("/dev/video*");
         devices = []
         current_device = None
-        for line in lines:
-            if not line.startswith("\t"):
-                current_device = line.strip()
-            else:
-                if "/dev/video" in line and is_uvc_device(line.strip()):
-                    devices.append(
-                        line.strip()
-                    )  # We return the path like '/dev/video0'
+        for line in result:
+            if is_uvc_device(line):
+                devices.append(
+                    line
+                )  # We return the path like '/dev/video0'
 
         return devices
 
@@ -147,10 +143,9 @@ def get_camera_index_by_name(name):
     cam_list = list_camera_names()
 
     # On Linux, we use device paths like '/dev/video0' and match directly
+    # OpenCV expects the actual /dev/video#, not the offset into the device list
     if os_type == "Linux":
-        for i, device_path in enumerate(cam_list):
-            if device_path == name:
-                return i
+        return int(str.replace(name,"/dev/video",""));
 
     # On Windows, match by camera name
     elif is_nt: