Start improving high-throughput multi-camera recording path and diagnostics

.gitignore

@@ -113,3 +113,8 @@ venv.bak/
 !dlclivegui/config.py
 # uv package files
 uv.lock
+
+# profiling
+profile*.svg
+scalene*.json
+scalene*.html

dlclivegui/cameras/backends/basler_backend.py

@@ -9,7 +9,7 @@
 
 import numpy as np
 
-from ...config import BASLER_DO_LOG_TIMING, CameraTriggerSettings
+from ...config import BASLER_DO_LOG_TIMING, DEBUG_TRIGGER_LOGS, CameraTriggerSettings
 from ...utils.stats import WorkerTimingStats
 from ..base import CameraBackend, SupportLevel, register_backend
 
@@ -638,7 +638,8 @@ def open(self) -> None:
  pass
 
  self._camera.StartGrabbing(
- pylon.GrabStrategy_LatestImageOnly,
+ # pylon.GrabStrategy_LatestImageOnly,
+ pylon.GrabStrategy_OneByOne,
  )
  LOG.info(
  "[Basler] grabbing=%s max_buffers=%s",
@@ -661,7 +662,7 @@ def open(self) -> None:
  )
 
  # ----------------------------
- # Persist stable identity into namespace (migration-safe)
+ # Persist stable identity into namespace
  # ----------------------------
  try:
  serial = device.GetSerialNumber()
@@ -958,6 +959,9 @@ def _set_numeric_feature(self, name: str, value, *, strict: bool = False) -> boo
  return False
 
  def _debug_trigger_nodes(self, *, context: str = "") -> None:
+ if not LOG.isEnabledFor(logging.DEBUG) or not DEBUG_TRIGGER_LOGS:
+ return
+
  names = (
  "TriggerSelector",
  "TriggerMode",

dlclivegui/cameras/backends/gentl_backend.py

@@ -12,7 +12,7 @@
 import cv2
 import numpy as np
 
-from ...config import CameraTriggerSettings
+from ...config import DEBUG_TRIGGER_LOGS, CameraTriggerSettings
 from ..base import CameraBackend, SupportLevel, register_backend
 from ..factory import DetectedCamera
 from .utils import gentl_discovery as cti_finder
@@ -199,6 +199,9 @@ def static_capabilities(cls) -> dict[str, SupportLevel]:
  }
 
  def _debug_trigger_nodes(self, node_map, *, context: str = "") -> None:
+ if not LOG.isEnabledFor(logging.DEBUG) or not DEBUG_TRIGGER_LOGS:
+ return
+
  names = (
  "TriggerMode",
  "TriggerSelector",

dlclivegui/config.py

@@ -31,6 +31,8 @@
 SINGLE_CAMERA_WORKER_DO_LOG_TIMING: bool = False
 MULTI_CAMERA_WORKER_DO_LOG_TIMING: bool = False
 REC_DO_LOG_TIMING: bool = False
+### Trigger debug logging
+DEBUG_TRIGGER_LOGS = False
 # MAIN_WINDOW_DO_LOG_TIMING: bool = False
 #### Backends
 BASLER_DO_LOG_TIMING: bool = False

dlclivegui/gui/main_window.py

@@ -138,6 +138,7 @@ def __init__(self, config: ApplicationSettings | None = None):
  self._raw_frame: np.ndarray | None = None
  self._last_pose: PoseResult | None = None
  self._dlc_active: bool = False
+ self._pending_recording_after_preview = False
  self._active_camera_settings: CameraSettings | None = None
  self._last_drop_warning = 0.0
  self._last_recorder_summary = "Recorder idle"
@@ -633,13 +634,29 @@ def _build_recording_group(self) -> QGroupBox:
 
  form.addRow(grid)
 
- # Record with overlays
+ # Recording options
  self.record_with_overlays_checkbox = QCheckBox("Record video with overlays")
  self.record_with_overlays_checkbox.setToolTip(
  "Enable to include pose overlays in recorded video (keypoints & bounding boxes)"
  )
  self.record_with_overlays_checkbox.setChecked(False)
- form.addRow(self.record_with_overlays_checkbox)
+
+ self.fast_encoding_checkbox = QCheckBox("Use faster encoding parameters")
+ self.fast_encoding_checkbox.setToolTip(
+ "Use faster FFmpeg parameters for supported codecs.\n"
+ "For libx264/libx265 this uses preset=ultrafast and tune=zerolatency.\n"
+ "This can improve recording throughput but may increase file size."
+ )
+ self.fast_encoding_checkbox.setChecked(False)
+
+ recording_options = QWidget()
+ recording_options_layout = QHBoxLayout(recording_options)
+ recording_options_layout.setContentsMargins(0, 0, 0, 0)
+ recording_options_layout.addWidget(self.record_with_overlays_checkbox)
+ recording_options_layout.addWidget(self.fast_encoding_checkbox)
+ recording_options_layout.addStretch(1)
+
+ form.addRow(recording_options)
 
  # Wrap recording buttons in a widget to prevent shifting
  recording_button_widget = QWidget()
@@ -772,6 +789,7 @@ def _connect_signals(self) -> None:
  # Multi-camera controller signals (used for both single and multi-camera modes)
  self.multi_camera_controller.frame_ready.connect(self._on_multi_frame_processing_ready)
  self.multi_camera_controller.display_ready.connect(self._on_multi_frame_display_ready)
+ self.multi_camera_controller.recording_frame_ready.connect(self._on_recording_frame_ready)
  self.multi_camera_controller.all_started.connect(self._on_multi_camera_started)
  self.multi_camera_controller.all_stopped.connect(self._on_multi_camera_stopped)
  self.multi_camera_controller.camera_error.connect(self._on_multi_camera_error)
@@ -783,19 +801,17 @@ def _connect_signals(self) -> None:
  self._dlc.initialized.connect(self._on_dlc_initialised)
  self.dlc_camera_combo.currentIndexChanged.connect(self._on_dlc_camera_changed)
  self.dlc_camera_combo.currentTextChanged.connect(self.dlc_camera_combo.update_shrink_width)
+ self.allow_processor_ctrl_checkbox.stateChanged.connect(lambda _s: self._update_dlc_controls_enabled())
+ self.allow_processor_ctrl_checkbox.stateChanged.connect(lambda _s: self._update_processor_status())
 
  # Recording settings
  ## Session name persistence + preview updates
- if hasattr(self, "session_name_edit"):
- self.session_name_edit.editingFinished.connect(self._on_session_name_editing_finished)
- if hasattr(self, "use_timestamp_checkbox"):
- self.use_timestamp_checkbox.stateChanged.connect(self._on_use_timestamp_changed)
- if hasattr(self, "output_directory_edit"):
- self.output_directory_edit.textChanged.connect(lambda _t: self._update_recording_path_preview())
- if hasattr(self, "filename_edit"):
- self.filename_edit.textChanged.connect(lambda _t: self._update_recording_path_preview())
- if hasattr(self, "container_combo"):
- self.container_combo.currentTextChanged.connect(lambda _t: self._update_recording_path_preview())
+ self.session_name_edit.editingFinished.connect(self._on_session_name_editing_finished)
+ self.use_timestamp_checkbox.stateChanged.connect(self._on_use_timestamp_changed)
+ self.output_directory_edit.textChanged.connect(lambda _t: self._update_recording_path_preview())
+ self.filename_edit.textChanged.connect(lambda _t: self._update_recording_path_preview())
+ self.container_combo.currentTextChanged.connect(lambda _t: self._update_recording_path_preview())
+ self.fast_encoding_checkbox.stateChanged.connect(self._on_fast_encoding_changed)
 
  # ------------------------------------------------------------------
  # Config
@@ -822,6 +838,11 @@ def _apply_config(self, config: ApplicationSettings) -> None:
  self.codec_combo.addItem(recording.codec)
  self.codec_combo.setCurrentIndex(self.codec_combo.count() - 1)
  self.crf_spin.setValue(int(recording.crf))
+
+ if hasattr(self, "fast_encoding_checkbox"):
+ config_fast_encoding = bool(getattr(recording, "fast_encoding", False))
+ self.fast_encoding_checkbox.setChecked(self._settings_store.get_fast_encoding(default=config_fast_encoding))
+
  ## Restore persisted session name if empty
  if hasattr(self, "session_name_edit"):
  if not self.session_name_edit.text().strip():
@@ -934,6 +955,9 @@ def _recording_settings_from_ui(self) -> RecordingSettings:
  container=self.container_combo.currentText().strip() or "mp4",
  codec=self.codec_combo.currentText().strip() or "libx264",
  crf=int(self.crf_spin.value()),
+ fast_encoding=bool(
+ getattr(self, "fast_encoding_checkbox", None) and self.fast_encoding_checkbox.isChecked()
+ ),
  )
 
  def _bbox_settings_from_ui(self) -> BoundingBoxSettings:
@@ -1193,6 +1217,9 @@ def _on_use_timestamp_changed(self, _state: int) -> None:
  self._settings_store.set_use_timestamp(self.use_timestamp_checkbox.isChecked())
  self._update_recording_path_preview()
 
+ def _on_fast_encoding_changed(self, _state: int) -> None:
+ self._settings_store.set_fast_encoding(self.fast_encoding_checkbox.isChecked())
+
  def _on_colormap_changed(self, _index: int) -> None:
  self._colormap = color_ui.get_cmap_name_from_combo(self.cmap_combo, fallback=self._colormap)
  if self._current_frame is not None:
@@ -1376,15 +1403,33 @@ def _render_overlays_for_recording(self, cam_id, frame):
  )
  return output
 
+ def _on_recording_frame_ready(self, camera_id: str, frame: np.ndarray, timestamp: float) -> None:
+ """Handle full-rate per-camera frames for recording only.
+
+ Intentionally lean:
+ - no MultiFrameData processing
+ - no DLC routing
+ - no display state updates
+ - no FPS tracker
+ - optional overlays only if user requested recording overlays
+ """
+ if not self._rec_manager.is_active:
+ return
+
+ if self.record_with_overlays_checkbox.isChecked():
+ frame = self._render_overlays_for_recording(camera_id, frame)
+
+ self._rec_manager.write_frame(camera_id, frame, timestamp)
+
  def _on_multi_frame_processing_ready(self, frame_data: MultiFrameData) -> None:
  """Handle frames from multiple cameras.
 
  Priority:
- 1. DLC processing (highest priority - enqueue immediately, only for DLC camera)
- 2. Recording (queued writes, non-blocking)
+ - DLC processing (highest priority - enqueue immediately, only for DLC camera)
  """
  self._multi_camera_frames = frame_data.frames
  self._multi_camera_display_ids = frame_data.display_ids or {}
+ self._try_start_pending_recording()
  src_id = frame_data.source_camera_id
  if src_id:
  self._fps_tracker.note_frame(src_id) # Track FPS
@@ -1422,23 +1467,12 @@ def _on_multi_frame_processing_ready(self, frame_data: MultiFrameData) -> None:
  self._raw_frame = frame
  self._dlc_tile_offset, self._dlc_tile_scale = compute_tile_info(dlc_cam_id, frame, frame_data.frames)
 
- # PRIORITY 1: DLC processing - only enqueue when DLC camera frame arrives!
+ # PRIORITY: DLC processing - only enqueue when DLC camera frame arrives!
  if self._dlc_active and is_dlc_camera_frame and dlc_cam_id in frame_data.frames:
  frame = frame_data.frames[dlc_cam_id]
  timestamp = frame_data.timestamps.get(dlc_cam_id, time.time())
  self._dlc.enqueue_frame(frame, timestamp)
 
- # PRIORITY 2: Recording (queued, non-blocking)
- if self._rec_manager.is_active and src_id in frame_data.frames:
- frame = frame_data.frames[src_id]
-
- if self.record_with_overlays_checkbox.isChecked():
- # Draw overlays for recording
- frame = self._render_overlays_for_recording(src_id, frame)
-
- ts = frame_data.timestamps.get(src_id, time.time())
- self._rec_manager.write_frame(src_id, frame, ts)
-
  def _on_multi_frame_display_ready(self, frame_data: MultiFrameData) -> None:
  """Throttled UI/display path.
 
@@ -1456,11 +1490,13 @@ def _on_multi_camera_started(self) -> None:
  self.statusBar().showMessage(f"Multi-camera preview started: {active_count} camera(s)", 5000)
  self._update_inference_buttons()
  self._update_camera_controls_enabled()
+ self._update_dlc_controls_enabled()
 
  def _on_multi_camera_stopped(self) -> None:
  """Handle all cameras stopped event."""
  # Stop all multi-camera recorders
  self._stop_multi_camera_recording()
+ self._pending_recording_after_preview = False
 
  self.preview_button.setEnabled(True)
  self.stop_preview_button.setEnabled(False)
@@ -1472,9 +1508,11 @@ def _on_multi_camera_stopped(self) -> None:
  self.statusBar().showMessage("Multi-camera preview stopped", 3000)
  self._update_inference_buttons()
  self._update_camera_controls_enabled()
+ self._update_dlc_controls_enabled()
 
  def _on_multi_camera_error(self, camera_id: str, message: str) -> None:
  """Handle error from a camera in multi-camera mode."""
+ self._pending_recording_after_preview = False
  self._show_warning(f"Camera {camera_id} error: {message}\nRecording stopped.")
  self._refresh_dlc_camera_list_running()
  if self.dlc_camera_combo.count() <= 1:
@@ -1483,6 +1521,7 @@ def _on_multi_camera_error(self, camera_id: str, message: str) -> None:
 
  def _on_multi_camera_initialization_failed(self, failures: list) -> None:
  """Handle complete failure to initialize cameras."""
+ self._pending_recording_after_preview = False
  # Build error message with details for each failed camera
  error_lines = ["Failed to initialize camera(s):"]
  for camera_id, error_msg in failures:
@@ -1518,6 +1557,7 @@ def _start_multi_camera_recording(self) -> None:
  if run_dir is None:
  self._show_error("Failed to start recording.")
  return
+ self.multi_camera_controller.set_recording_frame_do_emit(True)
 
  self._settings_store.set_session_name(session_name)
  self.start_record_button.setEnabled(False)
@@ -1528,6 +1568,9 @@ def _start_multi_camera_recording(self) -> None:
  def _stop_multi_camera_recording(self) -> None:
  if not self._rec_manager.is_active:
  return
+
+ self.multi_camera_controller.set_recording_frame_do_emit(False)
+
  self._rec_manager.stop_all()
  self.start_record_button.setEnabled(True)
  self.stop_record_button.setEnabled(False)
@@ -1639,6 +1682,7 @@ def _stop_preview(self) -> None:
  self._stop_multi_camera_recording()
 
  self.multi_camera_controller.stop(wait=True)
+ self._pending_recording_after_preview = False
  self._stop_inference(show_message=False)
  self._fps_tracker.clear()
  self._last_display_time = 0.0
@@ -1719,6 +1763,8 @@ def _update_camera_controls_enabled(self) -> None:
  recording_editable = not multi_cam_recording
  self.codec_combo.setEnabled(recording_editable)
  self.crf_spin.setEnabled(recording_editable)
+ if hasattr(self, "fast_encoding_checkbox"):
+ self.fast_encoding_checkbox.setEnabled(recording_editable)
 
  # Config cameras button should be available when not in preview/recording
  self.config_cameras_button.setEnabled(allow_changes)
@@ -1920,17 +1966,44 @@ def _start_recording(self) -> None:
  """Start recording from all active cameras."""
  # Auto-start preview if not running
  if not self.multi_camera_controller.is_running():
+ self._pending_recording_after_preview = True
  self._start_preview()
  # Wait a moment for cameras to initialize before recording
  # The recording will start after preview is confirmed running
  self.statusBar().showMessage("Starting preview before recording...", 3000)
  # Use a single-shot timer to start recording after preview starts
- QTimer.singleShot(500, self._start_multi_camera_recording)
+# QTimer.singleShot(500, self._start_multi_camera_recording)
  return
 
  # Preview already running, start recording immediately
  self._start_multi_camera_recording()
 
+ def _try_start_pending_recording(self) -> None:
+ if not self._pending_recording_after_preview:
+ return
+
+ if self._rec_manager.is_active:
+ self._pending_recording_after_preview = False
+ return
+
+ if not self.multi_camera_controller.is_running():
+ return
+
+ active_cams = self._config.multi_camera.get_active_cameras()
+ expected_ids = {get_camera_id(cam) for cam in active_cams}
+
+ if not expected_ids:
+ self._pending_recording_after_preview = False
+ return
+
+ available_ids = set(self._multi_camera_frames.keys())
+
+ if not expected_ids.issubset(available_ids):
+ return
+
+ self._pending_recording_after_preview = False
+ self._start_multi_camera_recording()
+
  def _stop_recording(self) -> None:
  """Stop recording from all cameras."""
  self._stop_multi_camera_recording()