Fix clap detection silently analyzing the wrong audio track

Root cause of the reported 'timestamps off by different amounts,
sometimes early, sometimes late' bug: _extract_audio_wav() never
passed an explicit -map to ffmpeg, so on multi-audio-track files it
relied on ffmpeg's own default-stream heuristic -- which has no
knowledge of, and isn't guaranteed to agree with, whichever audio
track mpv is actually configured to play. A wrong-track clap is
essentially unrelated to the true one (not a small timestamp-math
error), which explains the inconsistent, file-dependent symptom --
and our single-audio-track synthetic fixtures never exercised this
path at all, since they only ever had one track to pick.

Fixed by threading an explicit ffmpeg stream index through
detect_claps() -> _extract_audio_wav(), sourced in main.py from mpv's
own track-list (the ff-index of whichever audio track has
selected=True). Detection now always analyzes the exact track the
user is listening to, on both 'Detect All' and 'Detect Next'.

Also: _probe_audio_start_time() used to silently fall back to 0.0 if
ffprobe couldn't parse start_time as a plain float (e.g. N/A) -- same
failure class (silently wrong instead of loud). Now emits a Python
warning instead, so a genuinely unparseable/unusual container is at
least visible rather than silently mis-corrected.

Verified two ways: (1) ruled out an initial alternate hypothesis
(video stream's own start_time mattering) empirically against the
existing single-track fixtures before ruling it out. (2) Built a new
two-audio-track fixture (sync_test_MULTITRACK.mp4: track 1 has a beep
at 3s, track 2 at 7s) and confirmed end-to-end through the real app --
switching mpv's active track from aid=1 to aid=2 correctly flips which
clap gets detected, which is exactly the mechanism this bug required.
This commit is contained in:
ThePendulum
2026-07-26 05:10:12 +02:00
parent acd63f570d
commit 76b5f827be
5 changed files with 154 additions and 11 deletions
+23 -4
View File
@@ -45,11 +45,12 @@ class DetectWorker(QThread):
finished_ok = pyqtSignal(list) # List[ClapEvent]
finished_err = pyqtSignal(str)
def __init__(self, input_path, start_offset=0.0, max_results=None):
def __init__(self, input_path, start_offset=0.0, max_results=None, audio_stream_index=None):
super().__init__()
self.input_path = input_path
self.start_offset = start_offset
self.max_results = max_results
self.audio_stream_index = audio_stream_index
def run(self):
try:
@@ -57,6 +58,7 @@ class DetectWorker(QThread):
self.input_path,
start_offset=self.start_offset,
max_results=self.max_results,
audio_stream_index=self.audio_stream_index,
)
self.finished_ok.emit(events)
except (ClapDetectionUnavailable, RuntimeError) as e:
@@ -66,7 +68,7 @@ class DetectWorker(QThread):
class MainWindow(QMainWindow):
def __init__(self):
super().__init__()
self.setWindowTitle("A/V Sync Marker")
self.setWindowTitle("WeAreSyncing")
self.resize(960, 720)
self.input_path: Optional[str] = None
@@ -241,7 +243,7 @@ class MainWindow(QMainWindow):
self.override_spinbox.setEnabled(False)
self.log_view.clear()
self.setWindowTitle(f"A/V Sync Marker — {Path(path).name}")
self.setWindowTitle(f"WeAreSyncing — {Path(path).name}")
self.statusBar().showMessage(f"Loaded {Path(path).name}", 4000)
self.detect_all_action.setEnabled(True)
self.detect_all_button.setEnabled(True)
@@ -357,6 +359,20 @@ class MainWindow(QMainWindow):
cursor = self.source_panel.player.get_position()
self._start_detection(start_offset=cursor, max_results=1, label=f"Scanning from {format_timestamp(cursor)} for next clap…")
def _current_audio_stream_index(self) -> Optional[int]:
"""The ffmpeg/ffprobe global stream index of whichever audio track
mpv is currently playing in the Source panel. Passed through to
clap detection so it analyzes the exact track being listened to --
without this, ffmpeg's own default-stream selection could silently
pick a *different* audio track on multi-audio-track files (extra
language dubs, a separate mic/system-audio track from a screen
recorder), producing markers for the wrong track's timing entirely.
"""
for track in self.source_panel.player.mpv.track_list:
if track.get("type") == "audio" and track.get("selected"):
return track.get("ff-index")
return None
def _start_detection(self, start_offset: float, max_results, label: str):
if self.input_path is None:
return
@@ -366,7 +382,10 @@ class MainWindow(QMainWindow):
self.detect_next_button.setEnabled(False)
self.statusBar().showMessage(label)
self._detect_worker = DetectWorker(self.input_path, start_offset=start_offset, max_results=max_results)
self._detect_worker = DetectWorker(
self.input_path, start_offset=start_offset, max_results=max_results,
audio_stream_index=self._current_audio_stream_index(),
)
self._detect_worker.finished_ok.connect(self._on_detect_done)
self._detect_worker.finished_err.connect(self._on_detect_failed)
self._detect_worker.start()