Fix clap detection silently analyzing the wrong audio track
Root cause of the reported 'timestamps off by different amounts, sometimes early, sometimes late' bug: _extract_audio_wav() never passed an explicit -map to ffmpeg, so on multi-audio-track files it relied on ffmpeg's own default-stream heuristic -- which has no knowledge of, and isn't guaranteed to agree with, whichever audio track mpv is actually configured to play. A wrong-track clap is essentially unrelated to the true one (not a small timestamp-math error), which explains the inconsistent, file-dependent symptom -- and our single-audio-track synthetic fixtures never exercised this path at all, since they only ever had one track to pick. Fixed by threading an explicit ffmpeg stream index through detect_claps() -> _extract_audio_wav(), sourced in main.py from mpv's own track-list (the ff-index of whichever audio track has selected=True). Detection now always analyzes the exact track the user is listening to, on both 'Detect All' and 'Detect Next'. Also: _probe_audio_start_time() used to silently fall back to 0.0 if ffprobe couldn't parse start_time as a plain float (e.g. N/A) -- same failure class (silently wrong instead of loud). Now emits a Python warning instead, so a genuinely unparseable/unusual container is at least visible rather than silently mis-corrected. Verified two ways: (1) ruled out an initial alternate hypothesis (video stream's own start_time mattering) empirically against the existing single-track fixtures before ruling it out. (2) Built a new two-audio-track fixture (sync_test_MULTITRACK.mp4: track 1 has a beep at 3s, track 2 at 7s) and confirmed end-to-end through the real app -- switching mpv's active track from aid=1 to aid=2 correctly flips which clap gets detected, which is exactly the mechanism this bug required.
This commit is contained in:
+36
-7
@@ -5,6 +5,7 @@ paths use one implementation instead of drifting apart.
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional
|
||||
|
||||
@@ -24,10 +25,10 @@ class ClapEvent:
|
||||
strength: float
|
||||
|
||||
|
||||
def _probe_audio_start_time(input_path: str) -> float:
|
||||
def _probe_audio_start_time(input_path: str, stream_specifier: str) -> Optional[float]:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe", "-v", "error", "-select_streams", "a:0",
|
||||
"ffprobe", "-v", "error", "-select_streams", stream_specifier,
|
||||
"-show_entries", "stream=start_time",
|
||||
"-of", "default=noprint_wrappers=1:nokey=1",
|
||||
input_path,
|
||||
@@ -37,10 +38,20 @@ def _probe_audio_start_time(input_path: str) -> float:
|
||||
try:
|
||||
return float(result.stdout.strip())
|
||||
except ValueError:
|
||||
return 0.0
|
||||
# Some containers/codecs legitimately report "N/A" (genuinely no
|
||||
# start offset to speak of), but this could also mean ffprobe just
|
||||
# couldn't express a real nonzero offset for this container -- we
|
||||
# can't tell those apart here, so surface it instead of silently
|
||||
# assuming 0, which would leave onset times subtly, silently wrong.
|
||||
warnings.warn(
|
||||
f"could not determine audio start_time for {input_path!r} "
|
||||
f"(ffprobe returned {result.stdout.strip()!r}); assuming 0.0, "
|
||||
"which may be wrong for this file's container/codec."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _extract_audio_wav(input_path: str) -> str:
|
||||
def _extract_audio_wav(input_path: str, audio_stream_index: Optional[int] = None) -> str:
|
||||
# The audio stream's own start_time is exactly how a container encodes
|
||||
# "this track starts playing N seconds later than the others" -- i.e.
|
||||
# it's the very thing this whole tool exists to fix. ffmpeg's default
|
||||
@@ -48,11 +59,24 @@ def _extract_audio_wav(input_path: str) -> str:
|
||||
# which would silently make onset detection blind to that offset.
|
||||
# Compensate by inserting the missing lead-in as real silence, so times
|
||||
# measured from this WAV's t=0 still match the file's true timeline.
|
||||
audio_start = _probe_audio_start_time(input_path)
|
||||
#
|
||||
# audio_stream_index (ffprobe/ffmpeg global stream index, e.g. mpv's
|
||||
# own `ff-index` for the track it's currently playing) makes stream
|
||||
# selection explicit and -- critically -- lets the caller guarantee
|
||||
# this matches whichever audio track is actually audible in the
|
||||
# player. Without it, ffmpeg's own default-stream heuristic picks
|
||||
# *a* audio track, which isn't guaranteed to be the same one on
|
||||
# multi-audio-track files (extra language dubs, a separate mic/system
|
||||
# audio track from a screen recorder, etc.) -- something our own
|
||||
# single-track synthetic test fixtures never exercise, but real-world
|
||||
# files often have.
|
||||
stream_specifier = f"{audio_stream_index}" if audio_stream_index is not None else "a:0"
|
||||
audio_start = _probe_audio_start_time(input_path, stream_specifier) or 0.0
|
||||
|
||||
fd, wav_path = tempfile.mkstemp(suffix=".wav")
|
||||
os.close(fd)
|
||||
cmd = ["ffmpeg", "-y", "-i", input_path, "-vn"]
|
||||
cmd += ["-map", f"0:{audio_stream_index}" if audio_stream_index is not None else "a:0"]
|
||||
if audio_start > 0.001:
|
||||
delay_ms = int(round(audio_start * 1000))
|
||||
cmd += ["-af", f"adelay={delay_ms}|{delay_ms}"]
|
||||
@@ -71,11 +95,16 @@ def detect_claps(
|
||||
threshold: float = 0.7,
|
||||
min_interval: float = 0.3,
|
||||
max_results: Optional[int] = None,
|
||||
audio_stream_index: Optional[int] = None,
|
||||
) -> List[ClapEvent]:
|
||||
"""Detect sharp audio onsets in input_path. `start_offset`: only onsets
|
||||
at or after this absolute timestamp are returned (for a from-cursor
|
||||
search). `max_results` caps how many are returned, for a "find the next
|
||||
one" search instead of a full scan.
|
||||
one" search instead of a full scan. `audio_stream_index` (an ffmpeg
|
||||
global stream index) pins detection to a specific audio track on
|
||||
multi-audio-track files -- pass the ff-index of whichever track mpv is
|
||||
actually playing (see main.py) so detection can't silently analyze a
|
||||
different track than the one being listened to.
|
||||
|
||||
The whole file is always decoded and analyzed regardless of
|
||||
start_offset -- an earlier version tried to skip ahead with ffmpeg's
|
||||
@@ -92,7 +121,7 @@ def detect_claps(
|
||||
"librosa is not installed. Install with: pip install librosa numpy soundfile"
|
||||
)
|
||||
|
||||
wav_path = _extract_audio_wav(input_path)
|
||||
wav_path = _extract_audio_wav(input_path, audio_stream_index=audio_stream_index)
|
||||
try:
|
||||
y, sr = librosa.load(wav_path, sr=22050)
|
||||
onset_env = librosa.onset.onset_strength(y=y, sr=sr)
|
||||
|
||||
Reference in New Issue
Block a user