Add clap/onset auto-detection feature
Ports scripts/detect_claps.py (a librosa-based onset detector from an earlier session) into clap_detection.py, shared by both the CLI script and a new GUI feature: 'Detect All Claps' (whole-file scan) and 'Detect Next Clap' (from cursor, first result only), with results shown as clickable markers on a new MarkerStrip widget above the seek slider -- clicking one seeks the player to that exact timestamp. Caught a real correctness bug while testing against the out-of-sync fixture (not just the plain in-sync one): ffmpeg's default -vn audio extraction silently drops the audio stream's own start_time and renormalizes to t=0 -- which is exactly the offset this whole tool exists to fix, so losing it meant detected timestamps pointed at the wrong track's timing entirely. Fixed by probing the true start_time via ffprobe and re-inserting it as explicit silence (adelay) before analysis. Also dropped an earlier attempt at an ffmpeg -ss seek optimization for the from-cursor case, since combining it with the start_time fix meant two independent timestamp-origin corrections stacked on each other -- correct in theory, too easy to get subtly wrong in a tool where a silent wrong-timestamp bug is the worst possible failure mode. Full-file decode + filter in Python is slower but unambiguous. Verified with a smoke test that drives the real MainWindow buttons, confirms the detected marker lands near the fixture's known beep (t=5.350s, allowing for onset-detection lag), and simulates an actual QMouseEvent click on the marker to confirm click-to-seek works end-to-end.
This commit is contained in:
@@ -0,0 +1,124 @@
|
||||
"""Sharp audio onset (clap/snap/beep) detection, shared by the GUI and
|
||||
scripts/detect_claps.py's CLI. Ported from that standalone script so both
|
||||
paths use one implementation instead of drifting apart.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional
|
||||
|
||||
try:
|
||||
import librosa
|
||||
except ImportError:
|
||||
librosa = None
|
||||
|
||||
|
||||
class ClapDetectionUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ClapEvent:
|
||||
time_sec: float
|
||||
strength: float
|
||||
|
||||
|
||||
def _probe_audio_start_time(input_path: str) -> float:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe", "-v", "error", "-select_streams", "a:0",
|
||||
"-show_entries", "stream=start_time",
|
||||
"-of", "default=noprint_wrappers=1:nokey=1",
|
||||
input_path,
|
||||
],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
try:
|
||||
return float(result.stdout.strip())
|
||||
except ValueError:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _extract_audio_wav(input_path: str) -> str:
|
||||
# The audio stream's own start_time is exactly how a container encodes
|
||||
# "this track starts playing N seconds later than the others" -- i.e.
|
||||
# it's the very thing this whole tool exists to fix. ffmpeg's default
|
||||
# `-vn` extraction drops it and renormalizes the output to start at 0,
|
||||
# which would silently make onset detection blind to that offset.
|
||||
# Compensate by inserting the missing lead-in as real silence, so times
|
||||
# measured from this WAV's t=0 still match the file's true timeline.
|
||||
audio_start = _probe_audio_start_time(input_path)
|
||||
|
||||
fd, wav_path = tempfile.mkstemp(suffix=".wav")
|
||||
os.close(fd)
|
||||
cmd = ["ffmpeg", "-y", "-i", input_path, "-vn"]
|
||||
if audio_start > 0.001:
|
||||
delay_ms = int(round(audio_start * 1000))
|
||||
cmd += ["-af", f"adelay={delay_ms}|{delay_ms}"]
|
||||
cmd += ["-acodec", "pcm_s16le", "-ar", "22050", "-ac", "1", wav_path]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
os.unlink(wav_path)
|
||||
raise RuntimeError(f"ffmpeg audio extraction failed: {result.stderr}")
|
||||
return wav_path
|
||||
|
||||
|
||||
def detect_claps(
|
||||
input_path: str,
|
||||
start_offset: float = 0.0,
|
||||
threshold: float = 0.7,
|
||||
min_interval: float = 0.3,
|
||||
max_results: Optional[int] = None,
|
||||
) -> List[ClapEvent]:
|
||||
"""Detect sharp audio onsets in input_path. `start_offset`: only onsets
|
||||
at or after this absolute timestamp are returned (for a from-cursor
|
||||
search). `max_results` caps how many are returned, for a "find the next
|
||||
one" search instead of a full scan.
|
||||
|
||||
The whole file is always decoded and analyzed regardless of
|
||||
start_offset -- an earlier version tried to skip ahead with ffmpeg's
|
||||
`-ss` for a from-cursor scan, but combining that with the start_time
|
||||
compensation above (both are timestamp-origin shifts on the same file)
|
||||
was too easy to get subtly wrong, and getting a "which timestamp is
|
||||
correct" bug wrong silently is the one thing this tool cannot afford.
|
||||
Filtering the results in Python after a full decode is slower but
|
||||
unambiguous. Audio-only onset detection is fast enough in practice that
|
||||
this hasn't been a real cost.
|
||||
"""
|
||||
if librosa is None:
|
||||
raise ClapDetectionUnavailable(
|
||||
"librosa is not installed. Install with: pip install librosa numpy soundfile"
|
||||
)
|
||||
|
||||
wav_path = _extract_audio_wav(input_path)
|
||||
try:
|
||||
y, sr = librosa.load(wav_path, sr=22050)
|
||||
onset_env = librosa.onset.onset_strength(y=y, sr=sr)
|
||||
onset_frames = librosa.onset.onset_detect(
|
||||
onset_envelope=onset_env,
|
||||
sr=sr,
|
||||
units="frames",
|
||||
backtrack=False,
|
||||
delta=threshold,
|
||||
wait=int(min_interval * sr / 512),
|
||||
)
|
||||
onset_times = librosa.frames_to_time(onset_frames, sr=sr)
|
||||
strengths = onset_env[onset_frames]
|
||||
|
||||
all_events: List[ClapEvent] = []
|
||||
last_time = -min_interval
|
||||
for t, s in zip(onset_times, strengths):
|
||||
t = float(t)
|
||||
if t - last_time < min_interval:
|
||||
continue
|
||||
all_events.append(ClapEvent(time_sec=t, strength=float(s)))
|
||||
last_time = t
|
||||
|
||||
events = [e for e in all_events if e.time_sec >= start_offset]
|
||||
if max_results is not None:
|
||||
events = events[:max_results]
|
||||
return events
|
||||
finally:
|
||||
os.unlink(wav_path)
|
||||
Reference in New Issue
Block a user