From f7c15e924cc68be8e74d838667ad234f74d6255d Mon Sep 17 00:00:00 2001 From: ThePendulum Date: Sun, 26 Jul 2026 01:46:11 +0200 Subject: [PATCH] Add clap/onset auto-detection feature Ports scripts/detect_claps.py (a librosa-based onset detector from an earlier session) into clap_detection.py, shared by both the CLI script and a new GUI feature: 'Detect All Claps' (whole-file scan) and 'Detect Next Clap' (from cursor, first result only), with results shown as clickable markers on a new MarkerStrip widget above the seek slider -- clicking one seeks the player to that exact timestamp. Caught a real correctness bug while testing against the out-of-sync fixture (not just the plain in-sync one): ffmpeg's default -vn audio extraction silently drops the audio stream's own start_time and renormalizes to t=0 -- which is exactly the offset this whole tool exists to fix, so losing it meant detected timestamps pointed at the wrong track's timing entirely. Fixed by probing the true start_time via ffprobe and re-inserting it as explicit silence (adelay) before analysis. Also dropped an earlier attempt at an ffmpeg -ss seek optimization for the from-cursor case, since combining it with the start_time fix meant two independent timestamp-origin corrections stacked on each other -- correct in theory, too easy to get subtly wrong in a tool where a silent wrong-timestamp bug is the worst possible failure mode. Full-file decode + filter in Python is slower but unambiguous. Verified with a smoke test that drives the real MainWindow buttons, confirms the detected marker lands near the fixture's known beep (t=5.350s, allowing for onset-detection lag), and simulates an actual QMouseEvent click on the marker to confirm click-to-seek works end-to-end. --- clap_detection.py | 124 +++++++++++++++++++++++++++ main.py | 107 ++++++++++++++++++++++- marker_strip.py | 68 +++++++++++++++ requirements.txt | 5 ++ scripts/detect_claps.py | 79 +++++++++++++++++ scripts/smoke_test_clap_detection.py | 86 +++++++++++++++++++ 6 files changed, 468 insertions(+), 1 deletion(-) create mode 100644 clap_detection.py create mode 100644 marker_strip.py create mode 100644 requirements.txt create mode 100755 scripts/detect_claps.py create mode 100644 scripts/smoke_test_clap_detection.py diff --git a/clap_detection.py b/clap_detection.py new file mode 100644 index 0000000..1254896 --- /dev/null +++ b/clap_detection.py @@ -0,0 +1,124 @@ +"""Sharp audio onset (clap/snap/beep) detection, shared by the GUI and +scripts/detect_claps.py's CLI. Ported from that standalone script so both +paths use one implementation instead of drifting apart. +""" +import os +import subprocess +import tempfile +from dataclasses import dataclass +from typing import List, Optional + +try: + import librosa +except ImportError: + librosa = None + + +class ClapDetectionUnavailable(RuntimeError): + pass + + +@dataclass(frozen=True) +class ClapEvent: + time_sec: float + strength: float + + +def _probe_audio_start_time(input_path: str) -> float: + result = subprocess.run( + [ + "ffprobe", "-v", "error", "-select_streams", "a:0", + "-show_entries", "stream=start_time", + "-of", "default=noprint_wrappers=1:nokey=1", + input_path, + ], + capture_output=True, text=True, + ) + try: + return float(result.stdout.strip()) + except ValueError: + return 0.0 + + +def _extract_audio_wav(input_path: str) -> str: + # The audio stream's own start_time is exactly how a container encodes + # "this track starts playing N seconds later than the others" -- i.e. + # it's the very thing this whole tool exists to fix. ffmpeg's default + # `-vn` extraction drops it and renormalizes the output to start at 0, + # which would silently make onset detection blind to that offset. + # Compensate by inserting the missing lead-in as real silence, so times + # measured from this WAV's t=0 still match the file's true timeline. + audio_start = _probe_audio_start_time(input_path) + + fd, wav_path = tempfile.mkstemp(suffix=".wav") + os.close(fd) + cmd = ["ffmpeg", "-y", "-i", input_path, "-vn"] + if audio_start > 0.001: + delay_ms = int(round(audio_start * 1000)) + cmd += ["-af", f"adelay={delay_ms}|{delay_ms}"] + cmd += ["-acodec", "pcm_s16le", "-ar", "22050", "-ac", "1", wav_path] + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + os.unlink(wav_path) + raise RuntimeError(f"ffmpeg audio extraction failed: {result.stderr}") + return wav_path + + +def detect_claps( + input_path: str, + start_offset: float = 0.0, + threshold: float = 0.7, + min_interval: float = 0.3, + max_results: Optional[int] = None, +) -> List[ClapEvent]: + """Detect sharp audio onsets in input_path. `start_offset`: only onsets + at or after this absolute timestamp are returned (for a from-cursor + search). `max_results` caps how many are returned, for a "find the next + one" search instead of a full scan. + + The whole file is always decoded and analyzed regardless of + start_offset -- an earlier version tried to skip ahead with ffmpeg's + `-ss` for a from-cursor scan, but combining that with the start_time + compensation above (both are timestamp-origin shifts on the same file) + was too easy to get subtly wrong, and getting a "which timestamp is + correct" bug wrong silently is the one thing this tool cannot afford. + Filtering the results in Python after a full decode is slower but + unambiguous. Audio-only onset detection is fast enough in practice that + this hasn't been a real cost. + """ + if librosa is None: + raise ClapDetectionUnavailable( + "librosa is not installed. Install with: pip install librosa numpy soundfile" + ) + + wav_path = _extract_audio_wav(input_path) + try: + y, sr = librosa.load(wav_path, sr=22050) + onset_env = librosa.onset.onset_strength(y=y, sr=sr) + onset_frames = librosa.onset.onset_detect( + onset_envelope=onset_env, + sr=sr, + units="frames", + backtrack=False, + delta=threshold, + wait=int(min_interval * sr / 512), + ) + onset_times = librosa.frames_to_time(onset_frames, sr=sr) + strengths = onset_env[onset_frames] + + all_events: List[ClapEvent] = [] + last_time = -min_interval + for t, s in zip(onset_times, strengths): + t = float(t) + if t - last_time < min_interval: + continue + all_events.append(ClapEvent(time_sec=t, strength=float(s))) + last_time = t + + events = [e for e in all_events if e.time_sec >= start_offset] + if max_results is not None: + events = events[:max_results] + return events + finally: + os.unlink(wav_path) diff --git a/main.py b/main.py index 994b187..172e700 100644 --- a/main.py +++ b/main.py @@ -13,6 +13,8 @@ from PyQt6.QtWidgets import ( from player_widget import PlayerWidget from sync_logic import compute_offset, SyncOffset from ffmpeg_export import export_synced, FfmpegExportError, ExportResult +from clap_detection import detect_claps, ClapDetectionUnavailable, ClapEvent +from marker_strip import MarkerStrip def format_timestamp(seconds: float) -> str: @@ -38,6 +40,28 @@ class ExportWorker(QThread): self.finished_err.emit(str(e), e.stderr) +class DetectWorker(QThread): + finished_ok = pyqtSignal(list) # List[ClapEvent] + finished_err = pyqtSignal(str) + + def __init__(self, input_path, start_offset=0.0, max_results=None): + super().__init__() + self.input_path = input_path + self.start_offset = start_offset + self.max_results = max_results + + def run(self): + try: + events = detect_claps( + self.input_path, + start_offset=self.start_offset, + max_results=self.max_results, + ) + self.finished_ok.emit(events) + except (ClapDetectionUnavailable, RuntimeError) as e: + self.finished_err.emit(str(e)) + + class MainWindow(QMainWindow): def __init__(self): super().__init__() @@ -49,6 +73,7 @@ class MainWindow(QMainWindow): self.audio_mark: Optional[float] = None self.offset: Optional[SyncOffset] = None self._export_worker: Optional[ExportWorker] = None + self._detect_worker: Optional[DetectWorker] = None self._slider_being_dragged = False self.player = PlayerWidget() @@ -99,16 +124,42 @@ class MainWindow(QMainWindow): self.position_label = QLabel("00:00.000") seek_row.addWidget(self.position_label) + slider_stack = QVBoxLayout() + slider_stack.setSpacing(0) + self.marker_strip = MarkerStrip() + self.marker_strip.markerClicked.connect(self._on_marker_clicked) + slider_stack.addWidget(self.marker_strip) + self.seek_slider = QSlider(Qt.Orientation.Horizontal) self.seek_slider.setRange(0, 0) self.seek_slider.sliderPressed.connect(lambda: setattr(self, "_slider_being_dragged", True)) self.seek_slider.sliderReleased.connect(self._on_slider_released) - seek_row.addWidget(self.seek_slider, stretch=1) + slider_stack.addWidget(self.seek_slider) + seek_row.addLayout(slider_stack, stretch=1) self.duration_label = QLabel("00:00.000") seek_row.addWidget(self.duration_label) layout.addLayout(seek_row) + # Clap/onset auto-detect row + detect_group = QGroupBox("Clap detection") + detect_layout = QHBoxLayout(detect_group) + + self.detect_all_button = QPushButton("Detect All Claps (whole file)") + self.detect_all_button.setEnabled(False) + self.detect_all_button.clicked.connect(self._on_detect_all_clicked) + detect_layout.addWidget(self.detect_all_button) + + self.detect_next_button = QPushButton("Detect Next Clap (from cursor)") + self.detect_next_button.setEnabled(False) + self.detect_next_button.clicked.connect(self._on_detect_next_clicked) + detect_layout.addWidget(self.detect_next_button) + + self.detect_status_label = QLabel("") + detect_layout.addWidget(self.detect_status_label, stretch=1) + + layout.addWidget(detect_group) + # Marker row marker_group = QGroupBox("Sync markers") marker_layout = QHBoxLayout(marker_group) @@ -193,6 +244,12 @@ class MainWindow(QMainWindow): self.duration_label.setText(format_timestamp(duration)) self.setWindowTitle(f"A/V Sync Marker — {Path(path).name}") + self.marker_strip.set_duration(duration) + self.marker_strip.clear() + self.detect_status_label.setText("") + self.detect_all_button.setEnabled(True) + self.detect_next_button.setEnabled(True) + def _toggle_play(self): if self.player.mpv.pause: self.player.play() @@ -247,6 +304,54 @@ class MainWindow(QMainWindow): self.override_spinbox.setEnabled(True) self.export_button.setEnabled(True) + # -- clap detection ----------------------------------------------------- + + def _on_marker_clicked(self, time_sec: float): + self.player.seek(time_sec) + self._update_position_ui() + + def _on_detect_all_clicked(self): + self._start_detection(start_offset=0.0, max_results=None, label="Scanning whole file for claps…") + + def _on_detect_next_clicked(self): + cursor = self.player.get_position() + self._start_detection(start_offset=cursor, max_results=1, label=f"Scanning from {format_timestamp(cursor)} for next clap…") + + def _start_detection(self, start_offset: float, max_results, label: str): + if self.input_path is None: + return + self.detect_all_button.setEnabled(False) + self.detect_next_button.setEnabled(False) + self.detect_status_label.setText(label) + + self._detect_worker = DetectWorker(self.input_path, start_offset=start_offset, max_results=max_results) + self._detect_worker.finished_ok.connect(self._on_detect_done) + self._detect_worker.finished_err.connect(self._on_detect_failed) + self._detect_worker.start() + + def _on_detect_done(self, events: list): + self.detect_all_button.setEnabled(True) + self.detect_next_button.setEnabled(True) + + existing = set(self.marker_strip.markers()) + existing.update(e.time_sec for e in events) + self.marker_strip.set_markers(list(existing)) + + if not events: + self.detect_status_label.setText("No claps found.") + self.log_view.appendPlainText("Clap detection: no onsets found.") + return + + self.detect_status_label.setText(f"Found {len(events)} clap(s) — click a marker above the timeline to jump to it.") + for e in events: + self.log_view.appendPlainText(f"Clap detected at {format_timestamp(e.time_sec)} (strength {e.strength:.2f})") + + def _on_detect_failed(self, message: str): + self.detect_all_button.setEnabled(True) + self.detect_next_button.setEnabled(True) + self.detect_status_label.setText("Detection failed (see log).") + self.log_view.appendPlainText(f"Clap detection FAILED: {message}") + # -- export ----------------------------------------------------- def _on_export_clicked(self): diff --git a/marker_strip.py b/marker_strip.py new file mode 100644 index 0000000..d407eca --- /dev/null +++ b/marker_strip.py @@ -0,0 +1,68 @@ +from typing import List + +from PyQt6.QtCore import Qt, QPointF, pyqtSignal +from PyQt6.QtGui import QPainter, QColor, QPolygonF +from PyQt6.QtWidgets import QWidget + +MARKER_COLOR = QColor(230, 126, 34) # orange +CLICK_TOLERANCE_PX = 8.0 + + +class MarkerStrip(QWidget): + """Thin strip drawn directly above the seek slider (same width, via a + shared parent layout) showing detected clap/onset markers as small + triangles. Clicking one seeks the player to that exact timestamp. + """ + + markerClicked = pyqtSignal(float) + + def __init__(self, parent=None): + super().__init__(parent) + self.setFixedHeight(14) + self.setCursor(Qt.CursorShape.PointingHandCursor) + self._duration = 0.0 + self._markers: List[float] = [] + + def set_duration(self, duration: float): + self._duration = duration + self.update() + + def set_markers(self, markers: List[float]): + self._markers = sorted(markers) + self.update() + + def markers(self) -> List[float]: + return list(self._markers) + + def clear(self): + self._markers = [] + self.update() + + def _x_for_time(self, t: float) -> float: + if self._duration <= 0: + return 0.0 + return (t / self._duration) * self.width() + + def paintEvent(self, event): + if not self._markers or self._duration <= 0: + return + painter = QPainter(self) + painter.setRenderHint(QPainter.RenderHint.Antialiasing) + painter.setPen(Qt.PenStyle.NoPen) + painter.setBrush(MARKER_COLOR) + for t in self._markers: + x = self._x_for_time(t) + triangle = QPolygonF([ + QPointF(x - 5, 0), + QPointF(x + 5, 0), + QPointF(x, 10), + ]) + painter.drawPolygon(triangle) + + def mousePressEvent(self, event): + if not self._markers or self._duration <= 0: + return + click_x = event.position().x() + closest = min(self._markers, key=lambda t: abs(self._x_for_time(t) - click_x)) + if abs(self._x_for_time(closest) - click_x) <= CLICK_TOLERANCE_PX: + self.markerClicked.emit(closest) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..12f1802 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +PyQt6==6.11.0 +python-mpv==1.0.8 +librosa==0.11.0 +numpy==2.4.6 +soundfile==0.14.0 diff --git a/scripts/detect_claps.py b/scripts/detect_claps.py new file mode 100755 index 0000000..50dc8a9 --- /dev/null +++ b/scripts/detect_claps.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +Detect sharp audio onsets (claps, snaps) in audio/video files. +Outputs timestamps in hh:mm:ss format. +""" + +import sys +import argparse +import os +from datetime import timedelta +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from clap_detection import detect_claps, ClapDetectionUnavailable + + +def format_timestamp(seconds): + """Convert seconds to hh:mm:ss.mmm format.""" + td = timedelta(seconds=seconds) + total_seconds = int(td.total_seconds()) + hours = total_seconds // 3600 + minutes = (total_seconds % 3600) // 60 + secs = td.total_seconds() % 60 + return f"{hours:02d}:{minutes:02d}:{secs:06.3f}" + + +def main(): + parser = argparse.ArgumentParser( + description='Detect claps/snaps in audio/video files', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s audio.wav + %(prog)s video.mp4 -t 0.7 -m 1.0 + %(prog)s audio.wav -t 0.3 -s # Show onset strengths + +Threshold guide: + 0.3 = Very sensitive (detects most transients) + 0.5 = Moderate (typical claps/snaps) + 0.7 = Less sensitive (only loud, sharp claps) + 0.9 = Very strict (only very loud events) + """ + ) + + parser.add_argument('input_file', help='Audio or video file') + parser.add_argument('-t', '--threshold', type=float, default=0.7, + help='Detection threshold (0-1, default: 0.7, higher = less sensitive)') + parser.add_argument('-m', '--min-interval', type=float, default=0.3, + help='Minimum seconds between detections (default: 0.3)') + parser.add_argument('-s', '--show-strength', action='store_true', + help='Show onset strength values') + + args = parser.parse_args() + + if not os.path.exists(args.input_file): + print(f"Error: File not found: {args.input_file}", file=sys.stderr) + sys.exit(1) + + try: + events = detect_claps( + args.input_file, + threshold=args.threshold, + min_interval=args.min_interval, + ) + except ClapDetectionUnavailable as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + print(f"\nFound {len(events)} onset(s):\n", file=sys.stderr) + for event in events: + timestamp = format_timestamp(event.time_sec) + if args.show_strength: + print(f"{timestamp} (strength: {event.strength:.2f})") + else: + print(timestamp) + + +if __name__ == '__main__': + main() diff --git a/scripts/smoke_test_clap_detection.py b/scripts/smoke_test_clap_detection.py new file mode 100644 index 0000000..baa508a --- /dev/null +++ b/scripts/smoke_test_clap_detection.py @@ -0,0 +1,86 @@ +"""Drives the real MainWindow's clap-detection buttons (not a copy of the +logic) against the synthetic fixture with a known beep at t=5.350s, then +simulates a real mouse click on the MarkerStrip widget to confirm +click-to-seek actually moves the player -- not just that a signal exists. +""" +import sys +import time +from pathlib import Path + +from PyQt6.QtCore import Qt, QPointF +from PyQt6.QtGui import QMouseEvent +from PyQt6.QtWidgets import QApplication + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import main as app_main + +ROOT = Path(__file__).resolve().parent.parent +CLIP = str(ROOT / "test_media" / "sync_test_OUTOFSYNC_audio+350ms.mp4") +OUT_DIR = ROOT / "test_media" / "seek_validation" + + +def wait_for(predicate, qapp, timeout=20): + deadline = time.time() + timeout + while not predicate() and time.time() < deadline: + qapp.processEvents() + time.sleep(0.05) + return predicate() + + +def main(): + OUT_DIR.mkdir(exist_ok=True) + qapp = QApplication(sys.argv) + win = app_main.MainWindow() + win.show() + qapp.processEvents() + + win._load(CLIP) + qapp.processEvents() + time.sleep(0.3) + qapp.processEvents() + + assert win.detect_all_button.isEnabled() + win.detect_all_button.click() + qapp.processEvents() + assert not win.detect_all_button.isEnabled(), "button should disable while detection runs" + + ok = wait_for(lambda: win.detect_all_button.isEnabled(), qapp) + assert ok, "detection did not finish in time" + + markers = win.marker_strip.markers() + print(f"detected markers: {markers}") + assert len(markers) >= 1, "expected at least one clap marker" + # ground truth beep is at t=5.350s in this fixture; onset detection has + # some inherent lag, so allow a generous window rather than exact match. + assert any(abs(m - 5.350) < 0.5 for m in markers), f"no marker near the known beep: {markers}" + + win.grab().save(str(OUT_DIR / "clap_markers_on_timeline.png")) + + # Simulate a real mouse click on the marker to prove click-to-seek works + # end to end, not just that the signal is wired. + target_time = min(markers, key=lambda m: abs(m - 5.350)) + strip = win.marker_strip + strip.resize(win.seek_slider.width(), strip.height()) + x = strip._x_for_time(target_time) + click_pos = QPointF(x, 5) + + before_pos = win.player.get_position() + event = QMouseEvent( + QMouseEvent.Type.MouseButtonPress, click_pos, click_pos, + Qt.MouseButton.LeftButton, Qt.MouseButton.LeftButton, Qt.KeyboardModifier.NoModifier, + ) + strip.mousePressEvent(event) + qapp.processEvents() + time.sleep(0.2) + qapp.processEvents() + + after_pos = win.player.get_position() + print(f"clicked marker at x={x:.1f} (t={target_time:.3f}s); player position before={before_pos:.3f}s after={after_pos:.3f}s") + assert abs(after_pos - target_time) < 0.1, "clicking the marker should seek the player to that timestamp" + + win.player.shutdown() + print("\nCLAP DETECTION SMOKE TEST PASSED") + + +if __name__ == "__main__": + main()