Add clap/onset auto-detection feature

Ports scripts/detect_claps.py (a librosa-based onset detector from an
earlier session) into clap_detection.py, shared by both the CLI script
and a new GUI feature: 'Detect All Claps' (whole-file scan) and
'Detect Next Clap' (from cursor, first result only), with results
shown as clickable markers on a new MarkerStrip widget above the seek
slider -- clicking one seeks the player to that exact timestamp.

Caught a real correctness bug while testing against the out-of-sync
fixture (not just the plain in-sync one): ffmpeg's default -vn audio
extraction silently drops the audio stream's own start_time and
renormalizes to t=0 -- which is exactly the offset this whole tool
exists to fix, so losing it meant detected timestamps pointed at the
wrong track's timing entirely. Fixed by probing the true start_time
via ffprobe and re-inserting it as explicit silence (adelay) before
analysis. Also dropped an earlier attempt at an ffmpeg -ss seek
optimization for the from-cursor case, since combining it with the
start_time fix meant two independent timestamp-origin corrections
stacked on each other -- correct in theory, too easy to get subtly
wrong in a tool where a silent wrong-timestamp bug is the worst
possible failure mode. Full-file decode + filter in Python is slower
but unambiguous.

Verified with a smoke test that drives the real MainWindow buttons,
confirms the detected marker lands near the fixture's known beep
(t=5.350s, allowing for onset-detection lag), and simulates an actual
QMouseEvent click on the marker to confirm click-to-seek works
end-to-end.

clap_detection.py

@@ -0,0 +1,124 @@
+"""Sharp audio onset (clap/snap/beep) detection, shared by the GUI and
+scripts/detect_claps.py's CLI. Ported from that standalone script so both
+paths use one implementation instead of drifting apart.
+"""
+import os
+import subprocess
+import tempfile
+from dataclasses import dataclass
+from typing import List, Optional
+
+try:
+    import librosa
+except ImportError:
+    librosa = None
+
+
+class ClapDetectionUnavailable(RuntimeError):
+    pass
+
+
+@dataclass(frozen=True)
+class ClapEvent:
+    time_sec: float
+    strength: float
+
+
+def _probe_audio_start_time(input_path: str) -> float:
+    result = subprocess.run(
+        [
+            "ffprobe", "-v", "error", "-select_streams", "a:0",
+            "-show_entries", "stream=start_time",
+            "-of", "default=noprint_wrappers=1:nokey=1",
+            input_path,
+        ],
+        capture_output=True, text=True,
+    )
+    try:
+        return float(result.stdout.strip())
+    except ValueError:
+        return 0.0
+
+
+def _extract_audio_wav(input_path: str) -> str:
+    # The audio stream's own start_time is exactly how a container encodes
+    # "this track starts playing N seconds later than the others" -- i.e.
+    # it's the very thing this whole tool exists to fix. ffmpeg's default
+    # `-vn` extraction drops it and renormalizes the output to start at 0,
+    # which would silently make onset detection blind to that offset.
+    # Compensate by inserting the missing lead-in as real silence, so times
+    # measured from this WAV's t=0 still match the file's true timeline.
+    audio_start = _probe_audio_start_time(input_path)
+
+    fd, wav_path = tempfile.mkstemp(suffix=".wav")
+    os.close(fd)
+    cmd = ["ffmpeg", "-y", "-i", input_path, "-vn"]
+    if audio_start > 0.001:
+        delay_ms = int(round(audio_start * 1000))
+        cmd += ["-af", f"adelay={delay_ms}|{delay_ms}"]
+    cmd += ["-acodec", "pcm_s16le", "-ar", "22050", "-ac", "1", wav_path]
+
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.returncode != 0:
+        os.unlink(wav_path)
+        raise RuntimeError(f"ffmpeg audio extraction failed: {result.stderr}")
+    return wav_path
+
+
+def detect_claps(
+    input_path: str,
+    start_offset: float = 0.0,
+    threshold: float = 0.7,
+    min_interval: float = 0.3,
+    max_results: Optional[int] = None,
+) -> List[ClapEvent]:
+    """Detect sharp audio onsets in input_path. `start_offset`: only onsets
+    at or after this absolute timestamp are returned (for a from-cursor
+    search). `max_results` caps how many are returned, for a "find the next
+    one" search instead of a full scan.
+
+    The whole file is always decoded and analyzed regardless of
+    start_offset -- an earlier version tried to skip ahead with ffmpeg's
+    `-ss` for a from-cursor scan, but combining that with the start_time
+    compensation above (both are timestamp-origin shifts on the same file)
+    was too easy to get subtly wrong, and getting a "which timestamp is
+    correct" bug wrong silently is the one thing this tool cannot afford.
+    Filtering the results in Python after a full decode is slower but
+    unambiguous. Audio-only onset detection is fast enough in practice that
+    this hasn't been a real cost.
+    """
+    if librosa is None:
+        raise ClapDetectionUnavailable(
+            "librosa is not installed. Install with: pip install librosa numpy soundfile"
+        )
+
+    wav_path = _extract_audio_wav(input_path)
+    try:
+        y, sr = librosa.load(wav_path, sr=22050)
+        onset_env = librosa.onset.onset_strength(y=y, sr=sr)
+        onset_frames = librosa.onset.onset_detect(
+            onset_envelope=onset_env,
+            sr=sr,
+            units="frames",
+            backtrack=False,
+            delta=threshold,
+            wait=int(min_interval * sr / 512),
+        )
+        onset_times = librosa.frames_to_time(onset_frames, sr=sr)
+        strengths = onset_env[onset_frames]
+
+        all_events: List[ClapEvent] = []
+        last_time = -min_interval
+        for t, s in zip(onset_times, strengths):
+            t = float(t)
+            if t - last_time < min_interval:
+                continue
+            all_events.append(ClapEvent(time_sec=t, strength=float(s)))
+            last_time = t
+
+        events = [e for e in all_events if e.time_sec >= start_offset]
+        if max_results is not None:
+            events = events[:max_results]
+        return events
+    finally:
+        os.unlink(wav_path)

main.py

@@ -13,6 +13,8 @@ from PyQt6.QtWidgets import (
 from player_widget import PlayerWidget
 from sync_logic import compute_offset, SyncOffset
 from ffmpeg_export import export_synced, FfmpegExportError, ExportResult
+from clap_detection import detect_claps, ClapDetectionUnavailable, ClapEvent
+from marker_strip import MarkerStrip
 
 
 def format_timestamp(seconds: float) -> str:
@@ -38,6 +40,28 @@ class ExportWorker(QThread):
             self.finished_err.emit(str(e), e.stderr)
 
 
+class DetectWorker(QThread):
+    finished_ok = pyqtSignal(list)   # List[ClapEvent]
+    finished_err = pyqtSignal(str)
+
+    def __init__(self, input_path, start_offset=0.0, max_results=None):
+        super().__init__()
+        self.input_path = input_path
+        self.start_offset = start_offset
+        self.max_results = max_results
+
+    def run(self):
+        try:
+            events = detect_claps(
+                self.input_path,
+                start_offset=self.start_offset,
+                max_results=self.max_results,
+            )
+            self.finished_ok.emit(events)
+        except (ClapDetectionUnavailable, RuntimeError) as e:
+            self.finished_err.emit(str(e))
+
+
 class MainWindow(QMainWindow):
     def __init__(self):
         super().__init__()
@@ -49,6 +73,7 @@ class MainWindow(QMainWindow):
         self.audio_mark: Optional[float] = None
         self.offset: Optional[SyncOffset] = None
         self._export_worker: Optional[ExportWorker] = None
+        self._detect_worker: Optional[DetectWorker] = None
         self._slider_being_dragged = False
 
         self.player = PlayerWidget()
@@ -99,16 +124,42 @@ class MainWindow(QMainWindow):
         self.position_label = QLabel("00:00.000")
         seek_row.addWidget(self.position_label)
 
+        slider_stack = QVBoxLayout()
+        slider_stack.setSpacing(0)
+        self.marker_strip = MarkerStrip()
+        self.marker_strip.markerClicked.connect(self._on_marker_clicked)
+        slider_stack.addWidget(self.marker_strip)
+
         self.seek_slider = QSlider(Qt.Orientation.Horizontal)
         self.seek_slider.setRange(0, 0)
         self.seek_slider.sliderPressed.connect(lambda: setattr(self, "_slider_being_dragged", True))
         self.seek_slider.sliderReleased.connect(self._on_slider_released)
-        seek_row.addWidget(self.seek_slider, stretch=1)
+        slider_stack.addWidget(self.seek_slider)
+        seek_row.addLayout(slider_stack, stretch=1)
 
         self.duration_label = QLabel("00:00.000")
         seek_row.addWidget(self.duration_label)
         layout.addLayout(seek_row)
 
+        # Clap/onset auto-detect row
+        detect_group = QGroupBox("Clap detection")
+        detect_layout = QHBoxLayout(detect_group)
+
+        self.detect_all_button = QPushButton("Detect All Claps (whole file)")
+        self.detect_all_button.setEnabled(False)
+        self.detect_all_button.clicked.connect(self._on_detect_all_clicked)
+        detect_layout.addWidget(self.detect_all_button)
+
+        self.detect_next_button = QPushButton("Detect Next Clap (from cursor)")
+        self.detect_next_button.setEnabled(False)
+        self.detect_next_button.clicked.connect(self._on_detect_next_clicked)
+        detect_layout.addWidget(self.detect_next_button)
+
+        self.detect_status_label = QLabel("")
+        detect_layout.addWidget(self.detect_status_label, stretch=1)
+
+        layout.addWidget(detect_group)
+
         # Marker row
         marker_group = QGroupBox("Sync markers")
         marker_layout = QHBoxLayout(marker_group)
@@ -193,6 +244,12 @@ class MainWindow(QMainWindow):
         self.duration_label.setText(format_timestamp(duration))
         self.setWindowTitle(f"A/V Sync Marker — {Path(path).name}")
 
+        self.marker_strip.set_duration(duration)
+        self.marker_strip.clear()
+        self.detect_status_label.setText("")
+        self.detect_all_button.setEnabled(True)
+        self.detect_next_button.setEnabled(True)
+
     def _toggle_play(self):
         if self.player.mpv.pause:
             self.player.play()
@@ -247,6 +304,54 @@ class MainWindow(QMainWindow):
         self.override_spinbox.setEnabled(True)
         self.export_button.setEnabled(True)
 
+    # -- clap detection -----------------------------------------------------
+
+    def _on_marker_clicked(self, time_sec: float):
+        self.player.seek(time_sec)
+        self._update_position_ui()
+
+    def _on_detect_all_clicked(self):
+        self._start_detection(start_offset=0.0, max_results=None, label="Scanning whole file for claps…")
+
+    def _on_detect_next_clicked(self):
+        cursor = self.player.get_position()
+        self._start_detection(start_offset=cursor, max_results=1, label=f"Scanning from {format_timestamp(cursor)} for next clap…")
+
+    def _start_detection(self, start_offset: float, max_results, label: str):
+        if self.input_path is None:
+            return
+        self.detect_all_button.setEnabled(False)
+        self.detect_next_button.setEnabled(False)
+        self.detect_status_label.setText(label)
+
+        self._detect_worker = DetectWorker(self.input_path, start_offset=start_offset, max_results=max_results)
+        self._detect_worker.finished_ok.connect(self._on_detect_done)
+        self._detect_worker.finished_err.connect(self._on_detect_failed)
+        self._detect_worker.start()
+
+    def _on_detect_done(self, events: list):
+        self.detect_all_button.setEnabled(True)
+        self.detect_next_button.setEnabled(True)
+
+        existing = set(self.marker_strip.markers())
+        existing.update(e.time_sec for e in events)
+        self.marker_strip.set_markers(list(existing))
+
+        if not events:
+            self.detect_status_label.setText("No claps found.")
+            self.log_view.appendPlainText("Clap detection: no onsets found.")
+            return
+
+        self.detect_status_label.setText(f"Found {len(events)} clap(s) — click a marker above the timeline to jump to it.")
+        for e in events:
+            self.log_view.appendPlainText(f"Clap detected at {format_timestamp(e.time_sec)} (strength {e.strength:.2f})")
+
+    def _on_detect_failed(self, message: str):
+        self.detect_all_button.setEnabled(True)
+        self.detect_next_button.setEnabled(True)
+        self.detect_status_label.setText("Detection failed (see log).")
+        self.log_view.appendPlainText(f"Clap detection FAILED: {message}")
+
     # -- export -----------------------------------------------------
 
     def _on_export_clicked(self):

marker_strip.py

@@ -0,0 +1,68 @@
+from typing import List
+
+from PyQt6.QtCore import Qt, QPointF, pyqtSignal
+from PyQt6.QtGui import QPainter, QColor, QPolygonF
+from PyQt6.QtWidgets import QWidget
+
+MARKER_COLOR = QColor(230, 126, 34)  # orange
+CLICK_TOLERANCE_PX = 8.0
+
+
+class MarkerStrip(QWidget):
+    """Thin strip drawn directly above the seek slider (same width, via a
+    shared parent layout) showing detected clap/onset markers as small
+    triangles. Clicking one seeks the player to that exact timestamp.
+    """
+
+    markerClicked = pyqtSignal(float)
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self.setFixedHeight(14)
+        self.setCursor(Qt.CursorShape.PointingHandCursor)
+        self._duration = 0.0
+        self._markers: List[float] = []
+
+    def set_duration(self, duration: float):
+        self._duration = duration
+        self.update()
+
+    def set_markers(self, markers: List[float]):
+        self._markers = sorted(markers)
+        self.update()
+
+    def markers(self) -> List[float]:
+        return list(self._markers)
+
+    def clear(self):
+        self._markers = []
+        self.update()
+
+    def _x_for_time(self, t: float) -> float:
+        if self._duration <= 0:
+            return 0.0
+        return (t / self._duration) * self.width()
+
+    def paintEvent(self, event):
+        if not self._markers or self._duration <= 0:
+            return
+        painter = QPainter(self)
+        painter.setRenderHint(QPainter.RenderHint.Antialiasing)
+        painter.setPen(Qt.PenStyle.NoPen)
+        painter.setBrush(MARKER_COLOR)
+        for t in self._markers:
+            x = self._x_for_time(t)
+            triangle = QPolygonF([
+                QPointF(x - 5, 0),
+                QPointF(x + 5, 0),
+                QPointF(x, 10),
+            ])
+            painter.drawPolygon(triangle)
+
+    def mousePressEvent(self, event):
+        if not self._markers or self._duration <= 0:
+            return
+        click_x = event.position().x()
+        closest = min(self._markers, key=lambda t: abs(self._x_for_time(t) - click_x))
+        if abs(self._x_for_time(closest) - click_x) <= CLICK_TOLERANCE_PX:
+            self.markerClicked.emit(closest)

requirements.txt

@@ -0,0 +1,5 @@
+PyQt6==6.11.0
+python-mpv==1.0.8
+librosa==0.11.0
+numpy==2.4.6
+soundfile==0.14.0

scripts/detect_claps.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""
+Detect sharp audio onsets (claps, snaps) in audio/video files.
+Outputs timestamps in hh:mm:ss format.
+"""
+
+import sys
+import argparse
+import os
+from datetime import timedelta
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from clap_detection import detect_claps, ClapDetectionUnavailable
+
+
+def format_timestamp(seconds):
+    """Convert seconds to hh:mm:ss.mmm format."""
+    td = timedelta(seconds=seconds)
+    total_seconds = int(td.total_seconds())
+    hours = total_seconds // 3600
+    minutes = (total_seconds % 3600) // 60
+    secs = td.total_seconds() % 60
+    return f"{hours:02d}:{minutes:02d}:{secs:06.3f}"
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Detect claps/snaps in audio/video files',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  %(prog)s audio.wav
+  %(prog)s video.mp4 -t 0.7 -m 1.0
+  %(prog)s audio.wav -t 0.3 -s  # Show onset strengths
+
+Threshold guide:
+  0.3 = Very sensitive (detects most transients)
+  0.5 = Moderate (typical claps/snaps)
+  0.7 = Less sensitive (only loud, sharp claps)
+  0.9 = Very strict (only very loud events)
+        """
+    )
+
+    parser.add_argument('input_file', help='Audio or video file')
+    parser.add_argument('-t', '--threshold', type=float, default=0.7,
+                        help='Detection threshold (0-1, default: 0.7, higher = less sensitive)')
+    parser.add_argument('-m', '--min-interval', type=float, default=0.3,
+                        help='Minimum seconds between detections (default: 0.3)')
+    parser.add_argument('-s', '--show-strength', action='store_true',
+                        help='Show onset strength values')
+
+    args = parser.parse_args()
+
+    if not os.path.exists(args.input_file):
+        print(f"Error: File not found: {args.input_file}", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        events = detect_claps(
+            args.input_file,
+            threshold=args.threshold,
+            min_interval=args.min_interval,
+        )
+    except ClapDetectionUnavailable as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"\nFound {len(events)} onset(s):\n", file=sys.stderr)
+    for event in events:
+        timestamp = format_timestamp(event.time_sec)
+        if args.show_strength:
+            print(f"{timestamp}  (strength: {event.strength:.2f})")
+        else:
+            print(timestamp)
+
+
+if __name__ == '__main__':
+    main()

scripts/smoke_test_clap_detection.py

@@ -0,0 +1,86 @@
+"""Drives the real MainWindow's clap-detection buttons (not a copy of the
+logic) against the synthetic fixture with a known beep at t=5.350s, then
+simulates a real mouse click on the MarkerStrip widget to confirm
+click-to-seek actually moves the player -- not just that a signal exists.
+"""
+import sys
+import time
+from pathlib import Path
+
+from PyQt6.QtCore import Qt, QPointF
+from PyQt6.QtGui import QMouseEvent
+from PyQt6.QtWidgets import QApplication
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+import main as app_main
+
+ROOT = Path(__file__).resolve().parent.parent
+CLIP = str(ROOT / "test_media" / "sync_test_OUTOFSYNC_audio+350ms.mp4")
+OUT_DIR = ROOT / "test_media" / "seek_validation"
+
+
+def wait_for(predicate, qapp, timeout=20):
+    deadline = time.time() + timeout
+    while not predicate() and time.time() < deadline:
+        qapp.processEvents()
+        time.sleep(0.05)
+    return predicate()
+
+
+def main():
+    OUT_DIR.mkdir(exist_ok=True)
+    qapp = QApplication(sys.argv)
+    win = app_main.MainWindow()
+    win.show()
+    qapp.processEvents()
+
+    win._load(CLIP)
+    qapp.processEvents()
+    time.sleep(0.3)
+    qapp.processEvents()
+
+    assert win.detect_all_button.isEnabled()
+    win.detect_all_button.click()
+    qapp.processEvents()
+    assert not win.detect_all_button.isEnabled(), "button should disable while detection runs"
+
+    ok = wait_for(lambda: win.detect_all_button.isEnabled(), qapp)
+    assert ok, "detection did not finish in time"
+
+    markers = win.marker_strip.markers()
+    print(f"detected markers: {markers}")
+    assert len(markers) >= 1, "expected at least one clap marker"
+    # ground truth beep is at t=5.350s in this fixture; onset detection has
+    # some inherent lag, so allow a generous window rather than exact match.
+    assert any(abs(m - 5.350) < 0.5 for m in markers), f"no marker near the known beep: {markers}"
+
+    win.grab().save(str(OUT_DIR / "clap_markers_on_timeline.png"))
+
+    # Simulate a real mouse click on the marker to prove click-to-seek works
+    # end to end, not just that the signal is wired.
+    target_time = min(markers, key=lambda m: abs(m - 5.350))
+    strip = win.marker_strip
+    strip.resize(win.seek_slider.width(), strip.height())
+    x = strip._x_for_time(target_time)
+    click_pos = QPointF(x, 5)
+
+    before_pos = win.player.get_position()
+    event = QMouseEvent(
+        QMouseEvent.Type.MouseButtonPress, click_pos, click_pos,
+        Qt.MouseButton.LeftButton, Qt.MouseButton.LeftButton, Qt.KeyboardModifier.NoModifier,
+    )
+    strip.mousePressEvent(event)
+    qapp.processEvents()
+    time.sleep(0.2)
+    qapp.processEvents()
+
+    after_pos = win.player.get_position()
+    print(f"clicked marker at x={x:.1f} (t={target_time:.3f}s); player position before={before_pos:.3f}s after={after_pos:.3f}s")
+    assert abs(after_pos - target_time) < 0.1, "clicking the marker should seek the player to that timestamp"
+
+    win.player.shutdown()
+    print("\nCLAP DETECTION SMOKE TEST PASSED")
+
+
+if __name__ == "__main__":
+    main()