DebaucheryLibrarian/WeAreSyncing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add clap/onset auto-detection feature

Browse Source 
Ports scripts/detect_claps.py (a librosa-based onset detector from an
earlier session) into clap_detection.py, shared by both the CLI script
and a new GUI feature: 'Detect All Claps' (whole-file scan) and
'Detect Next Clap' (from cursor, first result only), with results
shown as clickable markers on a new MarkerStrip widget above the seek
slider -- clicking one seeks the player to that exact timestamp.

Caught a real correctness bug while testing against the out-of-sync
fixture (not just the plain in-sync one): ffmpeg's default -vn audio
extraction silently drops the audio stream's own start_time and
renormalizes to t=0 -- which is exactly the offset this whole tool
exists to fix, so losing it meant detected timestamps pointed at the
wrong track's timing entirely. Fixed by probing the true start_time
via ffprobe and re-inserting it as explicit silence (adelay) before
analysis. Also dropped an earlier attempt at an ffmpeg -ss seek
optimization for the from-cursor case, since combining it with the
start_time fix meant two independent timestamp-origin corrections
stacked on each other -- correct in theory, too easy to get subtly
wrong in a tool where a silent wrong-timestamp bug is the worst
possible failure mode. Full-file decode + filter in Python is slower
but unambiguous.

Verified with a smoke test that drives the real MainWindow buttons,
confirms the detected marker lands near the fixture's known beep
(t=5.350s, allowing for onset-detection lag), and simulates an actual
QMouseEvent click on the marker to confirm click-to-seek works
end-to-end.
This commit is contained in:
ThePendulum
2026-07-26 01:46:11 +02:00
parent f6e6f79a34
commit f7c15e924c
6 changed files with 468 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+106 -1
View File
@@ -13,6 +13,8 @@ from PyQt6.QtWidgets import (
from player_widget import PlayerWidget
from sync_logic import compute_offset, SyncOffset
from ffmpeg_export import export_synced, FfmpegExportError, ExportResult
from clap_detection import detect_claps, ClapDetectionUnavailable, ClapEvent
from marker_strip import MarkerStrip
def format_timestamp(seconds: float) -> str:
@@ -38,6 +40,28 @@ class ExportWorker(QThread):
self.finished_err.emit(str(e), e.stderr)
class DetectWorker(QThread):
finished_ok = pyqtSignal(list) # List[ClapEvent]
finished_err = pyqtSignal(str)
def __init__(self, input_path, start_offset=0.0, max_results=None):
super().__init__()
self.input_path = input_path
self.start_offset = start_offset
self.max_results = max_results
def run(self):
try:
events = detect_claps(
self.input_path,
start_offset=self.start_offset,
max_results=self.max_results,
)
self.finished_ok.emit(events)
except (ClapDetectionUnavailable, RuntimeError) as e:
self.finished_err.emit(str(e))
class MainWindow(QMainWindow):
def __init__(self):
super().__init__()
@@ -49,6 +73,7 @@ class MainWindow(QMainWindow):
self.audio_mark: Optional[float] = None
self.offset: Optional[SyncOffset] = None
self._export_worker: Optional[ExportWorker] = None
self._detect_worker: Optional[DetectWorker] = None
self._slider_being_dragged = False
self.player = PlayerWidget()
@@ -99,16 +124,42 @@ class MainWindow(QMainWindow):
self.position_label = QLabel("00:00.000")
seek_row.addWidget(self.position_label)
slider_stack = QVBoxLayout()
slider_stack.setSpacing(0)
self.marker_strip = MarkerStrip()
self.marker_strip.markerClicked.connect(self._on_marker_clicked)
slider_stack.addWidget(self.marker_strip)
self.seek_slider = QSlider(Qt.Orientation.Horizontal)
self.seek_slider.setRange(0, 0)
self.seek_slider.sliderPressed.connect(lambda: setattr(self, "_slider_being_dragged", True))
self.seek_slider.sliderReleased.connect(self._on_slider_released)
seek_row.addWidget(self.seek_slider, stretch=1)
slider_stack.addWidget(self.seek_slider)
seek_row.addLayout(slider_stack, stretch=1)
self.duration_label = QLabel("00:00.000")
seek_row.addWidget(self.duration_label)
layout.addLayout(seek_row)
# Clap/onset auto-detect row
detect_group = QGroupBox("Clap detection")
detect_layout = QHBoxLayout(detect_group)
self.detect_all_button = QPushButton("Detect All Claps (whole file)")
self.detect_all_button.setEnabled(False)
self.detect_all_button.clicked.connect(self._on_detect_all_clicked)
detect_layout.addWidget(self.detect_all_button)
self.detect_next_button = QPushButton("Detect Next Clap (from cursor)")
self.detect_next_button.setEnabled(False)
self.detect_next_button.clicked.connect(self._on_detect_next_clicked)
detect_layout.addWidget(self.detect_next_button)
self.detect_status_label = QLabel("")
detect_layout.addWidget(self.detect_status_label, stretch=1)
layout.addWidget(detect_group)
# Marker row
marker_group = QGroupBox("Sync markers")
marker_layout = QHBoxLayout(marker_group)
@@ -193,6 +244,12 @@ class MainWindow(QMainWindow):
self.duration_label.setText(format_timestamp(duration))
self.setWindowTitle(f"A/V Sync Marker — {Path(path).name}")
self.marker_strip.set_duration(duration)
self.marker_strip.clear()
self.detect_status_label.setText("")
self.detect_all_button.setEnabled(True)
self.detect_next_button.setEnabled(True)
def _toggle_play(self):
if self.player.mpv.pause:
self.player.play()
@@ -247,6 +304,54 @@ class MainWindow(QMainWindow):
self.override_spinbox.setEnabled(True)
self.export_button.setEnabled(True)
# -- clap detection -----------------------------------------------------
def _on_marker_clicked(self, time_sec: float):
self.player.seek(time_sec)
self._update_position_ui()
def _on_detect_all_clicked(self):
self._start_detection(start_offset=0.0, max_results=None, label="Scanning whole file for claps…")
def _on_detect_next_clicked(self):
cursor = self.player.get_position()
self._start_detection(start_offset=cursor, max_results=1, label=f"Scanning from {format_timestamp(cursor)} for next clap…")
def _start_detection(self, start_offset: float, max_results, label: str):
if self.input_path is None:
return
self.detect_all_button.setEnabled(False)
self.detect_next_button.setEnabled(False)
self.detect_status_label.setText(label)
self._detect_worker = DetectWorker(self.input_path, start_offset=start_offset, max_results=max_results)
self._detect_worker.finished_ok.connect(self._on_detect_done)
self._detect_worker.finished_err.connect(self._on_detect_failed)
self._detect_worker.start()
def _on_detect_done(self, events: list):
self.detect_all_button.setEnabled(True)
self.detect_next_button.setEnabled(True)
existing = set(self.marker_strip.markers())
existing.update(e.time_sec for e in events)
self.marker_strip.set_markers(list(existing))
if not events:
self.detect_status_label.setText("No claps found.")
self.log_view.appendPlainText("Clap detection: no onsets found.")
return
self.detect_status_label.setText(f"Found {len(events)} clap(s) — click a marker above the timeline to jump to it.")
for e in events:
self.log_view.appendPlainText(f"Clap detected at {format_timestamp(e.time_sec)} (strength {e.strength:.2f})")
def _on_detect_failed(self, message: str):
self.detect_all_button.setEnabled(True)
self.detect_next_button.setEnabled(True)
self.detect_status_label.setText("Detection failed (see log).")
self.log_view.appendPlainText(f"Clap detection FAILED: {message}")
# -- export -----------------------------------------------------
def _on_export_clicked(self):