From e6f3f370da2357d6236e67823fc13e478f47a763 Mon Sep 17 00:00:00 2001 From: ThePendulum Date: Tue, 29 Sep 2026 21:50:24 +0200 Subject: [PATCH] Added automatic offset detection using the difference between the audio and video stream ends. --- .gitignore | 1 + ...ync Marker.desktop => WeAreSyncing.desktop | 1 + main.py | 121 +++++++++++++- run.sh | 2 +- scripts/smoke_test_stream_ends_estimate.py | 157 ++++++++++++++++++ stream_edges.py | 145 ++++++++++++++++ 6 files changed, 423 insertions(+), 4 deletions(-) rename A-V Sync Marker.desktop => WeAreSyncing.desktop (94%) create mode 100644 scripts/smoke_test_stream_ends_estimate.py create mode 100644 stream_edges.py diff --git a/.gitignore b/.gitignore index 13a9dad..346c701 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__/ *.pyc .venv/ test_media/seek_validation/ +test_media/*_fixed* diff --git a/A-V Sync Marker.desktop b/WeAreSyncing.desktop similarity index 94% rename from A-V Sync Marker.desktop rename to WeAreSyncing.desktop index 2207594..5477647 100755 --- a/A-V Sync Marker.desktop +++ b/WeAreSyncing.desktop @@ -8,3 +8,4 @@ Terminal=false Categories=AudioVideo;Video; MimeType=video/mp4;video/quicktime;video/x-matroska;video/x-msvideo;video/webm;video/mpeg; StartupNotify=true +Name[en_US]=WeAreSyncing diff --git a/main.py b/main.py index 987382d..b16dc8d 100644 --- a/main.py +++ b/main.py @@ -19,6 +19,7 @@ from clap_detection import ( detect_claps, ClapDetectionUnavailable, ClapEvent, DEFAULT_THRESHOLD, DEFAULT_MIN_INTERVAL, ) +from stream_edges import probe_stream_edges, StreamEdges, StreamEdgesError class ClapSettingsDialog(QDialog): @@ -122,6 +123,28 @@ class DetectWorker(QThread): self.finished_err.emit(str(e)) +class EstimateWorker(QThread): + finished_ok = pyqtSignal(object) # StreamEdges + finished_err = pyqtSignal(str) + + def __init__(self, input_path, video_stream_index=None, audio_stream_index=None): + super().__init__() + self.input_path = input_path + self.video_stream_index = video_stream_index + self.audio_stream_index = audio_stream_index + + def run(self): + try: + edges = probe_stream_edges( + self.input_path, + video_stream_index=self.video_stream_index, + audio_stream_index=self.audio_stream_index, + ) + self.finished_ok.emit(edges) + except StreamEdgesError as e: + self.finished_err.emit(str(e)) + + class MainWindow(QMainWindow): def __init__(self): super().__init__() @@ -131,9 +154,13 @@ class MainWindow(QMainWindow): self.input_path: Optional[str] = None self.video_mark: Optional[float] = None self.audio_mark: Optional[float] = None + # The offset currently feeding Preview/Export -- derived either from + # the two markers or from the stream-ends estimate, whichever was + # applied last. self.offset: Optional[SyncOffset] = None self._export_worker: Optional[ExportWorker] = None self._detect_worker: Optional[DetectWorker] = None + self._estimate_worker: Optional[EstimateWorker] = None self._preview_worker: Optional[ExportWorker] = None self._preview_temp_path: Optional[str] = None # currently loaded in preview_panel self._pending_preview_path: Optional[str] = None # being built by _preview_worker @@ -187,6 +214,13 @@ class MainWindow(QMainWindow): self.video_mark_label = QLabel("video: —") marker_layout.addWidget(self.video_mark_label) + # Re-applies the marker-derived offset after the stream-ends + # estimate replaced it, without having to re-mark an event. + self.use_marker_offset_button = QPushButton("Use Marker Offset") + self.use_marker_offset_button.setEnabled(False) + self.use_marker_offset_button.clicked.connect(self._recompute_offset) + marker_layout.addWidget(self.use_marker_offset_button) + marker_layout.addStretch(1) self.detect_all_button = QPushButton("Detect All Claps") @@ -208,6 +242,11 @@ class MainWindow(QMainWindow): self.offset_label = QLabel("Offset: — (mark both events)") export_layout.addWidget(self.offset_label, stretch=1) + self.estimate_button = QPushButton("Estimate from Stream Ends") + self.estimate_button.setEnabled(False) + self.estimate_button.clicked.connect(self._on_estimate_clicked) + export_layout.addWidget(self.estimate_button) + export_layout.addWidget(QLabel("Override (ms):")) self.override_spinbox = QDoubleSpinBox() self.override_spinbox.setRange(-600000, 600000) @@ -263,6 +302,13 @@ class MainWindow(QMainWindow): tools_menu.addSeparator() + self.estimate_action = QAction("Estimate Offset from Stream Ends", self) + self.estimate_action.setEnabled(False) + self.estimate_action.triggered.connect(self._on_estimate_clicked) + tools_menu.addAction(self.estimate_action) + + tools_menu.addSeparator() + settings_action = QAction("Clap Detection Settings…", self) settings_action.triggered.connect(self._on_clap_settings_clicked) tools_menu.addAction(settings_action) @@ -315,6 +361,7 @@ class MainWindow(QMainWindow): self.export_button.setEnabled(False) self.preview_button.setEnabled(False) self.override_spinbox.setEnabled(False) + self.use_marker_offset_button.setEnabled(False) self.log_view.clear() self.setWindowTitle(f"WeAreSyncing — {Path(path).name}") @@ -323,6 +370,8 @@ class MainWindow(QMainWindow): self.detect_all_button.setEnabled(True) self.detect_next_action.setEnabled(True) self.detect_next_button.setEnabled(True) + self.estimate_action.setEnabled(True) + self.estimate_button.setEnabled(True) # A loaded preview belongs to the *previous* source/offset -- drop it. if self._preview_temp_path and os.path.exists(self._preview_temp_path): @@ -350,12 +399,71 @@ class MainWindow(QMainWindow): if self.video_mark is None or self.audio_mark is None: return self.offset = compute_offset(self.video_mark, self.audio_mark) - self.offset_label.setText(self.offset.describe()) - self.override_spinbox.setValue(self.offset.delay_ms) + self._apply_offset(self.offset, self.offset.describe()) + self.use_marker_offset_button.setEnabled(True) + + def _apply_offset(self, offset: SyncOffset, label: str): + """Shared by both offset sources (markers, stream-ends estimate): + whichever is applied last is what Preview/Export use.""" + self.offset = offset + self.offset_label.setText(label) + self.override_spinbox.setValue(offset.delay_ms) self.override_spinbox.setEnabled(True) self.export_button.setEnabled(True) self.preview_button.setEnabled(True) + # -- stream-ends estimate ---------------------------------------------- + + def _current_video_stream_index(self) -> Optional[int]: + for track in self.source_panel.player.mpv.track_list: + if track.get("type") == "video" and track.get("selected"): + return track.get("ff-index") + return None + + def _on_estimate_clicked(self): + if self.input_path is None: + return + self.estimate_action.setEnabled(False) + self.estimate_button.setEnabled(False) + self.statusBar().showMessage("Estimating offset from stream ends…") + + self._estimate_worker = EstimateWorker( + self.input_path, + video_stream_index=self._current_video_stream_index(), + audio_stream_index=self._current_audio_stream_index(), + ) + self._estimate_worker.finished_ok.connect(self._on_estimate_done) + self._estimate_worker.finished_err.connect(self._on_estimate_failed) + self._estimate_worker.start() + + def _on_estimate_done(self, edges: StreamEdges): + self.estimate_action.setEnabled(True) + self.estimate_button.setEnabled(True) + + offset = SyncOffset(delay_ms=round(edges.end_delay_ms)) + self._apply_offset(offset, f"{offset.describe()} — estimated from stream ends") + + self.log_view.appendPlainText( + f"Stream-ends estimate: video {edges.video_start:.3f}–{edges.video_end:.3f}s, " + f"audio {edges.audio_start:.3f}–{edges.audio_end:.3f}s → " + f"end-based {edges.end_delay_ms:+.0f} ms, start-based {edges.start_delay_ms:+.0f} ms" + ) + if edges.estimates_disagree: + message = ( + f"Estimated {offset.delay_ms:+.0f} ms from stream ends, but stream starts suggest " + f"{edges.start_delay_ms:+.0f} ms — verify with Preview." + ) + self.log_view.appendPlainText("WARNING: " + message) + else: + message = f"Estimated {offset.delay_ms:+.0f} ms from stream ends (assumes both tracks stopped recording together) — verify with Preview." + self.statusBar().showMessage(message, 10000) + + def _on_estimate_failed(self, message: str): + self.estimate_action.setEnabled(True) + self.estimate_button.setEnabled(True) + self.statusBar().showMessage("Offset estimate failed (see log).", 8000) + self.log_view.appendPlainText(f"Stream-ends estimate FAILED: {message}") + # -- preview ----------------------------------------------------- PREVIEW_MARGIN_SEC = 3.0 @@ -373,7 +481,14 @@ class MainWindow(QMainWindow): # keeps both the video and audio events (and wherever the offset # moves them to) inside the window for typical human-scale marking # errors. - center = (self.video_mark + self.audio_mark) / 2.0 + # + # An estimated offset may have no markers behind it; center on the + # Source playhead instead, so the user can park it on a sync event + # (or any stretch of speech) before previewing. + if self.video_mark is not None and self.audio_mark is not None: + center = (self.video_mark + self.audio_mark) / 2.0 + else: + center = self.source_panel.player.get_position() clip_start = max(center - self.PREVIEW_MARGIN_SEC, 0.0) input_p = Path(self.input_path) diff --git a/run.sh b/run.sh index 13e1885..11f0215 100755 --- a/run.sh +++ b/run.sh @@ -9,5 +9,5 @@ # /etc/ld.so.conf.d/ would fix this system-wide instead, if preferred. set -e cd "$(dirname "$0")" -export LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" +# export LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" exec .venv/bin/python3 "$@" diff --git a/scripts/smoke_test_stream_ends_estimate.py b/scripts/smoke_test_stream_ends_estimate.py new file mode 100644 index 0000000..bf8afe2 --- /dev/null +++ b/scripts/smoke_test_stream_ends_estimate.py @@ -0,0 +1,157 @@ +"""Verifies the stream-ends offset estimate: it lands on the known offset of +each synthetic fixture, it feeds Preview/Export exactly like a +marker-derived offset does (including with no markers set at all), and +Use Marker Offset brings the marker-derived value back afterwards without +re-marking -- while the manual marker workflow itself behaves as before. + +Also covers the real-world shape this feature exists for, which none of +the fixtures in test_media have: both tracks start at t=0 but the audio +ends early (it started capturing late, and the recorder stopped both +together). Generated on the fly as a 60s clip, long enough that the probe +only reads the tail window instead of the whole file. +""" +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +from PyQt6.QtWidgets import QApplication + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import main as app_main +from stream_edges import probe_stream_edges, TAIL_WINDOW_SEC + +ROOT = Path(__file__).resolve().parent.parent +MEDIA = ROOT / "test_media" + + +def wait_for(predicate, qapp, timeout=20): + deadline = time.time() + timeout + while not predicate() and time.time() < deadline: + qapp.processEvents() + time.sleep(0.05) + return predicate() + + +def load(win, qapp, path): + win._load(str(path)) + qapp.processEvents() + time.sleep(0.2) + qapp.processEvents() + + +def estimate(win, qapp): + win.estimate_button.click() + qapp.processEvents() + assert wait_for(lambda: win.estimate_button.isEnabled(), qapp), "estimate did not finish in time" + + +def make_audio_ends_early_clip(path: Path): + subprocess.run([ + "ffmpeg", "-y", "-v", "error", + "-f", "lavfi", "-i", "testsrc=size=320x240:rate=30:duration=60", + "-f", "lavfi", "-i", "sine=frequency=1000:duration=59", + "-c:v", "libx264", "-preset", "ultrafast", "-c:a", "aac", + str(path), + ], check=True) + + +def main(): + # -- Probe alone, against every fixture's known offset. + expected = { + "sync_test_INSYNC.mp4": 0, + "sync_test_OUTOFSYNC_audio+350ms.mp4": -350, + "sync_test_OUTOFSYNC_video+350ms.mp4": 350, + } + for name, want in expected.items(): + edges = probe_stream_edges(str(MEDIA / name)) + print(f"{name}: end-based {edges.end_delay_ms:+.1f} ms, start-based {edges.start_delay_ms:+.1f} ms") + assert abs(edges.end_delay_ms - want) < 5, f"{name}: expected ~{want} ms" + print("PASS: end-based estimate matches every fixture's known offset") + + with tempfile.TemporaryDirectory() as tmp: + clip = Path(tmp) / "audio_ends_early.mp4" + make_audio_ends_early_clip(clip) + assert 60 > TAIL_WINDOW_SEC, "clip should be long enough to exercise the tail-window read" + edges = probe_stream_edges(str(clip)) + print(f"audio-ends-early clip: {edges}, end-based {edges.end_delay_ms:+.1f} ms") + assert abs(edges.end_delay_ms - 1000) < 30, "expected ~+1000 ms (delay audio)" + assert edges.estimates_disagree, "starts both at 0 but ends 1s apart -- should flag disagreement" + print("PASS: audio-ends-early clip estimates ~+1000 ms and flags the start/end disagreement") + + qapp = QApplication(sys.argv) + win = app_main.MainWindow() + win.show() + qapp.processEvents() + + assert not win.estimate_button.isEnabled(), "estimate needs a loaded file" + assert not win.use_marker_offset_button.isEnabled() + + # -- Estimate with no markers: enables Preview/Export on its own. + load(win, qapp, MEDIA / "sync_test_OUTOFSYNC_video+350ms.mp4") + assert win.estimate_button.isEnabled() and win.estimate_action.isEnabled() + assert not win.export_button.isEnabled() and not win.preview_button.isEnabled() + estimate(win, qapp) + print(f"offset label after estimate: {win.offset_label.text()!r}") + assert win.offset.delay_ms == 350 + assert win.override_spinbox.value() == 350 and win.override_spinbox.isEnabled() + assert win.export_button.isEnabled() and win.preview_button.isEnabled() + assert "estimated from stream ends" in win.offset_label.text() + assert not win.use_marker_offset_button.isEnabled(), "no markers yet, nothing to go back to" + assert win.video_mark is None and win.audio_mark is None, "estimate must not touch the markers" + print("PASS: estimate with no markers sets the offset and enables Preview/Export") + + # -- Preview works from an estimate with no markers (centered on the playhead). + win.source_panel.player.seek(5.0) + qapp.processEvents() + win.preview_button.click() + qapp.processEvents() + assert wait_for(lambda: win.video_tabs.currentIndex() == 1, qapp), "preview did not build" + print("PASS: Preview builds from an estimated offset with no markers set") + win.video_tabs.setCurrentIndex(0) + + # -- Manual markers behave as before and override the estimate. + load(win, qapp, MEDIA / "sync_test_OUTOFSYNC_audio+350ms.mp4") + assert win.offset_label.text() == "Offset: — (mark both events)" + win.source_panel.player.seek(5.000) + qapp.processEvents() + win._mark_video() + win.source_panel.player.seek(5.350) + qapp.processEvents() + win._mark_audio() + marker_label = win.offset_label.text() + print(f"offset label from markers: {marker_label!r}") + assert abs(win.offset.delay_ms - (-350)) < 40 + assert marker_label == win.offset.describe() + assert win.use_marker_offset_button.isEnabled() + marker_delay = win.offset.delay_ms + print("PASS: manual markers set the offset exactly as before") + + # -- Estimate replaces it; Use Marker Offset restores it without re-marking. + estimate(win, qapp) + assert win.offset.delay_ms == -349, f"got {win.offset.delay_ms}" + assert win.override_spinbox.value() == -349 + assert win.video_mark is not None and win.audio_mark is not None, "estimate must not clear markers" + win.use_marker_offset_button.click() + qapp.processEvents() + assert win.offset.delay_ms == marker_delay + assert win.override_spinbox.value() == round(marker_delay) + assert win.offset_label.text() == marker_label + print("PASS: Use Marker Offset restores the marker-derived offset after an estimate") + + # -- Loading a new file resets everything, including Use Marker Offset. + load(win, qapp, MEDIA / "sync_test_INSYNC.mp4") + assert not win.use_marker_offset_button.isEnabled() + assert not win.export_button.isEnabled() + estimate(win, qapp) + assert win.offset.delay_ms == 0 + print("PASS: new file resets marker state; in-sync file estimates 0 ms") + + win.close() + qapp.processEvents() + print("\nSTREAM ENDS ESTIMATE SMOKE TEST PASSED") + + +if __name__ == "__main__": + main() diff --git a/stream_edges.py b/stream_edges.py new file mode 100644 index 0000000..4930898 --- /dev/null +++ b/stream_edges.py @@ -0,0 +1,145 @@ +"""Offset estimation from where the video and audio tracks start and end, +as an optional alternative to marking a sync event by hand. + +Premise: a recorder that stops both tracks at the same real-world moment, +but whose audio started capturing late (or early) while still being +written as starting at t=0, produces a file where the audio runs out +before (or after) the video by exactly the sync error. So + + delay_ms = (video_end - audio_end) * 1000 + +using the same sign convention as sync_logic.compute_offset (positive: +audio is ahead and gets delayed; negative: video gets delayed). + +This is only an *estimate*: a file whose audio genuinely stopped early +(a mic cutting out before the camera did) looks identical but has nothing +to do with sync. The start-based figure is reported alongside it as a +cross-check -- it's what carries the offset when the container encodes it +as a stream start_time instead, and a disagreement between the two is +worth surfacing. +""" +import subprocess +from dataclasses import dataclass +from typing import List, Optional, Tuple + +# How far before the end of the file to start reading packets. ffprobe +# seeks there instead of scanning the whole file, which keeps this fast on +# multi-hour recordings. If a track ends earlier than this window (so no +# packets for it turn up), the whole track is scanned instead. +TAIL_WINDOW_SEC = 30.0 + +# Start- and end-based estimates are compared against this. AAC encoder +# delay alone typically puts the start ~20-45ms off, so anything much +# tighter would warn on perfectly ordinary files. +DISAGREEMENT_TOLERANCE_MS = 50.0 + + +class StreamEdgesError(RuntimeError): + pass + + +@dataclass(frozen=True) +class StreamEdges: + video_start: float + video_end: float + audio_start: float + audio_end: float + + @property + def end_delay_ms(self) -> float: + return (self.video_end - self.audio_end) * 1000.0 + + @property + def start_delay_ms(self) -> float: + return (self.video_start - self.audio_start) * 1000.0 + + @property + def estimates_disagree(self) -> bool: + return abs(self.end_delay_ms - self.start_delay_ms) > DISAGREEMENT_TOLERANCE_MS + + +def _ffprobe(args: List[str]) -> str: + result = subprocess.run(["ffprobe", "-v", "error", *args], capture_output=True, text=True) + if result.returncode != 0: + raise StreamEdgesError(f"ffprobe failed: {result.stderr.strip()}") + return result.stdout + + +def _parse_float(value: str) -> Optional[float]: + try: + return float(value) + except ValueError: + return None # "N/A" + + +def _format_duration(input_path: str) -> Optional[float]: + out = _ffprobe(["-show_entries", "format=duration", "-of", "csv=p=0", input_path]) + return _parse_float(out.strip()) + + +def _stream_start(input_path: str, stream_specifier: str) -> float: + out = _ffprobe([ + "-select_streams", stream_specifier, + "-show_entries", "stream=start_time", "-of", "csv=p=0", input_path, + ]) + start = _parse_float(out.strip()) + return start if start is not None else 0.0 + + +def _packets(input_path: str, stream_specifier: str, read_from: Optional[float]) -> List[Tuple[float, Optional[float]]]: + args = ["-select_streams", stream_specifier] + if read_from is not None: + args += ["-read_intervals", f"{read_from:.3f}%"] + args += ["-show_entries", "packet=pts_time,duration_time", "-of", "csv=p=0", input_path] + packets = [] + for line in _ffprobe(args).splitlines(): + fields = line.strip().rstrip(",").split(",") + pts = _parse_float(fields[0]) if fields and fields[0] else None + if pts is None: + continue + duration = _parse_float(fields[1]) if len(fields) > 1 else None + packets.append((pts, duration)) + return packets + + +def _stream_end(input_path: str, stream_specifier: str, file_duration: Optional[float]) -> float: + """End of the last packet (pts + duration), not the container's stored + duration -- that's frequently missing (e.g. per-stream in MKV) or + rounded, and this needs to be accurate to well under a frame.""" + read_from = None + if file_duration is not None and file_duration > TAIL_WINDOW_SEC: + read_from = file_duration - TAIL_WINDOW_SEC + packets = _packets(input_path, stream_specifier, read_from) + if not packets and read_from is not None: + packets = _packets(input_path, stream_specifier, None) + if not packets: + raise StreamEdgesError(f"no packets found for stream {stream_specifier}") + + # Max rather than last: video with B-frames isn't stored in pts order. + ends = [pts + duration for pts, duration in packets if duration is not None] + if ends: + return max(ends) + # No per-packet durations (some containers): fall back to the gap + # between the last two packets as the final packet's duration. + pts_sorted = sorted(pts for pts, _ in packets) + step = pts_sorted[-1] - pts_sorted[-2] if len(pts_sorted) > 1 else 0.0 + return pts_sorted[-1] + step + + +def probe_stream_edges( + input_path: str, + video_stream_index: Optional[int] = None, + audio_stream_index: Optional[int] = None, +) -> StreamEdges: + """Stream indices are ffmpeg global indices (mpv's `ff-index`), so the + estimate uses the same tracks the player is showing -- see + clap_detection.py for why that matters on multi-track files.""" + video_spec = f"{video_stream_index}" if video_stream_index is not None else "v:0" + audio_spec = f"{audio_stream_index}" if audio_stream_index is not None else "a:0" + file_duration = _format_duration(input_path) + return StreamEdges( + video_start=_stream_start(input_path, video_spec), + video_end=_stream_end(input_path, video_spec, file_duration), + audio_start=_stream_start(input_path, audio_spec), + audio_end=_stream_end(input_path, audio_spec, file_duration), + )