Added automatic offset detection using the difference between the audio and video stream ends.

.gitignore

@@ -2,3 +2,4 @@ __pycache__/
 *.pyc
 .venv/
 test_media/seek_validation/
+test_media/*_fixed*

WeAreSyncing.desktop

@@ -8,3 +8,4 @@ Terminal=false
 Categories=AudioVideo;Video;
 MimeType=video/mp4;video/quicktime;video/x-matroska;video/x-msvideo;video/webm;video/mpeg;
 StartupNotify=true
+Name[en_US]=WeAreSyncing

main.py

@@ -19,6 +19,7 @@ from clap_detection import (
     detect_claps, ClapDetectionUnavailable, ClapEvent,
     DEFAULT_THRESHOLD, DEFAULT_MIN_INTERVAL,
 )
+from stream_edges import probe_stream_edges, StreamEdges, StreamEdgesError
 
 
 class ClapSettingsDialog(QDialog):
@@ -122,6 +123,28 @@ class DetectWorker(QThread):
             self.finished_err.emit(str(e))
 
 
+class EstimateWorker(QThread):
+    finished_ok = pyqtSignal(object)   # StreamEdges
+    finished_err = pyqtSignal(str)
+
+    def __init__(self, input_path, video_stream_index=None, audio_stream_index=None):
+        super().__init__()
+        self.input_path = input_path
+        self.video_stream_index = video_stream_index
+        self.audio_stream_index = audio_stream_index
+
+    def run(self):
+        try:
+            edges = probe_stream_edges(
+                self.input_path,
+                video_stream_index=self.video_stream_index,
+                audio_stream_index=self.audio_stream_index,
+            )
+            self.finished_ok.emit(edges)
+        except StreamEdgesError as e:
+            self.finished_err.emit(str(e))
+
+
 class MainWindow(QMainWindow):
     def __init__(self):
         super().__init__()
@@ -131,9 +154,13 @@ class MainWindow(QMainWindow):
         self.input_path: Optional[str] = None
         self.video_mark: Optional[float] = None
         self.audio_mark: Optional[float] = None
+        # The offset currently feeding Preview/Export -- derived either from
+        # the two markers or from the stream-ends estimate, whichever was
+        # applied last.
         self.offset: Optional[SyncOffset] = None
         self._export_worker: Optional[ExportWorker] = None
         self._detect_worker: Optional[DetectWorker] = None
+        self._estimate_worker: Optional[EstimateWorker] = None
         self._preview_worker: Optional[ExportWorker] = None
         self._preview_temp_path: Optional[str] = None  # currently loaded in preview_panel
         self._pending_preview_path: Optional[str] = None  # being built by _preview_worker
@@ -187,6 +214,13 @@ class MainWindow(QMainWindow):
         self.video_mark_label = QLabel("video: —")
         marker_layout.addWidget(self.video_mark_label)
 
+        # Re-applies the marker-derived offset after the stream-ends
+        # estimate replaced it, without having to re-mark an event.
+        self.use_marker_offset_button = QPushButton("Use Marker Offset")
+        self.use_marker_offset_button.setEnabled(False)
+        self.use_marker_offset_button.clicked.connect(self._recompute_offset)
+        marker_layout.addWidget(self.use_marker_offset_button)
+
         marker_layout.addStretch(1)
 
         self.detect_all_button = QPushButton("Detect All Claps")
@@ -208,6 +242,11 @@ class MainWindow(QMainWindow):
         self.offset_label = QLabel("Offset: — (mark both events)")
         export_layout.addWidget(self.offset_label, stretch=1)
 
+        self.estimate_button = QPushButton("Estimate from Stream Ends")
+        self.estimate_button.setEnabled(False)
+        self.estimate_button.clicked.connect(self._on_estimate_clicked)
+        export_layout.addWidget(self.estimate_button)
+
         export_layout.addWidget(QLabel("Override (ms):"))
         self.override_spinbox = QDoubleSpinBox()
         self.override_spinbox.setRange(-600000, 600000)
@@ -263,6 +302,13 @@ class MainWindow(QMainWindow):
 
         tools_menu.addSeparator()
 
+        self.estimate_action = QAction("Estimate Offset from Stream Ends", self)
+        self.estimate_action.setEnabled(False)
+        self.estimate_action.triggered.connect(self._on_estimate_clicked)
+        tools_menu.addAction(self.estimate_action)
+
+        tools_menu.addSeparator()
+
         settings_action = QAction("Clap Detection Settings…", self)
         settings_action.triggered.connect(self._on_clap_settings_clicked)
         tools_menu.addAction(settings_action)
@@ -315,6 +361,7 @@ class MainWindow(QMainWindow):
         self.export_button.setEnabled(False)
         self.preview_button.setEnabled(False)
         self.override_spinbox.setEnabled(False)
+        self.use_marker_offset_button.setEnabled(False)
         self.log_view.clear()
 
         self.setWindowTitle(f"WeAreSyncing — {Path(path).name}")
@@ -323,6 +370,8 @@ class MainWindow(QMainWindow):
         self.detect_all_button.setEnabled(True)
         self.detect_next_action.setEnabled(True)
         self.detect_next_button.setEnabled(True)
+        self.estimate_action.setEnabled(True)
+        self.estimate_button.setEnabled(True)
 
         # A loaded preview belongs to the *previous* source/offset -- drop it.
         if self._preview_temp_path and os.path.exists(self._preview_temp_path):
@@ -350,12 +399,71 @@ class MainWindow(QMainWindow):
         if self.video_mark is None or self.audio_mark is None:
             return
         self.offset = compute_offset(self.video_mark, self.audio_mark)
-        self.offset_label.setText(self.offset.describe())
-        self.override_spinbox.setValue(self.offset.delay_ms)
+        self._apply_offset(self.offset, self.offset.describe())
+        self.use_marker_offset_button.setEnabled(True)
+
+    def _apply_offset(self, offset: SyncOffset, label: str):
+        """Shared by both offset sources (markers, stream-ends estimate):
+        whichever is applied last is what Preview/Export use."""
+        self.offset = offset
+        self.offset_label.setText(label)
+        self.override_spinbox.setValue(offset.delay_ms)
         self.override_spinbox.setEnabled(True)
         self.export_button.setEnabled(True)
         self.preview_button.setEnabled(True)
 
+    # -- stream-ends estimate ----------------------------------------------
+
+    def _current_video_stream_index(self) -> Optional[int]:
+        for track in self.source_panel.player.mpv.track_list:
+            if track.get("type") == "video" and track.get("selected"):
+                return track.get("ff-index")
+        return None
+
+    def _on_estimate_clicked(self):
+        if self.input_path is None:
+            return
+        self.estimate_action.setEnabled(False)
+        self.estimate_button.setEnabled(False)
+        self.statusBar().showMessage("Estimating offset from stream ends…")
+
+        self._estimate_worker = EstimateWorker(
+            self.input_path,
+            video_stream_index=self._current_video_stream_index(),
+            audio_stream_index=self._current_audio_stream_index(),
+        )
+        self._estimate_worker.finished_ok.connect(self._on_estimate_done)
+        self._estimate_worker.finished_err.connect(self._on_estimate_failed)
+        self._estimate_worker.start()
+
+    def _on_estimate_done(self, edges: StreamEdges):
+        self.estimate_action.setEnabled(True)
+        self.estimate_button.setEnabled(True)
+
+        offset = SyncOffset(delay_ms=round(edges.end_delay_ms))
+        self._apply_offset(offset, f"{offset.describe()} — estimated from stream ends")
+
+        self.log_view.appendPlainText(
+            f"Stream-ends estimate: video {edges.video_start:.3f}–{edges.video_end:.3f}s, "
+            f"audio {edges.audio_start:.3f}–{edges.audio_end:.3f}s → "
+            f"end-based {edges.end_delay_ms:+.0f} ms, start-based {edges.start_delay_ms:+.0f} ms"
+        )
+        if edges.estimates_disagree:
+            message = (
+                f"Estimated {offset.delay_ms:+.0f} ms from stream ends, but stream starts suggest "
+                f"{edges.start_delay_ms:+.0f} ms — verify with Preview."
+            )
+            self.log_view.appendPlainText("WARNING: " + message)
+        else:
+            message = f"Estimated {offset.delay_ms:+.0f} ms from stream ends (assumes both tracks stopped recording together) — verify with Preview."
+        self.statusBar().showMessage(message, 10000)
+
+    def _on_estimate_failed(self, message: str):
+        self.estimate_action.setEnabled(True)
+        self.estimate_button.setEnabled(True)
+        self.statusBar().showMessage("Offset estimate failed (see log).", 8000)
+        self.log_view.appendPlainText(f"Stream-ends estimate FAILED: {message}")
+
     # -- preview -----------------------------------------------------
 
     PREVIEW_MARGIN_SEC = 3.0
@@ -373,7 +481,14 @@ class MainWindow(QMainWindow):
         # keeps both the video and audio events (and wherever the offset
         # moves them to) inside the window for typical human-scale marking
         # errors.
-        center = (self.video_mark + self.audio_mark) / 2.0
+        #
+        # An estimated offset may have no markers behind it; center on the
+        # Source playhead instead, so the user can park it on a sync event
+        # (or any stretch of speech) before previewing.
+        if self.video_mark is not None and self.audio_mark is not None:
+            center = (self.video_mark + self.audio_mark) / 2.0
+        else:
+            center = self.source_panel.player.get_position()
         clip_start = max(center - self.PREVIEW_MARGIN_SEC, 0.0)
 
         input_p = Path(self.input_path)

run.sh

@@ -9,5 +9,5 @@
 # /etc/ld.so.conf.d/ would fix this system-wide instead, if preferred.
 set -e
 cd "$(dirname "$0")"
-export LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}"
+# export LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}"
 exec .venv/bin/python3 "$@"

scripts/smoke_test_stream_ends_estimate.py

@@ -0,0 +1,157 @@
+"""Verifies the stream-ends offset estimate: it lands on the known offset of
+each synthetic fixture, it feeds Preview/Export exactly like a
+marker-derived offset does (including with no markers set at all), and
+Use Marker Offset brings the marker-derived value back afterwards without
+re-marking -- while the manual marker workflow itself behaves as before.
+
+Also covers the real-world shape this feature exists for, which none of
+the fixtures in test_media have: both tracks start at t=0 but the audio
+ends early (it started capturing late, and the recorder stopped both
+together). Generated on the fly as a 60s clip, long enough that the probe
+only reads the tail window instead of the whole file.
+"""
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+from PyQt6.QtWidgets import QApplication
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+import main as app_main
+from stream_edges import probe_stream_edges, TAIL_WINDOW_SEC
+
+ROOT = Path(__file__).resolve().parent.parent
+MEDIA = ROOT / "test_media"
+
+
+def wait_for(predicate, qapp, timeout=20):
+    deadline = time.time() + timeout
+    while not predicate() and time.time() < deadline:
+        qapp.processEvents()
+        time.sleep(0.05)
+    return predicate()
+
+
+def load(win, qapp, path):
+    win._load(str(path))
+    qapp.processEvents()
+    time.sleep(0.2)
+    qapp.processEvents()
+
+
+def estimate(win, qapp):
+    win.estimate_button.click()
+    qapp.processEvents()
+    assert wait_for(lambda: win.estimate_button.isEnabled(), qapp), "estimate did not finish in time"
+
+
+def make_audio_ends_early_clip(path: Path):
+    subprocess.run([
+        "ffmpeg", "-y", "-v", "error",
+        "-f", "lavfi", "-i", "testsrc=size=320x240:rate=30:duration=60",
+        "-f", "lavfi", "-i", "sine=frequency=1000:duration=59",
+        "-c:v", "libx264", "-preset", "ultrafast", "-c:a", "aac",
+        str(path),
+    ], check=True)
+
+
+def main():
+    # -- Probe alone, against every fixture's known offset.
+    expected = {
+        "sync_test_INSYNC.mp4": 0,
+        "sync_test_OUTOFSYNC_audio+350ms.mp4": -350,
+        "sync_test_OUTOFSYNC_video+350ms.mp4": 350,
+    }
+    for name, want in expected.items():
+        edges = probe_stream_edges(str(MEDIA / name))
+        print(f"{name}: end-based {edges.end_delay_ms:+.1f} ms, start-based {edges.start_delay_ms:+.1f} ms")
+        assert abs(edges.end_delay_ms - want) < 5, f"{name}: expected ~{want} ms"
+    print("PASS: end-based estimate matches every fixture's known offset")
+
+    with tempfile.TemporaryDirectory() as tmp:
+        clip = Path(tmp) / "audio_ends_early.mp4"
+        make_audio_ends_early_clip(clip)
+        assert 60 > TAIL_WINDOW_SEC, "clip should be long enough to exercise the tail-window read"
+        edges = probe_stream_edges(str(clip))
+        print(f"audio-ends-early clip: {edges}, end-based {edges.end_delay_ms:+.1f} ms")
+        assert abs(edges.end_delay_ms - 1000) < 30, "expected ~+1000 ms (delay audio)"
+        assert edges.estimates_disagree, "starts both at 0 but ends 1s apart -- should flag disagreement"
+    print("PASS: audio-ends-early clip estimates ~+1000 ms and flags the start/end disagreement")
+
+    qapp = QApplication(sys.argv)
+    win = app_main.MainWindow()
+    win.show()
+    qapp.processEvents()
+
+    assert not win.estimate_button.isEnabled(), "estimate needs a loaded file"
+    assert not win.use_marker_offset_button.isEnabled()
+
+    # -- Estimate with no markers: enables Preview/Export on its own.
+    load(win, qapp, MEDIA / "sync_test_OUTOFSYNC_video+350ms.mp4")
+    assert win.estimate_button.isEnabled() and win.estimate_action.isEnabled()
+    assert not win.export_button.isEnabled() and not win.preview_button.isEnabled()
+    estimate(win, qapp)
+    print(f"offset label after estimate: {win.offset_label.text()!r}")
+    assert win.offset.delay_ms == 350
+    assert win.override_spinbox.value() == 350 and win.override_spinbox.isEnabled()
+    assert win.export_button.isEnabled() and win.preview_button.isEnabled()
+    assert "estimated from stream ends" in win.offset_label.text()
+    assert not win.use_marker_offset_button.isEnabled(), "no markers yet, nothing to go back to"
+    assert win.video_mark is None and win.audio_mark is None, "estimate must not touch the markers"
+    print("PASS: estimate with no markers sets the offset and enables Preview/Export")
+
+    # -- Preview works from an estimate with no markers (centered on the playhead).
+    win.source_panel.player.seek(5.0)
+    qapp.processEvents()
+    win.preview_button.click()
+    qapp.processEvents()
+    assert wait_for(lambda: win.video_tabs.currentIndex() == 1, qapp), "preview did not build"
+    print("PASS: Preview builds from an estimated offset with no markers set")
+    win.video_tabs.setCurrentIndex(0)
+
+    # -- Manual markers behave as before and override the estimate.
+    load(win, qapp, MEDIA / "sync_test_OUTOFSYNC_audio+350ms.mp4")
+    assert win.offset_label.text() == "Offset: — (mark both events)"
+    win.source_panel.player.seek(5.000)
+    qapp.processEvents()
+    win._mark_video()
+    win.source_panel.player.seek(5.350)
+    qapp.processEvents()
+    win._mark_audio()
+    marker_label = win.offset_label.text()
+    print(f"offset label from markers: {marker_label!r}")
+    assert abs(win.offset.delay_ms - (-350)) < 40
+    assert marker_label == win.offset.describe()
+    assert win.use_marker_offset_button.isEnabled()
+    marker_delay = win.offset.delay_ms
+    print("PASS: manual markers set the offset exactly as before")
+
+    # -- Estimate replaces it; Use Marker Offset restores it without re-marking.
+    estimate(win, qapp)
+    assert win.offset.delay_ms == -349, f"got {win.offset.delay_ms}"
+    assert win.override_spinbox.value() == -349
+    assert win.video_mark is not None and win.audio_mark is not None, "estimate must not clear markers"
+    win.use_marker_offset_button.click()
+    qapp.processEvents()
+    assert win.offset.delay_ms == marker_delay
+    assert win.override_spinbox.value() == round(marker_delay)
+    assert win.offset_label.text() == marker_label
+    print("PASS: Use Marker Offset restores the marker-derived offset after an estimate")
+
+    # -- Loading a new file resets everything, including Use Marker Offset.
+    load(win, qapp, MEDIA / "sync_test_INSYNC.mp4")
+    assert not win.use_marker_offset_button.isEnabled()
+    assert not win.export_button.isEnabled()
+    estimate(win, qapp)
+    assert win.offset.delay_ms == 0
+    print("PASS: new file resets marker state; in-sync file estimates 0 ms")
+
+    win.close()
+    qapp.processEvents()
+    print("\nSTREAM ENDS ESTIMATE SMOKE TEST PASSED")
+
+
+if __name__ == "__main__":
+    main()

stream_edges.py

@@ -0,0 +1,145 @@
+"""Offset estimation from where the video and audio tracks start and end,
+as an optional alternative to marking a sync event by hand.
+
+Premise: a recorder that stops both tracks at the same real-world moment,
+but whose audio started capturing late (or early) while still being
+written as starting at t=0, produces a file where the audio runs out
+before (or after) the video by exactly the sync error. So
+
+    delay_ms = (video_end - audio_end) * 1000
+
+using the same sign convention as sync_logic.compute_offset (positive:
+audio is ahead and gets delayed; negative: video gets delayed).
+
+This is only an *estimate*: a file whose audio genuinely stopped early
+(a mic cutting out before the camera did) looks identical but has nothing
+to do with sync. The start-based figure is reported alongside it as a
+cross-check -- it's what carries the offset when the container encodes it
+as a stream start_time instead, and a disagreement between the two is
+worth surfacing.
+"""
+import subprocess
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+# How far before the end of the file to start reading packets. ffprobe
+# seeks there instead of scanning the whole file, which keeps this fast on
+# multi-hour recordings. If a track ends earlier than this window (so no
+# packets for it turn up), the whole track is scanned instead.
+TAIL_WINDOW_SEC = 30.0
+
+# Start- and end-based estimates are compared against this. AAC encoder
+# delay alone typically puts the start ~20-45ms off, so anything much
+# tighter would warn on perfectly ordinary files.
+DISAGREEMENT_TOLERANCE_MS = 50.0
+
+
+class StreamEdgesError(RuntimeError):
+    pass
+
+
+@dataclass(frozen=True)
+class StreamEdges:
+    video_start: float
+    video_end: float
+    audio_start: float
+    audio_end: float
+
+    @property
+    def end_delay_ms(self) -> float:
+        return (self.video_end - self.audio_end) * 1000.0
+
+    @property
+    def start_delay_ms(self) -> float:
+        return (self.video_start - self.audio_start) * 1000.0
+
+    @property
+    def estimates_disagree(self) -> bool:
+        return abs(self.end_delay_ms - self.start_delay_ms) > DISAGREEMENT_TOLERANCE_MS
+
+
+def _ffprobe(args: List[str]) -> str:
+    result = subprocess.run(["ffprobe", "-v", "error", *args], capture_output=True, text=True)
+    if result.returncode != 0:
+        raise StreamEdgesError(f"ffprobe failed: {result.stderr.strip()}")
+    return result.stdout
+
+
+def _parse_float(value: str) -> Optional[float]:
+    try:
+        return float(value)
+    except ValueError:
+        return None  # "N/A"
+
+
+def _format_duration(input_path: str) -> Optional[float]:
+    out = _ffprobe(["-show_entries", "format=duration", "-of", "csv=p=0", input_path])
+    return _parse_float(out.strip())
+
+
+def _stream_start(input_path: str, stream_specifier: str) -> float:
+    out = _ffprobe([
+        "-select_streams", stream_specifier,
+        "-show_entries", "stream=start_time", "-of", "csv=p=0", input_path,
+    ])
+    start = _parse_float(out.strip())
+    return start if start is not None else 0.0
+
+
+def _packets(input_path: str, stream_specifier: str, read_from: Optional[float]) -> List[Tuple[float, Optional[float]]]:
+    args = ["-select_streams", stream_specifier]
+    if read_from is not None:
+        args += ["-read_intervals", f"{read_from:.3f}%"]
+    args += ["-show_entries", "packet=pts_time,duration_time", "-of", "csv=p=0", input_path]
+    packets = []
+    for line in _ffprobe(args).splitlines():
+        fields = line.strip().rstrip(",").split(",")
+        pts = _parse_float(fields[0]) if fields and fields[0] else None
+        if pts is None:
+            continue
+        duration = _parse_float(fields[1]) if len(fields) > 1 else None
+        packets.append((pts, duration))
+    return packets
+
+
+def _stream_end(input_path: str, stream_specifier: str, file_duration: Optional[float]) -> float:
+    """End of the last packet (pts + duration), not the container's stored
+    duration -- that's frequently missing (e.g. per-stream in MKV) or
+    rounded, and this needs to be accurate to well under a frame."""
+    read_from = None
+    if file_duration is not None and file_duration > TAIL_WINDOW_SEC:
+        read_from = file_duration - TAIL_WINDOW_SEC
+    packets = _packets(input_path, stream_specifier, read_from)
+    if not packets and read_from is not None:
+        packets = _packets(input_path, stream_specifier, None)
+    if not packets:
+        raise StreamEdgesError(f"no packets found for stream {stream_specifier}")
+
+    # Max rather than last: video with B-frames isn't stored in pts order.
+    ends = [pts + duration for pts, duration in packets if duration is not None]
+    if ends:
+        return max(ends)
+    # No per-packet durations (some containers): fall back to the gap
+    # between the last two packets as the final packet's duration.
+    pts_sorted = sorted(pts for pts, _ in packets)
+    step = pts_sorted[-1] - pts_sorted[-2] if len(pts_sorted) > 1 else 0.0
+    return pts_sorted[-1] + step
+
+
+def probe_stream_edges(
+    input_path: str,
+    video_stream_index: Optional[int] = None,
+    audio_stream_index: Optional[int] = None,
+) -> StreamEdges:
+    """Stream indices are ffmpeg global indices (mpv's `ff-index`), so the
+    estimate uses the same tracks the player is showing -- see
+    clap_detection.py for why that matters on multi-track files."""
+    video_spec = f"{video_stream_index}" if video_stream_index is not None else "v:0"
+    audio_spec = f"{audio_stream_index}" if audio_stream_index is not None else "a:0"
+    file_duration = _format_duration(input_path)
+    return StreamEdges(
+        video_start=_stream_start(input_path, video_spec),
+        video_end=_stream_end(input_path, video_spec, file_duration),
+        audio_start=_stream_start(input_path, audio_spec),
+        audio_end=_stream_end(input_path, audio_spec, file_duration),
+    )