diff --git a/.jules/bolt.md b/.jules/bolt.md index 63607c3..cab8b43 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -60,3 +60,6 @@ ## 2026-06-25 - [Optimize Path.exists() when paired with stat()] **Learning:** Checking `Path.exists()` before `Path.stat()` introduces a redundant system call because `exists()` internally uses `stat()`. **Action:** Rely on catching the `OSError` from `Path.stat()` to simultaneously check for existence and retrieve file attributes, saving measurable I/O overhead on large filesystems. +## 2024-11-20 - [Optimize massive log parsing with regex finditer] +**Learning:** When parsing large text logs from command-line tools like `ffmpeg` (which use carriage returns `\r` for progress updates), using `str.splitlines()` causes massive memory allocations and high CPU usage. +**Action:** Compile a unified regex and use `re.finditer()` directly on the raw output string for safe, memory-efficient parsing without allocating new strings for each line. diff --git a/media_shrinker.py b/media_shrinker.py index d72a8d5..e11753e 100644 --- a/media_shrinker.py +++ b/media_shrinker.py @@ -84,8 +84,9 @@ BITRATE_SAFETY_MARGIN = 0.92 OPUS_MAX_BITRATE_BPS = 510_000 OPUS_MIN_REASONABLE_BITRATE_BPS = 16_000 -SILENCE_START_RE = re.compile(r"silence_start:\s*(?P[0-9]+(?:\.[0-9]+)?)") -SILENCE_END_RE = re.compile(r"silence_end:\s*(?P[0-9]+(?:\.[0-9]+)?)") +SILENCE_EVENT_RE = re.compile( + r"silence_(?Pstart|end):\s*(?P[0-9]+(?:\.[0-9]+)?)" +) class MediaShrinkerError(RuntimeError): @@ -544,23 +545,19 @@ def parse_silencedetect_intervals(stderr: str) -> list[SilenceInterval]: intervals: list[SilenceInterval] = [] current_start: float | None = None - for line in stderr.splitlines(): - # Fast path: Substring search is much faster than regex. - # Most lines are ffmpeg progress updates (e.g., 'frame=...') - if "silence" not in line: - continue - start_match = SILENCE_START_RE.search(line) - if start_match: - current_start = float(start_match.group("value")) - continue - end_match = SILENCE_END_RE.search(line) - if end_match and current_start is not None: - end_seconds = float(end_match.group("value")) - if end_seconds > current_start: + # Using re.finditer directly on the raw string avoids creating massive + # lists of strings via splitlines() when parsing dense ffmpeg output. + for match in SILENCE_EVENT_RE.finditer(stderr): + event_type = match.group("type") + value = float(match.group("value")) + if event_type == "start": + current_start = value + elif event_type == "end" and current_start is not None: + if value > current_start: intervals.append( SilenceInterval( - start_seconds=current_start, end_seconds=end_seconds + start_seconds=current_start, end_seconds=value ) ) current_start = None