From 83e9969d05a9c461726dad0c6afe351e98d74e2d Mon Sep 17 00:00:00 2001 From: Stoupy51 Date: Sat, 9 May 2026 18:00:58 +0200 Subject: [PATCH 1/3] feat(output): Implement incremental save functionality for packs (fixes #512) --- src/beet/contrib/output.py | 90 +++++++++++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 6 deletions(-) diff --git a/src/beet/contrib/output.py b/src/beet/contrib/output.py index 48657e91f..453264d6a 100644 --- a/src/beet/contrib/output.py +++ b/src/beet/contrib/output.py @@ -6,31 +6,109 @@ ] -from typing import Optional +import filecmp +import os +from pathlib import Path from beet import Context, ListOption, PluginOptions, configurable from beet.core.utils import FileSystemPath, log_time_scope +from beet.library.base import Pack, PackFile +from beet.library.utils import list_files as list_dir_files class OutputOptions(PluginOptions): - directory: Optional[ListOption[FileSystemPath]] = None + directory: ListOption[FileSystemPath] | None = None + incremental: bool = False def beet_default(ctx: Context): ctx.require(output) +def incremental_save(pack: Pack, output_path: Path) -> None: + """ Save a pack incrementally: delete removed files, write new/changed files, skip unchanged. """ + # Build expected set: posix-style relative paths -> PackFile + expected: dict[str, PackFile] = dict(pack.list_files()) + + # Build disk set: posix-style relative paths currently on disk + disk_set: set[str] = set() + if output_path.is_dir(): + for rel in list_dir_files(output_path): + disk_set.add(rel.as_posix()) + + # Delete files that are no longer present in the pack + deleted_files: set[str] = disk_set - expected.keys() + for rel_path in deleted_files: + disk_path: Path = output_path / rel_path + disk_path.unlink(missing_ok=True) + + # Remove empty directories left after deletions (bottom-up) + for dirpath, _dirnames, _filenames in os.walk(output_path, topdown=False): + dir_obj = Path(dirpath) + if dir_obj == output_path: + continue + try: + dir_obj.rmdir() # only succeeds if empty + except OSError: + pass # not empty - leave it + + # Ensure the root output directory exists + output_path.mkdir(parents=True, exist_ok=True) + + # For each expected file, compare with disk and write if new/changed + for rel_path, pack_file in expected.items(): + disk_path: Path = output_path / rel_path + + # New file, write directly without comparison + if not disk_path.exists(): + disk_path.parent.mkdir(parents=True, exist_ok=True) + pack_file.dump(output_path, rel_path) + + # Existing file: compare before writing + else: + changed: bool = True + + # Fast path: if the pack file still points directly to a source path, avoid loading content into memory + if ( + pack_file.source_path is not None + and pack_file.source_start is None + and pack_file.source_stop is None + ): + changed = not filecmp.cmp(pack_file.source_path, disk_path, shallow=False) + else: + # Standard path: use beet's own content equality + try: + existing_file: PackFile = type(pack_file)(source_path=disk_path) + changed = not pack_file.content_equal(existing_file) + except Exception: + changed = True # Fallback to overwrite on any error + + if changed: + pack_file.dump(output_path, rel_path) + + @configurable(validator=OutputOptions) def output(ctx: Context, opts: OutputOptions): - """Plugin that outputs the data pack and the resource pack in a local directory.""" + """ Plugin that outputs the data pack and the resource pack in a local directory. """ if opts.directory is None: return - paths = [ctx.directory / path for path in opts.directory.entries()] - packs = list(filter(None, ctx.packs)) + # Check both opts and ctx.meta.output for incremental flag + incremental: bool = opts.incremental + if not incremental: + meta_opts = ctx.meta.get("output") + if isinstance(meta_opts, dict): + incremental = bool(meta_opts.get("incremental", False)) + + paths: list[Path] = [ctx.directory / path for path in opts.directory.entries()] + packs: list[Pack] = list(filter(None, ctx.packs)) if paths and packs: with log_time_scope("Output files."): for pack in packs: for path in paths: - pack.save(path, overwrite=True) + if incremental and not pack.zipped and pack.name is not None: + incremental_save(pack, Path(path) / pack.name) + else: + pack.save(path, overwrite=True) + From 5febfcb95bded4b11a62c9ce1ed2127bce0b5fad Mon Sep 17 00:00:00 2001 From: Stoupy51 Date: Sat, 9 May 2026 18:32:25 +0200 Subject: [PATCH 2/3] fix(output): Update incremental option handling and ensure directory management --- src/beet/contrib/output.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/beet/contrib/output.py b/src/beet/contrib/output.py index 453264d6a..471b2975d 100644 --- a/src/beet/contrib/output.py +++ b/src/beet/contrib/output.py @@ -18,7 +18,7 @@ class OutputOptions(PluginOptions): directory: ListOption[FileSystemPath] | None = None - incremental: bool = False + incremental: bool | None = None def beet_default(ctx: Context): @@ -53,6 +53,8 @@ def incremental_save(pack: Pack, output_path: Path) -> None: pass # not empty - leave it # Ensure the root output directory exists + if output_path.exists() and not output_path.is_dir(): + output_path.unlink() output_path.mkdir(parents=True, exist_ok=True) # For each expected file, compare with disk and write if new/changed @@ -67,21 +69,20 @@ def incremental_save(pack: Pack, output_path: Path) -> None: # Existing file: compare before writing else: changed: bool = True - - # Fast path: if the pack file still points directly to a source path, avoid loading content into memory - if ( - pack_file.source_path is not None - and pack_file.source_start is None - and pack_file.source_stop is None - ): - changed = not filecmp.cmp(pack_file.source_path, disk_path, shallow=False) - else: - # Standard path: use beet's own content equality - try: + try: + # Fast path: if the pack file still points directly to a source path, avoid loading content into memory + if ( + pack_file.source_path is not None + and pack_file.source_start is None + and pack_file.source_stop is None + ): + changed = not filecmp.cmp(pack_file.source_path, disk_path, shallow=False) + else: + # Standard path: use beet's own content equality existing_file: PackFile = type(pack_file)(source_path=disk_path) changed = not pack_file.content_equal(existing_file) - except Exception: - changed = True # Fallback to overwrite on any error + except Exception: + changed = True # Fallback to overwrite on any error if changed: pack_file.dump(output_path, rel_path) @@ -94,8 +95,8 @@ def output(ctx: Context, opts: OutputOptions): return # Check both opts and ctx.meta.output for incremental flag - incremental: bool = opts.incremental - if not incremental: + incremental: bool | None = opts.incremental + if incremental is None: meta_opts = ctx.meta.get("output") if isinstance(meta_opts, dict): incremental = bool(meta_opts.get("incremental", False)) From 803f4212ebdb8caa38267ff68337a3439bf1ba1a Mon Sep 17 00:00:00 2001 From: Stoupy51 Date: Sat, 9 May 2026 21:25:37 +0200 Subject: [PATCH 3/3] fix(output): Fixed serialized issues with incremental mode --- src/beet/contrib/output.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/beet/contrib/output.py b/src/beet/contrib/output.py index 471b2975d..19e278577 100644 --- a/src/beet/contrib/output.py +++ b/src/beet/contrib/output.py @@ -78,9 +78,13 @@ def incremental_save(pack: Pack, output_path: Path) -> None: ): changed = not filecmp.cmp(pack_file.source_path, disk_path, shallow=False) else: - # Standard path: use beet's own content equality - existing_file: PackFile = type(pack_file)(source_path=disk_path) - changed = not pack_file.content_equal(existing_file) + # Standard path: compare the **exact** serialized output against disk content. + serialized: str | bytes = pack_file.ensure_serialized() + if isinstance(serialized, str): + encoding: str = getattr(pack_file, "encoding", None) or "utf-8" + changed = disk_path.read_text(encoding=encoding) != serialized + else: + changed = disk_path.read_bytes() != serialized except Exception: changed = True # Fallback to overwrite on any error