diff --git a/scripts/README.md b/scripts/README.md index 3a07b126..ee4dd398 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -91,6 +91,11 @@ python scripts/apache_release.py wheel 0.41.0 0 # Wheel dist python scripts/apache_release.py upload 0.41.0 0 your_apache_id python scripts/apache_release.py upload 0.41.0 0 your_apache_id --dry-run # Test first +# Promote a voted RC from dist/dev to dist/release +python scripts/apache_release.py promote 0.41.0-RC0 your_apache_id +python scripts/apache_release.py promote 0.41.0-RC0 your_apache_id --dry-run +python scripts/apache_release.py promote 0.41.0-RC0 your_apache_id --release-svn-root https://dist.apache.org/repos/dist/release/burr # TLP path override + # Verify artifacts locally python scripts/apache_release.py verify 0.41.0 0 @@ -100,6 +105,38 @@ python scripts/apache_release.py all 0.41.0 0 your_apache_id --no-upload Output: `dist/` directory with tar.gz (archive + sdist), whl, plus .asc and .sha512 files. The wheel is validated with `twine check` to ensure metadata correctness before signing. Install from the whl file to test it out after running the `wheel` subcommand. +## Promoting a voted RC + +After an RC vote passes, promote the exact voted artifacts from Apache SVN `dist/dev` into +`dist/release` with: + +```bash +python scripts/apache_release.py promote 0.41.0-RC0 your_apache_id +``` + +What it does: +- checks out the RC directory from `dist/dev` +- checks out the release directory from `dist/release` +- validates the expected source archive, sdist, wheel, and matching `.asc` / `.sha512` files +- removes the current release contents +- copies the voted RC artifacts into the release checkout +- commits the release checkout to SVN +- prints the final PyPI upload command for the sdist and wheel + +Use `--dry-run` to preview the actions without committing: + +```bash +python scripts/apache_release.py promote 0.41.0-RC0 your_apache_id --dry-run +``` + +For post-incubation path changes, override the default SVN roots: + +```bash +python scripts/apache_release.py promote 0.41.0-RC0 your_apache_id \ + --dev-svn-root https://dist.apache.org/repos/dist/dev/burr \ + --release-svn-root https://dist.apache.org/repos/dist/release/burr +``` + ## For Voters: Verifying a Release If you're voting on a release, follow these steps to verify the release candidate: diff --git a/scripts/apache_release.py b/scripts/apache_release.py index fe6dfb86..d6c9c3e9 100644 --- a/scripts/apache_release.py +++ b/scripts/apache_release.py @@ -38,12 +38,18 @@ import shutil import subprocess import sys +import tempfile from typing import NoReturn, Optional # --- Configuration --- PROJECT_SHORT_NAME = "burr" VERSION_FILE = "pyproject.toml" VERSION_PATTERN = r'version\s*=\s*"(\d+\.\d+\.\d+)"' +DEFAULT_DEV_SVN_ROOT = f"https://dist.apache.org/repos/dist/dev/incubator/{PROJECT_SHORT_NAME}" +DEFAULT_RELEASE_SVN_ROOT = f"https://dist.apache.org/repos/dist/release/incubator/{PROJECT_SHORT_NAME}" +RC_LABEL_PATTERN = re.compile( + r"^(?P\d+\.\d+\.\d+)(?:-incubating)?-RC(?P\d+)$", re.IGNORECASE +) # Required examples for wheel (from pyproject.toml) REQUIRED_EXAMPLES = [ @@ -120,6 +126,17 @@ def _run_command( _fail(f"{error_message}{error_detail}") +def _parse_rc_label(rc_label: str) -> tuple[str, str]: + """Parse an RC label like 0.42.0-RC1 or 0.42.0-incubating-RC1.""" + match = RC_LABEL_PATTERN.fullmatch(rc_label.strip()) + if not match: + _fail( + "Invalid RC label. Expected format like '0.42.0-RC1' " + "or '0.42.0-incubating-RC1'." + ) + return match.group("version"), match.group("rc_num") + + # ============================================================================ # Environment Validation # ============================================================================ @@ -137,6 +154,7 @@ def _validate_environment_for_command(args) -> None: "sdist": ["git", "gpg", "flit"], "wheel": ["git", "gpg", "flit", "node", "npm", "twine"], "upload": ["git", "gpg", "svn"], + "promote": ["svn"], "all": ["git", "gpg", "flit", "node", "npm", "svn", "twine"], "verify": ["git", "gpg", "twine"], } @@ -791,6 +809,166 @@ def _generate_vote_email(version: str, rc_num: str, svn_url: str) -> str: """ +def _promotion_source_url(version: str, rc_num: str, dev_svn_root: str) -> str: + """Return the SVN URL for a voted RC in dist/dev.""" + return f"{dev_svn_root}/{version}-incubating-RC{rc_num}" + + +def _promotion_release_url(release_svn_root: str) -> str: + """Return the SVN URL for the final release location.""" + return release_svn_root + + +def _promotion_commit_message(version: str, rc_num: str) -> str: + """Return the SVN commit message for a promotion.""" + return f"Promote Apache Burr {version}-incubating RC{rc_num} to release" + + +def _expected_promotion_artifact_patterns(version: str) -> dict[str, str]: + """Return the required artifact patterns for a final release promotion.""" + return { + "source_archive": f"apache-burr-{version}-incubating-src.tar.gz", + "sdist": f"apache-burr-{version}-incubating-sdist.tar.gz", + "wheel": f"apache_burr-{version}-*.whl", + } + + +def _promoted_artifact_name(filename: str, rc_num: str) -> str: + """Remove any RC suffix from a promoted artifact name if present.""" + return filename.replace(f"-RC{rc_num}", "") + + +def _find_single_glob_match(directory: str, pattern: str, description: str) -> str: + matches = sorted(glob.glob(os.path.join(directory, pattern))) + if not matches: + _fail(f"Missing required {description}: {pattern}") + if len(matches) > 1: + names = ", ".join(os.path.basename(match) for match in matches) + _fail(f"Expected exactly one {description} for pattern {pattern}, found: {names}") + return matches[0] + + +def _validate_promotion_artifacts(rc_checkout_dir: str, version: str) -> list[str]: + """Validate the expected release artifacts exist in the RC checkout.""" + artifacts: list[str] = [] + patterns = _expected_promotion_artifact_patterns(version) + + source_archive = _find_single_glob_match( + rc_checkout_dir, patterns["source_archive"], "source archive" + ) + sdist = _find_single_glob_match(rc_checkout_dir, patterns["sdist"], "source distribution") + wheel = _find_single_glob_match(rc_checkout_dir, patterns["wheel"], "wheel") + + for artifact_path in [source_archive, sdist, wheel]: + artifacts.append(artifact_path) + for suffix in [".asc", ".sha512"]: + companion_path = f"{artifact_path}{suffix}" + if not os.path.exists(companion_path): + _fail(f"Missing required companion artifact: {os.path.basename(companion_path)}") + artifacts.append(companion_path) + + return sorted(artifacts) + + +def _twine_upload_command(promoted_artifacts: list[str]) -> str: + """Return the PyPI upload command for the final release artifacts.""" + upload_candidates = [ + artifact + for artifact in promoted_artifacts + if artifact.endswith(".whl") or artifact.endswith("-incubating-sdist.tar.gz") + ] + upload_names = " ".join(sorted(os.path.basename(artifact) for artifact in upload_candidates)) + return f"twine upload {upload_names}" + + +def _svn_checkout(url: str, checkout_dir: str) -> None: + """Check out an SVN URL into a local directory.""" + _run_command( + ["svn", "checkout", url, checkout_dir], + description=f"Checking out SVN path: {url}", + error_message=f"SVN checkout failed for {url}", + success_message="SVN checkout completed", + ) + + +def _release_checkout_entries(release_checkout_dir: str) -> list[str]: + """List entries in the release checkout, excluding SVN metadata.""" + entries = [] + for name in sorted(os.listdir(release_checkout_dir)): + if name == ".svn": + continue + entries.append(os.path.join(release_checkout_dir, name)) + return entries + + +def _remove_existing_release_entries(release_checkout_dir: str, dry_run: bool = False) -> list[str]: + """Remove the current dist/release contents from the SVN working copy.""" + removed_entries = [] + for entry in _release_checkout_entries(release_checkout_dir): + removed_entries.append(os.path.basename(entry)) + if dry_run: + print(f" [DRY RUN] Would remove release entry: {os.path.basename(entry)}") + continue + _run_command( + ["svn", "rm", "--force", entry], + description=f"Removing old release entry: {os.path.basename(entry)}", + error_message=f"Failed to remove existing release entry: {entry}", + success_message="Removed", + ) + return removed_entries + + +def _copy_promoted_artifacts( + rc_checkout_dir: str, + release_checkout_dir: str, + artifacts: list[str], + rc_num: str, + dry_run: bool = False, +) -> list[str]: + """Copy validated RC artifacts into the release checkout with final names.""" + copied_artifacts = [] + for artifact in artifacts: + destination_name = _promoted_artifact_name(os.path.basename(artifact), rc_num) + copied_artifacts.append(destination_name) + if dry_run: + print(f" [DRY RUN] Would copy {os.path.basename(artifact)} -> {destination_name}") + continue + + source_path = artifact + destination_path = os.path.join(release_checkout_dir, destination_name) + shutil.copy2(source_path, destination_path) + _run_command( + ["svn", "add", "--force", destination_path], + description=f"Adding promoted artifact: {destination_name}", + error_message=f"Failed to add promoted artifact: {destination_name}", + success_message="Added", + ) + return copied_artifacts + + +def _commit_promoted_release( + release_checkout_dir: str, + version: str, + rc_num: str, + apache_id: str, + dry_run: bool = False, +) -> bool: + """Commit the promoted release checkout to SVN.""" + message = _promotion_commit_message(version, rc_num) + if dry_run: + print(f" [DRY RUN] Would commit release checkout with message: {message}") + return True + + _run_command( + ["svn", "commit", release_checkout_dir, "-m", message, "--username", apache_id], + description="Committing promoted release to SVN...", + error_message="SVN commit failed for promoted release", + success_message="SVN commit completed", + capture_output=False, + ) + return True + + # ============================================================================ # Command Handlers # ============================================================================ @@ -882,6 +1060,64 @@ def cmd_upload(args) -> bool: return True +def cmd_promote(args) -> bool: + """Handle 'promote' subcommand.""" + _print_section(f"Promoting Release Candidate - {args.rc_label}") + _verify_project_root() + + version, rc_num = _parse_rc_label(args.rc_label) + source_url = _promotion_source_url(version, rc_num, args.dev_svn_root) + release_url = _promotion_release_url(args.release_svn_root) + + print(f"Source RC URL: {source_url}") + print(f"Release URL: {release_url}") + if args.dry_run: + print("\n*** DRY RUN MODE ***") + + with tempfile.TemporaryDirectory(prefix="burr-promote-") as temp_dir: + rc_checkout_dir = os.path.join(temp_dir, "rc") + release_checkout_dir = os.path.join(temp_dir, "release") + + _svn_checkout(source_url, rc_checkout_dir) + _svn_checkout(release_url, release_checkout_dir) + + print("\nValidating expected artifacts...") + validated_artifacts = _validate_promotion_artifacts(rc_checkout_dir, version) + for artifact in validated_artifacts: + print(f" ✓ {os.path.basename(artifact)}") + + print("\nCleaning current release artifacts...") + _remove_existing_release_entries(release_checkout_dir, dry_run=args.dry_run) + + print("\nCopying voted RC artifacts into release checkout...") + promoted_artifacts = _copy_promoted_artifacts( + rc_checkout_dir, + release_checkout_dir, + validated_artifacts, + rc_num, + dry_run=args.dry_run, + ) + + print("\nCommitting promoted release...") + _commit_promoted_release( + release_checkout_dir, + version, + rc_num, + args.apache_id, + dry_run=args.dry_run, + ) + + print("\nPromotion summary:") + print(f" Release path: {release_url}") + for artifact_name in promoted_artifacts: + print(f" - {artifact_name}") + + print("\nPyPI upload command:") + print(f" {_twine_upload_command(promoted_artifacts)}") + + return True + + def cmd_verify(args) -> bool: """Handle 'verify' subcommand.""" _print_section(f"Verifying Artifacts - v{args.version}-RC{args.rc_num}") @@ -1008,6 +1244,26 @@ def main(): upload_parser.add_argument("--artifacts-dir", default="dist") upload_parser.add_argument("--dry-run", action="store_true") + # promote subcommand + promote_parser = subparsers.add_parser( + "promote", help="Promote a voted RC from dist/dev to dist/release" + ) + promote_parser.add_argument( + "rc_label", help="Release candidate label, e.g. '0.42.0-RC1' or '0.42.0-incubating-RC1'" + ) + promote_parser.add_argument("apache_id", help="Apache ID") + promote_parser.add_argument("--dry-run", action="store_true") + promote_parser.add_argument( + "--dev-svn-root", + default=DEFAULT_DEV_SVN_ROOT, + help="SVN root for RC artifacts in dist/dev", + ) + promote_parser.add_argument( + "--release-svn-root", + default=DEFAULT_RELEASE_SVN_ROOT, + help="SVN root for promoted artifacts in dist/release", + ) + # verify subcommand verify_parser = subparsers.add_parser("verify", help="Verify artifacts") verify_parser.add_argument("version", help="Version") @@ -1043,6 +1299,8 @@ def main(): success = cmd_wheel(args) elif args.command == "upload": success = cmd_upload(args) + elif args.command == "promote": + success = cmd_promote(args) elif args.command == "verify": success = cmd_verify(args) elif args.command == "all": diff --git a/tests/test_apache_release.py b/tests/test_apache_release.py new file mode 100644 index 00000000..018a4b93 --- /dev/null +++ b/tests/test_apache_release.py @@ -0,0 +1,178 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import sys +from argparse import Namespace +from pathlib import Path + +import pytest + + +def _load_release_module(): + module_path = Path(__file__).resolve().parent.parent / "scripts" / "apache_release.py" + spec = importlib.util.spec_from_file_location("apache_release", module_path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +release = _load_release_module() + + +def _write_artifact_set(directory: Path, version: str, wheel_name: str = None) -> None: + wheel_name = wheel_name or f"apache_burr-{version}-py3-none-any.whl" + artifact_names = [ + f"apache-burr-{version}-incubating-src.tar.gz", + f"apache-burr-{version}-incubating-sdist.tar.gz", + wheel_name, + ] + for artifact_name in artifact_names: + artifact_path = directory / artifact_name + artifact_path.write_bytes(b"artifact") + artifact_path.with_name(f"{artifact_name}.asc").write_text("sig", encoding="utf-8") + artifact_path.with_name(f"{artifact_name}.sha512").write_text("sha", encoding="utf-8") + + +def test_parse_rc_label_accepts_supported_formats(): + assert release._parse_rc_label("0.42.0-RC1") == ("0.42.0", "1") + assert release._parse_rc_label("0.42.0-incubating-RC1") == ("0.42.0", "1") + + +def test_parse_rc_label_rejects_invalid_format(): + with pytest.raises(SystemExit): + release._parse_rc_label("0.42.0") + + +def test_validate_promotion_artifacts_requires_expected_set(tmp_path): + _write_artifact_set(tmp_path, "0.42.0") + + artifacts = release._validate_promotion_artifacts(str(tmp_path), "0.42.0") + + assert len(artifacts) == 9 + assert any(path.endswith("apache-burr-0.42.0-incubating-src.tar.gz") for path in artifacts) + assert any(path.endswith("apache-burr-0.42.0-incubating-sdist.tar.gz") for path in artifacts) + assert any(path.endswith("apache_burr-0.42.0-py3-none-any.whl") for path in artifacts) + + +def test_validate_promotion_artifacts_fails_when_companion_missing(tmp_path): + _write_artifact_set(tmp_path, "0.42.0") + (tmp_path / "apache-burr-0.42.0-incubating-src.tar.gz.asc").unlink() + + with pytest.raises(SystemExit): + release._validate_promotion_artifacts(str(tmp_path), "0.42.0") + + +def test_promoted_artifact_name_removes_rc_suffix(): + assert release._promoted_artifact_name("apache-burr-0.42.0-RC1.txt", "1") == "apache-burr-0.42.0.txt" + assert release._promoted_artifact_name("apache_burr-0.42.0-py3-none-any.whl", "1") == ( + "apache_burr-0.42.0-py3-none-any.whl" + ) + + +def test_twine_upload_command_includes_only_sdist_and_wheel(): + command = release._twine_upload_command( + [ + "apache-burr-0.42.0-incubating-src.tar.gz", + "apache-burr-0.42.0-incubating-src.tar.gz.asc", + "apache-burr-0.42.0-incubating-sdist.tar.gz", + "apache_burr-0.42.0-py3-none-any.whl", + ] + ) + + assert command == ( + "twine upload apache-burr-0.42.0-incubating-sdist.tar.gz " + "apache_burr-0.42.0-py3-none-any.whl" + ) + + +def test_cmd_promote_dry_run_plans_without_committing(monkeypatch, tmp_path): + calls = {"checkout": [], "remove": None, "copy": None, "commit": None} + + class _TempDir: + def __enter__(self): + return str(tmp_path) + + def __exit__(self, exc_type, exc, tb): + return False + + def fake_checkout(url: str, checkout_dir: str): + calls["checkout"].append((url, checkout_dir)) + Path(checkout_dir).mkdir(parents=True, exist_ok=True) + + def fake_validate(rc_checkout_dir: str, version: str): + assert version == "0.42.0" + return [ + f"{rc_checkout_dir}/apache-burr-0.42.0-incubating-src.tar.gz", + f"{rc_checkout_dir}/apache-burr-0.42.0-incubating-src.tar.gz.asc", + f"{rc_checkout_dir}/apache-burr-0.42.0-incubating-src.tar.gz.sha512", + f"{rc_checkout_dir}/apache-burr-0.42.0-incubating-sdist.tar.gz", + f"{rc_checkout_dir}/apache-burr-0.42.0-incubating-sdist.tar.gz.asc", + f"{rc_checkout_dir}/apache-burr-0.42.0-incubating-sdist.tar.gz.sha512", + f"{rc_checkout_dir}/apache_burr-0.42.0-py3-none-any.whl", + f"{rc_checkout_dir}/apache_burr-0.42.0-py3-none-any.whl.asc", + f"{rc_checkout_dir}/apache_burr-0.42.0-py3-none-any.whl.sha512", + ] + + def fake_remove(release_checkout_dir: str, dry_run: bool = False): + calls["remove"] = (release_checkout_dir, dry_run) + return ["old-release"] + + def fake_copy( + rc_checkout_dir: str, + release_checkout_dir: str, + artifacts: list[str], + rc_num: str, + dry_run: bool = False, + ): + calls["copy"] = (rc_checkout_dir, release_checkout_dir, list(artifacts), rc_num, dry_run) + return [Path(artifact).name for artifact in artifacts] + + def fake_commit( + release_checkout_dir: str, + version: str, + rc_num: str, + apache_id: str, + dry_run: bool = False, + ): + calls["commit"] = (release_checkout_dir, version, rc_num, apache_id, dry_run) + return True + + monkeypatch.setattr(release.tempfile, "TemporaryDirectory", lambda prefix=None: _TempDir()) + monkeypatch.setattr(release, "_svn_checkout", fake_checkout) + monkeypatch.setattr(release, "_validate_promotion_artifacts", fake_validate) + monkeypatch.setattr(release, "_remove_existing_release_entries", fake_remove) + monkeypatch.setattr(release, "_copy_promoted_artifacts", fake_copy) + monkeypatch.setattr(release, "_commit_promoted_release", fake_commit) + + args = Namespace( + rc_label="0.42.0-RC1", + apache_id="hari", + dry_run=True, + dev_svn_root="https://dist.apache.org/repos/dist/dev/incubator/burr", + release_svn_root="https://dist.apache.org/repos/dist/release/incubator/burr", + ) + + assert release.cmd_promote(args) is True + assert calls["checkout"][0][0].endswith("/0.42.0-incubating-RC1") + assert calls["checkout"][1][0] == "https://dist.apache.org/repos/dist/release/incubator/burr" + assert calls["remove"][1] is True + assert calls["copy"][3] == "1" + assert calls["copy"][4] is True + assert calls["commit"] == (str(tmp_path / "release"), "0.42.0", "1", "hari", True)