cpython#121277: Replace next versions in docs by the just-released version

release.py

@@ -209,6 +209,21 @@ def committed_at(self) -> datetime.datetime:
             int(proc.stdout.decode().strip()), tz=datetime.timezone.utc
         )
 
+    @property
+    def includes_docs(self) -> bool:
+        """True if docs should be included in the release"""
+        return self.is_final or self.is_release_candidate
+
+    @property
+    def doc_version(self) -> str:
+        """Text used for notes in docs like 'Added in x.y'"""
+        # - ignore levels (alpha/beta/rc are preparation for the full release)
+        # - use just X.Y for patch 0
+        if self.patch == 0:
+            return f"{self.major}.{self.minor}"
+        else:
+            return f"{self.major}.{self.minor}.{self.patch}"
+
 
 def error(*msgs: str) -> None:
     print("**ERROR**", file=sys.stderr)

run_release.py

@@ -31,6 +31,7 @@
 
 import release as release_mod
 import sbom
+import update_version_next
 from buildbotapi import BuildBotAPI, Builder
 from release import ReleaseShelf, Tag, Task
 
@@ -497,6 +498,13 @@
     )
 
 
+def bump_version_in_docs(db: ReleaseShelf) -> None:
+    update_version_next.main([db["release"].doc_version, str(db["git_repo"])])
+    subprocess.check_call(
+        ["git", "commit", "-a", "--amend", "--no-edit"], cwd=db["git_repo"]
+    )
+
+
 def create_tag(db: ReleaseShelf) -> None:
     with cd(db["git_repo"]):
         if not release_mod.make_tag(db["release"], sign_gpg=db["sign_gpg"]):
@@ -509,7 +517,7 @@
 def wait_for_source_and_docs_artifacts(db: ReleaseShelf) -> None:
     # Determine if we need to wait for docs or only source artifacts.
     release_tag = db["release"]
-    should_wait_for_docs = release_tag.is_final or release_tag.is_release_candidate
+    should_wait_for_docs = release_tag.includes_docs
 
     # Create the directory so it's easier to place the artifacts there.
     release_path = Path(db["git_repo"] / str(release_tag))
@@ -549,6 +557,31 @@
         time.sleep(1)
 
 
+def check_doc_unreleased_version(db: ReleaseShelf) -> None:
+    print("Checking built docs for '(unreleased)'")
+    # This string is generated when a `versionadded:: next` directive is
+    # left in the docs, which means the `bump_version_in_docs` step
+    # didn't do its job.
+    # But, there could also be a false positive.
+    release_tag = db["release"]
+    docs_path = Path(db["git_repo"]) / str(release_tag) / "docs"
+    archive_path = docs_path / f"python-{release_tag}-docs-html.tar.bz2"
+    if release_tag.includes_docs:
+        assert archive_path.exists()
+    if archive_path.exists():
+        proc = subprocess.run(
+            [
+                "tar",
+                "-xjf",
+                archive_path,
+                '--to-command=! grep -Hn --label="$TAR_FILENAME" "[(]unreleased[)]"',
+            ],
+        )
+        if proc.returncode != 0:
+            if not ask_question("Are these `(unreleased)` strings in built docs OK?"):
+                raise AssertionError("`(unreleased)` strings found in docs")
+
+
 def sign_source_artifacts(db: ReleaseShelf) -> None:
     print("Signing tarballs with GPG")
     uid = os.environ.get("GPG_KEY_FOR_RELEASE")
@@ -1251,6 +1284,7 @@
         Task(check_cpython_repo_is_clean, "Checking Git repository is clean"),
         Task(prepare_pydoc_topics, "Preparing pydoc topics"),
         Task(bump_version, "Bump version"),
+        Task(bump_version_in_docs, "Bump version in docs"),
         Task(check_cpython_repo_is_clean, "Checking Git repository is clean"),
         Task(run_autoconf, "Running autoconf"),
         Task(check_cpython_repo_is_clean, "Checking Git repository is clean"),
@@ -1270,6 +1304,7 @@
             wait_for_source_and_docs_artifacts,
             "Wait for source and docs artifacts to build",
         ),
+        Task(check_doc_unreleased_version, "Check docs for `(unreleased)`"),
         Task(build_sbom_artifacts, "Building SBOM artifacts"),
         *([] if no_gpg else [Task(sign_source_artifacts, "Sign source artifacts")]),
         Task(upload_files_to_server, "Upload files to the PSF server"),

tests/test_release_tag.py

@@ -91,3 +91,25 @@ def test_tag_invalid() -> None:
     # Act / Assert
     with pytest.raises(SystemExit):
         release.Tag(tag_name)
+
+
+def test_tag_docs_attributes() -> None:
+    # Arrange
+    alpha = release.Tag("3.13.0a7")
+    beta = release.Tag("3.13.0b1")
+    rc = release.Tag("3.13.0rc3")
+    final_zero = release.Tag("3.13.0")
+    final_3 = release.Tag("3.13.3")
+
+    # Act / Assert
+    assert alpha.includes_docs is False
+    assert beta.includes_docs is False
+    assert rc.includes_docs is True
+    assert final_zero.includes_docs is True
+    assert final_3.includes_docs is True
+
+    assert alpha.doc_version == "3.13"
+    assert beta.doc_version == "3.13"
+    assert rc.doc_version == "3.13"
+    assert final_zero.doc_version == "3.13"
+    assert final_3.doc_version == "3.13.3"

tests/test_run_release.py

@@ -1,3 +1,7 @@
+import builtins
+import contextlib
+import io
+import tarfile
 from pathlib import Path
 from typing import cast
 
@@ -37,3 +41,86 @@ def test_check_magic_number() -> None:
         run_release.ReleaseException, match="Magic numbers in .* don't match"
     ):
         run_release.check_magic_number(cast(ReleaseShelf, db))
+
+
+def prepare_fake_docs(tmp_path: Path, content: str) -> None:
+    docs_path = tmp_path / "3.13.0rc1/docs"
+    docs_path.mkdir(parents=True)
+    tarball = tarfile.open(docs_path / "python-3.13.0rc1-docs-html.tar.bz2", "w:bz2")
+    with tarball:
+        tarinfo = tarfile.TarInfo("index.html")
+        tarinfo.size = len(content)
+        tarball.addfile(tarinfo, io.BytesIO(content.encode()))
+
+
+@contextlib.contextmanager
+def fake_answers(monkeypatch: pytest.MonkeyPatch, answers: list[str]) -> None:
+    """Monkey-patch input() to give the given answers. All must be consumed."""
+
+    answers_left = list(answers)
+
+    def fake_input(question):
+        print(question, "--", answers_left[0])
+        return answers_left.pop(0)
+
+    with monkeypatch.context() as ctx:
+        ctx.setattr(builtins, "input", fake_input)
+        yield
+    assert answers_left == []
+
+
+def test_check_doc_unreleased_version_no_file(tmp_path: Path) -> None:
+    db = {
+        "release": Tag("3.13.0rc1"),
+        "git_repo": str(tmp_path),
+    }
+    with pytest.raises(AssertionError):
+        # There should be a docs artefact available
+        run_release.check_doc_unreleased_version(cast(ReleaseShelf, db))
+
+
+def test_check_doc_unreleased_version_no_file_alpha(tmp_path: Path) -> None:
+    db = {
+        "release": Tag("3.13.0a1"),
+        "git_repo": str(tmp_path),
+    }
+    # No docs artefact needed for alphas
+    run_release.check_doc_unreleased_version(cast(ReleaseShelf, db))
+
+
+def test_check_doc_unreleased_version_ok(monkeypatch, tmp_path: Path) -> None:
+    prepare_fake_docs(
+        tmp_path,
+        "<div>New in 3.13</div>",
+    )
+    db = {
+        "release": Tag("3.13.0rc1"),
+        "git_repo": str(tmp_path),
+    }
+    run_release.check_doc_unreleased_version(cast(ReleaseShelf, db))
+
+
+def test_check_doc_unreleased_version_not_ok(monkeypatch, tmp_path: Path) -> None:
+    prepare_fake_docs(
+        tmp_path,
+        "<div>New in 3.13.0rc1 (unreleased)</div>",
+    )
+    db = {
+        "release": Tag("3.13.0rc1"),
+        "git_repo": str(tmp_path),
+    }
+    with fake_answers(monkeypatch, ["no"]), pytest.raises(AssertionError):
+        run_release.check_doc_unreleased_version(cast(ReleaseShelf, db))
+
+
+def test_check_doc_unreleased_version_waived(monkeypatch, tmp_path: Path) -> None:
+    prepare_fake_docs(
+        tmp_path,
+        "<div>New in 3.13.0rc1 (unreleased)</div>",
+    )
+    db = {
+        "release": Tag("3.13.0rc1"),
+        "git_repo": str(tmp_path),
+    }
+    with fake_answers(monkeypatch, ["yes"]):
+        run_release.check_doc_unreleased_version(cast(ReleaseShelf, db))

tests/test_update_version_next.py

@@ -0,0 +1,74 @@
+"""Tests for the update_version_next tool."""
+
+from pathlib import Path
+
+import update_version_next
+
+TO_CHANGE = """
+Directives to change
+--------------------
+
+Here, all occurences of NEXT (lowercase) should be changed:
+
+.. versionadded:: next
+
+.. versionchanged:: next
+
+.. deprecated:: next
+
+.. deprecated-removed:: next 4.0
+
+whitespace:
+
+..   versionchanged:: next
+
+.. versionchanged  :: next
+
+    .. versionadded:: next
+
+arguments:
+
+.. versionadded:: next
+    Foo bar
+
+.. versionadded:: next as ``previousname``
+"""
+
+UNCHANGED = """
+Unchanged
+---------
+
+Here, the word "next" should NOT be changed:
+
+.. versionchanged:: NEXT
+
+..versionchanged:: NEXT
+
+... versionchanged:: next
+
+foo .. versionchanged:: next
+
+.. otherdirective:: next
+
+.. VERSIONCHANGED: next
+
+.. deprecated-removed: 3.0 next
+"""
+
+EXPECTED_CHANGED = TO_CHANGE.replace("next", "VER")
+
+
+def test_freeze_simple_script(tmp_path: Path) -> None:
+    p = tmp_path.joinpath
+
+    p("source.rst").write_text(TO_CHANGE + UNCHANGED)
+    p("subdir").mkdir()
+    p("subdir/change.rst").write_text(".. versionadded:: next")
+    p("subdir/keep.not-rst").write_text(".. versionadded:: next")
+    p("subdir/keep.rst").write_text("nothing to see here")
+    args = ["VER", str(tmp_path)]
+    update_version_next.main(args)
+    assert p("source.rst").read_text() == EXPECTED_CHANGED + UNCHANGED
+    assert p("subdir/change.rst").read_text() == ".. versionadded:: VER"
+    assert p("subdir/keep.not-rst").read_text() == ".. versionadded:: next"
+    assert p("subdir/keep.rst").read_text() == "nothing to see here"

update_version_next.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""
+Replace `.. versionchanged:: next` lines in docs files by the given version.
+
+Run this at release time to replace `next` with the just-released version
+in the sources.
+
+No backups are made; add/commit to Git before running the script.
+
+Applies to all the VersionChange directives. For deprecated-removed, only
+handle the first argument (deprecation version, not the removal version).
+
+"""
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+DIRECTIVE_RE = re.compile(
+    r"""
+        (?P<before>
+            \s*\.\.\s+
+            (version(added|changed|removed)|deprecated(-removed)?)
+            \s*::\s*
+        )
+        next
+        (?P<after>
+            .*
+        )
+    """,
+    re.VERBOSE | re.DOTALL,
+)
+
+parser = argparse.ArgumentParser(
+    description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+)
+parser.add_argument(
+    "version",
+    help='String to replace "next" with. Usually `x.y`, but can be anything.',
+)
+parser.add_argument(
+    "directory",
+    type=Path,
+    help="Directory to process",
+)
+parser.add_argument(
+    "--verbose",
+    "-v",
+    action="count",
+    default=0,
+    help="Increase verbosity. Can be repeated (`-vv`).",
+)
+
+
+def main(argv: list[str]) -> None:
+    args = parser.parse_args(argv)
+    version = args.version
+    if args.verbose:
+        print(
+            f'Updating "next" versions in {args.directory} to {version!r}',
+            file=sys.stderr,
+        )
+    for path in Path(args.directory).glob("**/*.rst"):
+        num_changed_lines = 0
+        lines = []
+        with open(path, encoding="utf-8") as file:
+            for lineno, line in enumerate(file, start=1):
+                try:
+                    if match := DIRECTIVE_RE.fullmatch(line):
+                        line = match["before"] + version + match["after"]
+                        num_changed_lines += 1
+                    lines.append(line)
+                except Exception as exc:
+                    exc.add_note(f"processing line {path}:{lineno}")
+                    raise
+        if num_changed_lines:
+            if args.verbose:
+                s = "" if num_changed_lines == 1 else "s"
+                print(
+                    f"Updating file {path} ({num_changed_lines} change{s})",
+                    file=sys.stderr,
+                )
+            with open(path, "w", encoding="utf-8") as file:
+                file.writelines(lines)
+        else:
+            if args.verbose > 1:
+                print(f"Unchanged file {path}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])