diff --git a/ci/repartition-index.yml b/ci/repartition-index.yml index c498c55..e7538a2 100644 --- a/ci/repartition-index.yml +++ b/ci/repartition-index.yml @@ -45,6 +45,8 @@ stages: - powershell: | cd (mkdir -Force index) python "$(Build.SourcesDirectory)\scripts\repartition-index.py" --windows-default + # Show the report + cat index-windows.txt displayName: 'Repartition index' workingDirectory: $(Build.BinariesDirectory) diff --git a/scripts/repartition-index.py b/scripts/repartition-index.py index a255810..b8faa14 100644 --- a/scripts/repartition-index.py +++ b/scripts/repartition-index.py @@ -1,10 +1,9 @@ +import contextlib import json import re import sys -from collections import OrderedDict from pathlib import Path -from urllib.request import Request, urlopen REPO = Path(__file__).absolute().parent.parent sys.path.append(str(REPO / "src")) @@ -30,6 +29,7 @@ def usage(): print(" -t/--tag TAG Include only the specified tags (comma-separated)") print(" -r/--range RANGE Include only the specified range (comma-separated)") print(" --latest-micro Include only the latest x.y.z version") + print(" --report Write plain-text summary report") print() print("An output of 'nul' is permitted to drop entries.") print("Providing the same inputs and outputs is permitted, as all inputs are read") @@ -93,14 +93,15 @@ def __init__(self): self.allow_dup = False self.only_dup = False self.pre = False - self.tag_or_range = None + self.tag_or_range = [] self._expect_tag_or_range = False self.latest_micro = False + self.report = False def add_arg(self, arg): if arg[:1] != "-": if self._expect_tag_or_range: - self.tag_or_range = tag_or_range(arg) + self.tag_or_range.append(tag_or_range(arg)) self._expect_tag_or_range = False return False self.target = arg @@ -121,9 +122,16 @@ def add_arg(self, arg): if arg == "--latest-micro": self.latest_micro = True return False + if arg == "--report": + self.report = True + return False raise ValueError("Unknown argument: " + arg) def execute(self, versions, context): + if self.report: + if self.target != "nul": + context.setdefault("reports", []).append(self.target) + return written = context.setdefault("written", set()) written_now = set() outputs = context.setdefault("outputs", {}) @@ -150,7 +158,8 @@ def execute(self, versions, context): if not self.pre and v.is_prerelease: continue if self.tag_or_range and not any( - self.tag_or_range.satisfied_by(CompanyTag(i["company"], t)) + r.satisfied_by(CompanyTag(i["company"], t)) + for r in self.tag_or_range for t in i["install-for"] ): continue @@ -176,19 +185,56 @@ def add_arg(self, arg): return False raise ValueError("Unknown argument: " + arg) + @contextlib.contextmanager + def open(self, file): + file = Path(file) + if file.match("nul"): + import io + yield io.StringIO() + elif file.match("stdout"): + yield sys.stdout + else: + with open(file, "w", encoding="utf-8") as f: + yield f + + def st_size(self, file): + file = Path(file) + if file.match("nul"): + return "no data written" + if file.match("stdout"): + return "n/a" + return f"{Path(file).stat().st_size} bytes" + def execute(self, versions, context): outputs = context.get("outputs") or {} output_order = context.get("output_order", []) + report_data = {} for target, next_target in zip(output_order, [*output_order[1:], None]): data = { "versions": outputs[target] } if next_target: data["next"] = next_target - with open(target, "w", encoding="utf-8") as f: + for i in outputs[target]: + report_data.setdefault(target, {}).setdefault(i["sort-version"].casefold(), []).append(i) + with self.open(target) as f: json.dump(data, f, indent=self.indent) print("Wrote {} ({} entries, {} bytes)".format( - target, len(data["versions"]), Path(target).stat().st_size + target, len(data["versions"]), self.st_size(target) + )) + + reports = context.get("reports", []) + for target in reports: + with self.open(target) as f: + for output_target in output_order: + print("Written to", output_target, file=f) + data = report_data[output_target] + for key in data: + ids = ", ".join(i["id"] for i in data[key]) + print("{}: {}".format(key, ids), file=f) + print(file=f) + print("Wrote {} ({} bytes)".format( + target, self.st_size(target) )) @@ -203,14 +249,17 @@ def parse_cli(args): print("Using equivalent of: --pre --latest-micro -r >=3.11.0 index-windows.json") print(" --pre -r >=3.11.0 index-windows-recent.json") print(" index-windows-legacy.json") - plan_split = [SplitToFile(), SplitToFile(), SplitToFile()] + print(" --report index-windows.txt") + plan_split = [SplitToFile(), SplitToFile(), SplitToFile(), SplitToFile()] plan_split[0].target = "index-windows.json" plan_split[1].target = "index-windows-recent.json" plan_split[2].target = "index-windows-legacy.json" + plan_split[3].target = "index-windows.txt" + plan_split[3].report = True plan_split[0].pre = plan_split[1].pre = plan_split[2].pre = True plan_split[0].latest_micro = True - plan_split[0].tag_or_range = tag_or_range(">=3.11.0") - plan_split[1].tag_or_range = tag_or_range(">=3.11.0") + plan_split[0].tag_or_range = [tag_or_range(">=3.11"), tag_or_range(">=3.13t")] + plan_split[1].tag_or_range = [tag_or_range(">=3.11"), tag_or_range(">=3.13t")] elif a == "-i": action = ReadFile() plan_read.append(action) @@ -246,4 +295,3 @@ def parse_cli(args): CONTEXT = {} for p in plan: p.execute(VERSIONS, CONTEXT) -