From 859c89f5e7874ff9212928200762ed108026a638 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 9 Feb 2025 16:30:38 +0100 Subject: [PATCH 01/19] minimal changes needed for PGO with clang-cl --- PCbuild/pyproject.props | 6 +++++- PCbuild/pythoncore.vcxproj | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index dbdb6b743bea37..d945afade9eb47 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -10,6 +10,8 @@ $(MSBuildThisFileDirectory)obj\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\ $(IntDir.Replace(`\\`, `\`)) + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_PGInstrument\__clang_profiles\ + $(ClangProfileDir.Replace(`\\`, `\`)) $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\ @@ -70,6 +72,8 @@ /utf-8 %(AdditionalOptions) -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) -flto %(AdditionalOptions) + -fprofile-instr-generate=$(ClangProfileDir)\default_%m.profraw %(AdditionalOptions) + -fprofile-instr-use=$(ClangProfileDir)\profdata.profdata %(AdditionalOptions) -d2pattern-opt-disable:-932189325 %(AdditionalOptions) -d2ssa-patterns-all- %(AdditionalOptions) /sourceDependencies "$(IntDir.Trim(`\`))" %(AdditionalOptions) @@ -185,7 +189,7 @@ public override bool Execute() { Targets="CleanAll" /> - + <_PGCFiles Include="$(OutDir)instrumented\$(TargetName)!*.pgc" /> <_PGDFile Include="$(OutDir)instrumented\$(TargetName).pgd" /> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 9ebf58ae8a9bc4..b05c20d638a694 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -420,6 +420,7 @@ HACL_CAN_COMPILE_SIMD128;%(PreprocessorDefinitions) HACL_CAN_COMPILE_SIMD256;%(PreprocessorDefinitions) + /arch:AVX @@ -716,6 +717,18 @@ + + + + + + + + + git From a529c3905a59453d6986c9b4d5cfcf1206f10067 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 9 Feb 2025 16:32:36 +0100 Subject: [PATCH 02/19] like for MSVC, don't use link time optimization for _freeze_module in case of clang-cl to speed up the build --- PCbuild/_freeze_module.vcxproj | 1 + 1 file changed, 1 insertion(+) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 51b493f8a84c6f..8520f563d396fa 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -92,6 +92,7 @@ $(IntDir);%(AdditionalIncludeDirectories) Disabled false + %(AdditionalOptions) -fno-lto Console From 26fb51f81101fd361411b8e3ae53a3ce0d8bceb1 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 9 Feb 2025 16:34:56 +0100 Subject: [PATCH 03/19] Do not build _freeze_module twice in case of PGO Speeds up both MSVC and clang-cl builds. Should most probably done in a separate PR and issue, though. --- PCbuild/pcbuild.proj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index a2a637a3044373..c7ddc1d23b301c 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -95,7 +95,8 @@ - Date: Mon, 10 Feb 2025 20:13:15 +0100 Subject: [PATCH 04/19] remove /arch:AVX for blake2module.c again I've previously gotten compile errors from clang, because the needed intrinsics were not available without that option. Cannot reproduce anymore. Most probably, because I've upgraded to Visual Studio 17.13.0 Preview 5.0, which now ships with clang 19.1.1 instead of 18.1.8 and they've done that for compatibility with MSVC? Anyway, let's keep the PR small :) --- PCbuild/pythoncore.vcxproj | 1 - 1 file changed, 1 deletion(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index b05c20d638a694..3a7ec2f63b5d2a 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -420,7 +420,6 @@ HACL_CAN_COMPILE_SIMD128;%(PreprocessorDefinitions) HACL_CAN_COMPILE_SIMD256;%(PreprocessorDefinitions) - /arch:AVX From 7b46aeb149f785cfddf2c21bb7e88415f6431eae Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Mon, 10 Feb 2025 20:18:02 +0100 Subject: [PATCH 05/19] Revert "Do not build _freeze_module twice in case of PGO" This reverts commit 26fb51f81101fd361411b8e3ae53a3ce0d8bceb1. Shall be done in a separate PR. --- PCbuild/pcbuild.proj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index c7ddc1d23b301c..a2a637a3044373 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -95,8 +95,7 @@ - Date: Mon, 10 Feb 2025 21:26:18 +0100 Subject: [PATCH 06/19] always clean the profile directory when doing a new clang PGO build This better matches the behaviour of build.bat in case of MSVC PGO builds. --- PCbuild/pythoncore.vcxproj | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 3a7ec2f63b5d2a..75c07f3ffa5b4c 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -717,8 +717,16 @@ - + Condition="$(PlatformToolset) == 'ClangCL' and $(Configuration) == 'PGInstrument' and !Exists($(ClangProfileDir))"> + + + + + + + + Date: Thu, 13 Feb 2025 19:23:15 +0100 Subject: [PATCH 07/19] blurb it --- .../next/Build/2025-02-13-19-21-41.gh-issue-130090.3ngJaV.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2025-02-13-19-21-41.gh-issue-130090.3ngJaV.rst diff --git a/Misc/NEWS.d/next/Build/2025-02-13-19-21-41.gh-issue-130090.3ngJaV.rst b/Misc/NEWS.d/next/Build/2025-02-13-19-21-41.gh-issue-130090.3ngJaV.rst new file mode 100644 index 00000000000000..9c87bcc68423db --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-02-13-19-21-41.gh-issue-130090.3ngJaV.rst @@ -0,0 +1,2 @@ +Building with ``PlatformToolset=ClangCL`` on Windows now supports PGO +(profile guided optimization). Patch by Chris Eibl. From ad72df5496aa44b5c2914551e5009be8c9f146ab Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Thu, 13 Feb 2025 19:30:53 +0100 Subject: [PATCH 08/19] Adding myself to ACKS --- Misc/ACKS | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/ACKS b/Misc/ACKS index 2a68b69f161041..7bf1d9c99ea24c 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -504,6 +504,7 @@ Grant Edwards Vlad Efanov Zvi Effron John Ehresman +Chris Eibl Tal Einat Eric Eisner Andrew Eland From 74ec74e889066e53ad0767dbd711828de89bc650 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Thu, 13 Feb 2025 23:24:34 +0100 Subject: [PATCH 09/19] extract pyproject-clangcl.props --- PCbuild/pyproject-clangcl.props | 21 +++++++++++++++++++++ PCbuild/pyproject.props | 7 +------ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 PCbuild/pyproject-clangcl.props diff --git a/PCbuild/pyproject-clangcl.props b/PCbuild/pyproject-clangcl.props new file mode 100644 index 00000000000000..dbbf0e833f3be6 --- /dev/null +++ b/PCbuild/pyproject-clangcl.props @@ -0,0 +1,21 @@ + + + + <__PyprojectClangCl_Props_Imported>true + + + + $(MSBuildThisFileDirectory)obj\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_PGInstrument\__clang_profiles\ + $(ClangProfileDir.Replace(`\\`, `\`)) + + + + + -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) + -flto %(AdditionalOptions) + -fprofile-instr-generate=$(ClangProfileDir)\default_%m.profraw %(AdditionalOptions) + -fprofile-instr-use=$(ClangProfileDir)\profdata.profdata %(AdditionalOptions) + + + + diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 20794201d91c52..95f4759cd289ea 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -1,6 +1,7 @@ + <__PyProject_Props_Imported>true <_ProjectFileVersion>10.0.30319.1 @@ -10,8 +11,6 @@ $(MSBuildThisFileDirectory)obj\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\ $(IntDir.Replace(`\\`, `\`)) - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_PGInstrument\__clang_profiles\ - $(ClangProfileDir.Replace(`\\`, `\`)) $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\ @@ -71,10 +70,6 @@ $(EnableControlFlowGuard) true /utf-8 %(AdditionalOptions) - -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) - -flto %(AdditionalOptions) - -fprofile-instr-generate=$(ClangProfileDir)\default_%m.profraw %(AdditionalOptions) - -fprofile-instr-use=$(ClangProfileDir)\profdata.profdata %(AdditionalOptions) -d2pattern-opt-disable:-932189325 %(AdditionalOptions) -d2ssa-patterns-all- %(AdditionalOptions) /sourceDependencies "$(IntDir.Trim(`\`))" %(AdditionalOptions) From 8c8aa7934dcbd33d1f816fe71546f7ef8249e736 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 16 Feb 2025 11:55:53 +0100 Subject: [PATCH 10/19] move MergeClangProfileData to pyproject-clangcl.props and make it a target with inputs and outputs --- PCbuild/pyproject-clangcl.props | 32 +++++++++++++++++++++++++------- PCbuild/pythoncore.vcxproj | 20 -------------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/PCbuild/pyproject-clangcl.props b/PCbuild/pyproject-clangcl.props index dbbf0e833f3be6..cf62c46658c475 100644 --- a/PCbuild/pyproject-clangcl.props +++ b/PCbuild/pyproject-clangcl.props @@ -2,19 +2,37 @@ <__PyprojectClangCl_Props_Imported>true - - - - $(MSBuildThisFileDirectory)obj\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_PGInstrument\__clang_profiles\ - $(ClangProfileDir.Replace(`\\`, `\`)) + + <_profrawFiles Include="$(OutDir)instrumented\$(TargetName)_*.profraw" /> + + + + + + + + + + + + + + + -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) -flto %(AdditionalOptions) - -fprofile-instr-generate=$(ClangProfileDir)\default_%m.profraw %(AdditionalOptions) - -fprofile-instr-use=$(ClangProfileDir)\profdata.profdata %(AdditionalOptions) + -fprofile-instr-generate=$(OutDir)$(TargetName)_%m.profraw %(AdditionalOptions) + -fprofile-instr-use=$(OutDir)instrumented\profdata.profdata %(AdditionalOptions) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 75c07f3ffa5b4c..9ebf58ae8a9bc4 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -716,26 +716,6 @@ - - - - - - - - - - - - - - - - git From 2a9da272968c585559d38fc1b3ba729f8ec5daae Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 16 Feb 2025 11:58:33 +0100 Subject: [PATCH 11/19] rename RequirePGCFiles to RequireProfileData because the name is too MSVC specific --- PCbuild/pyproject-clangcl.props | 2 +- PCbuild/pyproject.props | 2 +- PCbuild/pythoncore.vcxproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/PCbuild/pyproject-clangcl.props b/PCbuild/pyproject-clangcl.props index cf62c46658c475..abef887a6a960a 100644 --- a/PCbuild/pyproject-clangcl.props +++ b/PCbuild/pyproject-clangcl.props @@ -11,7 +11,7 @@ + Condition="$(RequireProfileData) == 'true' and @(_profrawFiles) == ''" /> + Condition="$(RequireProfileData) == 'true' and @(_PGCFiles) == ''" /> true - true + true true false From ea4de96c40adc03ecd67a651c7a727f28bd6933f Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Tue, 18 Feb 2025 20:17:57 +0100 Subject: [PATCH 12/19] Apply suggestions from code review Co-authored-by: Steve Dower --- PCbuild/pyproject-clangcl.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/PCbuild/pyproject-clangcl.props b/PCbuild/pyproject-clangcl.props index abef887a6a960a..7a38bd4380da9f 100644 --- a/PCbuild/pyproject-clangcl.props +++ b/PCbuild/pyproject-clangcl.props @@ -17,13 +17,13 @@ + Outputs="$(OutDir)instrumented\profdata.profdata"> + Command='"$(LLVMInstallDir)\bin\llvm-profdata.exe" merge -output="$(OutDir)instrumented\profdata.profdata" "$(OutDir)instrumented\*_*.profraw"' /> - + From 3346b9d84673646dd295e8be22dd05a362b2a588 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Tue, 18 Feb 2025 20:47:43 +0100 Subject: [PATCH 13/19] Use -Wno-profile-instr-unprofiled in the PGUpdate case --- PCbuild/pyproject-clangcl.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/pyproject-clangcl.props b/PCbuild/pyproject-clangcl.props index 7a38bd4380da9f..26273096010f35 100644 --- a/PCbuild/pyproject-clangcl.props +++ b/PCbuild/pyproject-clangcl.props @@ -32,7 +32,7 @@ -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) -flto %(AdditionalOptions) -fprofile-instr-generate=$(OutDir)$(TargetName)_%m.profraw %(AdditionalOptions) - -fprofile-instr-use=$(OutDir)instrumented\profdata.profdata %(AdditionalOptions) + -fprofile-instr-use=$(OutDir)instrumented\profdata.profdata -Wno-profile-instr-unprofiled %(AdditionalOptions) From 9db1a297d95574cc3114854c8c427736d9521917 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Wed, 19 Feb 2025 06:40:46 +0100 Subject: [PATCH 14/19] introduce CLANG_PROFILE_PATH --- PCbuild/pyproject-clangcl.props | 11 ++++++++++- PCbuild/pyproject.props | 4 +++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/PCbuild/pyproject-clangcl.props b/PCbuild/pyproject-clangcl.props index 26273096010f35..21e591b7ca403d 100644 --- a/PCbuild/pyproject-clangcl.props +++ b/PCbuild/pyproject-clangcl.props @@ -4,6 +4,15 @@ <__PyprojectClangCl_Props_Imported>true + + + $(OutDir) + <_CLANG_PROFILE_PATH>$(CLANG_PROFILE_PATH) + <_CLANG_PROFILE_PATH Condition="!HasTrailingSlash($(_CLANG_PROFILE_PATH))">$(_CLANG_PROFILE_PATH)\ + + <_profrawFiles Include="$(OutDir)instrumented\$(TargetName)_*.profraw" /> @@ -31,7 +40,7 @@ -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) -flto %(AdditionalOptions) - -fprofile-instr-generate=$(OutDir)$(TargetName)_%m.profraw %(AdditionalOptions) + -fprofile-instr-generate=$(_CLANG_PROFILE_PATH)$(TargetName)_%m.profraw %(AdditionalOptions) -fprofile-instr-use=$(OutDir)instrumented\profdata.profdata -Wno-profile-instr-unprofiled %(AdditionalOptions) diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 3fe358f7968c4f..2681e2d42e8996 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -1,7 +1,6 @@ - <__PyProject_Props_Imported>true <_ProjectFileVersion>10.0.30319.1 @@ -25,6 +24,9 @@ false + + + $(TargetName)$(TargetExt) <_TargetNameSep>$(TargetNameExt.LastIndexOf(`.`)) From 0e6f95f6cb4201482d2cbe0db1dc2c887a15cb5d Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 9 Mar 2025 08:58:37 +0100 Subject: [PATCH 15/19] add build time analysis --- PCbuild/analyse_build_times.py | 264 +++++++++++++++++++++++++++++++++ PCbuild/pyproject.props | 24 +++ 2 files changed, 288 insertions(+) create mode 100644 PCbuild/analyse_build_times.py diff --git a/PCbuild/analyse_build_times.py b/PCbuild/analyse_build_times.py new file mode 100644 index 00000000000000..27cea92ed61075 --- /dev/null +++ b/PCbuild/analyse_build_times.py @@ -0,0 +1,264 @@ +import argparse +import glob +import re + +from dataclasses import dataclass +from datetime import datetime, date, time + +# Verstrichene Zeit 00:00:00.74 +msbuild_time_str = "Verstrichene Zeit" + +# Total duration: 1 min 15 sec +pgo_time_str = "Total duration:" +pgo_regex = re.compile( + r"%s\s(\d{1,2})\smin\s(\d{1,2})\ssec" % pgo_time_str) + +begin_ts_str = "BeginTimeStamp" +end_ts_str = "EndTimeStamp" + +# ReSTTable and MarkDownTable shamelessly taken from pyperf +class ReSTTable: + def __init__(self, headers, rows): + self.headers = headers + self.rows = rows + self.widths = [len(header) for header in self.headers] + for row in self.rows: + for column, cell in enumerate(row): + self.widths[column] = max(self.widths[column], len(cell)) + + def _render_line(self, char='-'): + parts = [''] + for width in self.widths: + parts.append(char * (width + 2)) + parts.append('') + return '+'.join(parts) + + def _render_row(self, row): + parts = [''] + for width, cell in zip(self.widths, row): + parts.append(' %s ' % cell.ljust(width)) + parts.append('') + return '|'.join(parts) + + def render(self, write_line): + write_line(self._render_line('-')) + write_line(self._render_row(self.headers)) + write_line(self._render_line('=')) + for row in self.rows: + write_line(self._render_row(row)) + write_line(self._render_line('-')) + + +class MarkDownTable: + def __init__(self, headers, rows): + self.headers = headers + self.rows = rows + self.widths = [len(header) for header in self.headers] + for row in self.rows: + for column, cell in enumerate(row): + self.widths[column] = max(self.widths[column], len(cell)) + + def _render_line(self, char='-'): + parts = [''] + for idx, width in enumerate(self.widths): + if idx == 0: + parts.append(char * (width + 2)) + else: + parts.append(f':{char * width}:') + parts.append('') + return '|'.join(parts) + + def _render_row(self, row): + parts = [''] + for width, cell in zip(self.widths, row): + parts.append(" %s " % cell.ljust(width)) + parts.append('') + return '|'.join(parts) + + def render(self, write_line): + write_line(self._render_row(self.headers)) + write_line(self._render_line('-')) + for row in self.rows: + write_line(self._render_row(row)) + +@dataclass +class Build: + name: str + build_times: list + project_times: dict + + +@dataclass +class PgoBuild: + name: str + build_times: list + project_times_pginstrument: dict + project_times_pgupdate: dict + + +def get_secs_from_pgo(t): + m = pgo_regex.match(t) + return float(m[1]) * 60.0 + float(m[2]) + + +def get_secs_from_msbuild(t): + t = t[len(msbuild_time_str):].strip() + secs = (datetime.combine(date.min, time.fromisoformat(t)) + - datetime.min).total_seconds() + return secs + + +def read_build(filepath): + with open(build_name) as fh: + content = fh.read() + return pgo_time_str in content, content.splitlines() + + +def dump_summary(builds, args): + if not builds: + return + num_builds = len(builds[0].build_times) + is_pgo = num_builds > 1 + headers = [""] + rows = [] + for i in range(num_builds): + row = [i] * (len(builds) + 1) + rows.append(row) + + if is_pgo: + for i, n in enumerate(("pginstr", "pgo", "kill", "pgupd", "total time")): + rows[i][0] = n + else: + rows[0][0] = "total time" + + for x, build in enumerate(builds): + headers.append(build.name) + for y, t in enumerate(build.build_times): + rows[y][x + 1] = "%6.1f" % t + + if args.table_format == "rest": + table = ReSTTable(headers, rows) + else: + table = MarkDownTable(headers, rows) + + if is_pgo: + print("\nPGO builds:") + else: + print("\nDebug and release builds:") + table.render(print) + + +def dump_details(builds, args): + if not builds: + return + if hasattr(builds[0], "project_times"): + attrs = ["project_times"] + else: + attrs = ["project_times_pginstrument", "project_times_pgupdate"] + is_pgo = len(attrs) > 1 + for attr in attrs: + proj_first_build = getattr(builds[0], attr) + headers = [""] + rows = [] + k = list(proj_first_build.keys()) + # dict is ordered :) + # thus, we get here _freeze_module and python314 ... + skip = set(k[0:2]) + # ... plus total + skip.add(k[-1]) + cpy = {k: v for k, v in proj_first_build.items() if k not in skip} + order = k[0:2] + list([name for name, val in sorted( + cpy.items(), key=lambda item: item[1], reverse=True)]) + order.append(k[-1]) + for i in range(len(proj_first_build)): + row = [i] * (len(builds) + 1) + rows.append(row) + + for i, n in enumerate(order): + rows[i][0] = n + + for x, build in enumerate(builds): + headers.append(build.name) + for y, n in enumerate(order): + rows[y][x + 1] = "%6.1f" % getattr(build, attr)[n] + + if args.table_format == "rest": + table = ReSTTable(headers, rows) + else: + table = MarkDownTable(headers, rows) + + if is_pgo: + print("\nDetails %s:" % attr.replace("project_times_", "")) + else: + print("\nDetails:") + table.render(print) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Display build times.') + parser.add_argument( + "--table-format", type=str, default="rest", choices=["rest", "md"], + help="Format of table rendering") + parser.add_argument( + 'filenames', metavar='build_log.txt', type=str, nargs='*', + help='Build logs to process. Defaults to globbing build_*.txt') + args = parser.parse_args() + print(args) + if len(args.filenames) == 0: + filenames = sorted(glob.glob("build_*.txt")) + elif len(args.filenames) == 1: + filenames = sorted(glob.glob(args.filenames[0])) + else: + filenames = args.filenames + + builds = [] + pgo_builds = [] + proj_begin_ts = {} + for build_name in filenames: + is_pgo_build, lines = read_build(build_name) + build_name = build_name.replace("build_", "") + build_name = build_name.replace(".txt", "") + if is_pgo_build: + build = PgoBuild(build_name, [], {}, {}) + pgo_builds.append(build) + project_times = build.project_times_pginstrument + else: + build = Build(build_name, [], {}) + builds.append(build) + project_times = build.project_times + + for line in lines: + line = line.strip() + if len(build.build_times) > 1: + project_times = build.project_times_pgupdate + if line.startswith(msbuild_time_str): + secs = get_secs_from_msbuild(line) + build.build_times.append(secs) + elif line.startswith(pgo_time_str): + secs = get_secs_from_pgo(line) + build.build_times.append(secs) + elif line.startswith(begin_ts_str): + proj, begin_ts = line[len(begin_ts_str):].strip().split(":") + if proj.endswith("_d"): + proj = proj[:-2] + proj_begin_ts[proj] = begin_ts + elif line.startswith(end_ts_str): + proj, end_ts = line[len(end_ts_str):].strip().split(":") + if proj.endswith("_d"): + proj = proj[:-2] + try: + begin_ts = proj_begin_ts.pop(proj) + except KeyError: + raise Exception( + "end for %r but no begin" % proj) + project_times[proj] = float(end_ts) - float(begin_ts) + project_times["total"] = sum(project_times.values()) + if is_pgo_build: + build.project_times_pginstrument["total"] = sum( + build.project_times_pginstrument.values()) + build.build_times.append(sum(build.build_times)) + + dump_summary(builds, args) + dump_summary(pgo_builds, args) + dump_details(builds, args) + dump_details(pgo_builds, args) diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 2681e2d42e8996..19a872df6945c9 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -167,6 +167,30 @@ public override bool Execute() { + + + + + + @(_BeginTs) + + + + + + + + + + @(_EndTs) + + + + From 0bd337d7c9dfb090026eef7762a552832d823d0f Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 9 Mar 2025 09:00:30 +0100 Subject: [PATCH 16/19] Interestingly, for debug builds I now need this, too. https://github.com/python/cpython/pull/130040#issuecomment-2654335927 --- Include/internal/pycore_debug_offsets.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 44feb079571a73..a0fde82d6a6bdd 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -23,7 +23,7 @@ extern "C" { declaration \ _GENERATE_DEBUG_SECTION_LINUX(name) -#if defined(MS_WINDOWS) +#if defined(MS_WINDOWS)&& !defined(__clang__) #define _GENERATE_DEBUG_SECTION_WINDOWS(name) \ _Pragma(Py_STRINGIFY(section(Py_STRINGIFY(name), read, write))) \ __declspec(allocate(Py_STRINGIFY(name))) From 981210b33b5b2744cdcf03b1ea46bbef43f24aa1 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 9 Mar 2025 09:01:33 +0100 Subject: [PATCH 17/19] fix for older clangs --- PCbuild/pythoncore.vcxproj | 1 + 1 file changed, 1 insertion(+) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 3b3c3972987db8..6f59f8679b3307 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -420,6 +420,7 @@ HACL_CAN_COMPILE_SIMD128;%(PreprocessorDefinitions) HACL_CAN_COMPILE_SIMD256;%(PreprocessorDefinitions) + /arch:AVX From e1dfc3952b35f3d29727a610de9db438c1c57219 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 9 Mar 2025 09:02:06 +0100 Subject: [PATCH 18/19] use -flto=thin --- PCbuild/pyproject-clangcl.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/pyproject-clangcl.props b/PCbuild/pyproject-clangcl.props index 21e591b7ca403d..79dcc960229fcf 100644 --- a/PCbuild/pyproject-clangcl.props +++ b/PCbuild/pyproject-clangcl.props @@ -39,7 +39,7 @@ -Wno-deprecated-non-prototype -Wno-unused-label -Wno-pointer-sign -Wno-incompatible-pointer-types-discards-qualifiers -Wno-unused-function %(AdditionalOptions) - -flto %(AdditionalOptions) + -flto=thin %(AdditionalOptions) -fprofile-instr-generate=$(_CLANG_PROFILE_PATH)$(TargetName)_%m.profraw %(AdditionalOptions) -fprofile-instr-use=$(OutDir)instrumented\profdata.profdata -Wno-profile-instr-unprofiled %(AdditionalOptions) From 9879c49e885fa33116d0b4db05a0af19c56b51d6 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 9 Mar 2025 11:37:05 +0100 Subject: [PATCH 19/19] remove debug print --- PCbuild/analyse_build_times.py | 1 - 1 file changed, 1 deletion(-) diff --git a/PCbuild/analyse_build_times.py b/PCbuild/analyse_build_times.py index 27cea92ed61075..e715757cfb7c89 100644 --- a/PCbuild/analyse_build_times.py +++ b/PCbuild/analyse_build_times.py @@ -203,7 +203,6 @@ def dump_details(builds, args): 'filenames', metavar='build_log.txt', type=str, nargs='*', help='Build logs to process. Defaults to globbing build_*.txt') args = parser.parse_args() - print(args) if len(args.filenames) == 0: filenames = sorted(glob.glob("build_*.txt")) elif len(args.filenames) == 1: