From d3217d12eee9eefad8444e80545b82b2a8c2be4c Mon Sep 17 00:00:00 2001 From: Ijtaba Hussain Date: Mon, 3 Apr 2023 17:28:32 +0500 Subject: [PATCH 1/6] Fetch CONFIG_ARGS from original python instance instead of fetching from intermediate instance. As "make clean" is called against the intermediate instance, the build directory is cleared and the config arguments lookup fails with a ModuleNotFoundError --- Tools/freeze/test/freeze.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index f6a5adb4519fdb..8a31085ff55efa 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -95,6 +95,8 @@ def ignore_non_src(src, names): shutil.copytree(oldroot, newroot, ignore=ignore_non_src) if os.path.exists(os.path.join(newroot, 'Makefile')): _run_quiet([MAKE, 'clean'], newroot) + else: + raise Exception("new frozen build Makefile missing!") def get_makefile_var(builddir, name): @@ -153,7 +155,7 @@ def prepare(script=None, outdir=None): print(f'configuring python in {builddir}...') cmd = [ os.path.join(srcdir, 'configure'), - *shlex.split(get_config_var(srcdir, 'CONFIG_ARGS') or ''), + *shlex.split(get_config_var(SRCDIR, 'CONFIG_ARGS') or ''), ] ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache')) prefix = os.path.join(outdir, 'python-installation') From e79d1e27d757bbf087bd1f4f6d2bd3e8c3de2a5d Mon Sep 17 00:00:00 2001 From: Ijtaba Hussain Date: Tue, 4 Apr 2023 19:45:39 +0500 Subject: [PATCH 2/6] Sometimes the Makefile is missing It depends on ./configure --- Tools/freeze/test/freeze.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 8a31085ff55efa..a3a639b5c5e5b8 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -95,9 +95,6 @@ def ignore_non_src(src, names): shutil.copytree(oldroot, newroot, ignore=ignore_non_src) if os.path.exists(os.path.join(newroot, 'Makefile')): _run_quiet([MAKE, 'clean'], newroot) - else: - raise Exception("new frozen build Makefile missing!") - def get_makefile_var(builddir, name): regex = re.compile(rf'^{name} *=\s*(.*?)\s*$') From 56345e466f3d9413a1965e68ae3f13bcf5227722 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 5 Apr 2023 07:20:19 +0400 Subject: [PATCH 3/6] Add a news entry --- .../Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst b/Misc/NEWS.d/next/Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst new file mode 100644 index 00000000000000..7e28ba6963216a --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst @@ -0,0 +1,2 @@ +``freeze`` now fetches ``CONFIG_ARGS`` from the original CPython instance +the Makefile uses to call utility scripts. Patch by Ijtaba Hussain. From 4f3060aec9a59af5246c6825644a206c9dec6959 Mon Sep 17 00:00:00 2001 From: Ijtaba Hussain Date: Wed, 10 May 2023 09:52:58 +0500 Subject: [PATCH 4/6] Improve naming conventions --- Tools/freeze/test/freeze.py | 48 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index a3a639b5c5e5b8..b336417c3ada17 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -6,9 +6,9 @@ import subprocess -TESTS_DIR = os.path.dirname(__file__) -TOOL_ROOT = os.path.dirname(TESTS_DIR) -SRCDIR = os.path.dirname(os.path.dirname(TOOL_ROOT)) +TESTS_DIR = os.path.dirname(__file__) +TOOL_ROOT = os.path.dirname(TESTS_DIR) +SOURCE_DIR = os.path.dirname(os.path.dirname(TOOL_ROOT)) MAKE = shutil.which('make') FREEZE = os.path.join(TOOL_ROOT, 'freeze.py') @@ -77,7 +77,7 @@ def ensure_opt(args, name, value): def copy_source_tree(newroot, oldroot): print(f'copying the source tree into {newroot}...') if os.path.exists(newroot): - if newroot == SRCDIR: + if newroot == SOURCE_DIR: raise Exception('this probably isn\'t what you wanted') shutil.rmtree(newroot) @@ -96,9 +96,9 @@ def ignore_non_src(src, names): if os.path.exists(os.path.join(newroot, 'Makefile')): _run_quiet([MAKE, 'clean'], newroot) -def get_makefile_var(builddir, name): +def get_makefile_var(final_dir, name): regex = re.compile(rf'^{name} *=\s*(.*?)\s*$') - filename = os.path.join(builddir, 'Makefile') + filename = os.path.join(final_dir, 'Makefile') try: infile = open(filename, encoding='utf-8') except FileNotFoundError: @@ -112,8 +112,8 @@ def get_makefile_var(builddir, name): return None -def get_config_var(builddir, name): - python = os.path.join(builddir, 'python') +def get_config_var(final_dir, name): + python = os.path.join(final_dir, 'python') if os.path.isfile(python): cmd = [python, '-c', f'import sysconfig; print(sysconfig.get_config_var("{name}"))'] @@ -121,7 +121,7 @@ def get_config_var(builddir, name): return _run_stdout(cmd) except subprocess.CalledProcessError: pass - return get_makefile_var(builddir, name) + return get_makefile_var(final_dir, name) ################################## @@ -141,23 +141,23 @@ def prepare(script=None, outdir=None): # Make a copy of the repo to avoid affecting the current build # (e.g. changing PREFIX). - srcdir = os.path.join(outdir, 'cpython') - copy_source_tree(srcdir, SRCDIR) + interim_dir = os.path.join(outdir, 'cpython') + copy_source_tree(interim_dir, SOURCE_DIR) - # We use an out-of-tree build (instead of srcdir). - builddir = os.path.join(outdir, 'python-build') - os.makedirs(builddir, exist_ok=True) + # We use an out-of-tree build (instead of interim_dir). + final_dir = os.path.join(outdir, 'python-build') + os.makedirs(final_dir, exist_ok=True) # Run configure. - print(f'configuring python in {builddir}...') + print(f'configuring python in {final_dir}...') cmd = [ - os.path.join(srcdir, 'configure'), - *shlex.split(get_config_var(SRCDIR, 'CONFIG_ARGS') or ''), + os.path.join(interim_dir, 'configure'), + *shlex.split(get_config_var(SOURCE_DIR, 'CONFIG_ARGS') or ''), ] ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache')) prefix = os.path.join(outdir, 'python-installation') ensure_opt(cmd, 'prefix', prefix) - _run_quiet(cmd, builddir) + _run_quiet(cmd, final_dir) if not MAKE: raise UnsupportedError('make') @@ -172,15 +172,15 @@ def prepare(script=None, outdir=None): parallel = '-j2' # Build python. - print(f'building python {parallel=} in {builddir}...') - if os.path.exists(os.path.join(srcdir, 'Makefile')): - # Out-of-tree builds require a clean srcdir. - _run_quiet([MAKE, '-C', srcdir, 'clean']) - _run_quiet([MAKE, '-C', builddir, parallel]) + print(f'building python {parallel=} in {final_dir}...') + if os.path.exists(os.path.join(interim_dir, 'Makefile')): + # Out-of-tree builds require a clean interim_dir. + _run_quiet([MAKE, '-C', interim_dir, 'clean']) + _run_quiet([MAKE, '-C', final_dir, parallel]) # Install the build. print(f'installing python into {prefix}...') - _run_quiet([MAKE, '-C', builddir, 'install']) + _run_quiet([MAKE, '-C', final_dir, 'install']) python = os.path.join(prefix, 'bin', 'python3') return outdir, scriptfile, python From 382947beb6c07ef4a6044a5514e1b6f4c302caee Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 11 Jul 2023 23:43:29 +0200 Subject: [PATCH 5/6] Revert "Improve naming conventions" This reverts commit 4f3060aec9a59af5246c6825644a206c9dec6959. --- Tools/freeze/test/freeze.py | 48 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index b336417c3ada17..a3a639b5c5e5b8 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -6,9 +6,9 @@ import subprocess -TESTS_DIR = os.path.dirname(__file__) -TOOL_ROOT = os.path.dirname(TESTS_DIR) -SOURCE_DIR = os.path.dirname(os.path.dirname(TOOL_ROOT)) +TESTS_DIR = os.path.dirname(__file__) +TOOL_ROOT = os.path.dirname(TESTS_DIR) +SRCDIR = os.path.dirname(os.path.dirname(TOOL_ROOT)) MAKE = shutil.which('make') FREEZE = os.path.join(TOOL_ROOT, 'freeze.py') @@ -77,7 +77,7 @@ def ensure_opt(args, name, value): def copy_source_tree(newroot, oldroot): print(f'copying the source tree into {newroot}...') if os.path.exists(newroot): - if newroot == SOURCE_DIR: + if newroot == SRCDIR: raise Exception('this probably isn\'t what you wanted') shutil.rmtree(newroot) @@ -96,9 +96,9 @@ def ignore_non_src(src, names): if os.path.exists(os.path.join(newroot, 'Makefile')): _run_quiet([MAKE, 'clean'], newroot) -def get_makefile_var(final_dir, name): +def get_makefile_var(builddir, name): regex = re.compile(rf'^{name} *=\s*(.*?)\s*$') - filename = os.path.join(final_dir, 'Makefile') + filename = os.path.join(builddir, 'Makefile') try: infile = open(filename, encoding='utf-8') except FileNotFoundError: @@ -112,8 +112,8 @@ def get_makefile_var(final_dir, name): return None -def get_config_var(final_dir, name): - python = os.path.join(final_dir, 'python') +def get_config_var(builddir, name): + python = os.path.join(builddir, 'python') if os.path.isfile(python): cmd = [python, '-c', f'import sysconfig; print(sysconfig.get_config_var("{name}"))'] @@ -121,7 +121,7 @@ def get_config_var(final_dir, name): return _run_stdout(cmd) except subprocess.CalledProcessError: pass - return get_makefile_var(final_dir, name) + return get_makefile_var(builddir, name) ################################## @@ -141,23 +141,23 @@ def prepare(script=None, outdir=None): # Make a copy of the repo to avoid affecting the current build # (e.g. changing PREFIX). - interim_dir = os.path.join(outdir, 'cpython') - copy_source_tree(interim_dir, SOURCE_DIR) + srcdir = os.path.join(outdir, 'cpython') + copy_source_tree(srcdir, SRCDIR) - # We use an out-of-tree build (instead of interim_dir). - final_dir = os.path.join(outdir, 'python-build') - os.makedirs(final_dir, exist_ok=True) + # We use an out-of-tree build (instead of srcdir). + builddir = os.path.join(outdir, 'python-build') + os.makedirs(builddir, exist_ok=True) # Run configure. - print(f'configuring python in {final_dir}...') + print(f'configuring python in {builddir}...') cmd = [ - os.path.join(interim_dir, 'configure'), - *shlex.split(get_config_var(SOURCE_DIR, 'CONFIG_ARGS') or ''), + os.path.join(srcdir, 'configure'), + *shlex.split(get_config_var(SRCDIR, 'CONFIG_ARGS') or ''), ] ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache')) prefix = os.path.join(outdir, 'python-installation') ensure_opt(cmd, 'prefix', prefix) - _run_quiet(cmd, final_dir) + _run_quiet(cmd, builddir) if not MAKE: raise UnsupportedError('make') @@ -172,15 +172,15 @@ def prepare(script=None, outdir=None): parallel = '-j2' # Build python. - print(f'building python {parallel=} in {final_dir}...') - if os.path.exists(os.path.join(interim_dir, 'Makefile')): - # Out-of-tree builds require a clean interim_dir. - _run_quiet([MAKE, '-C', interim_dir, 'clean']) - _run_quiet([MAKE, '-C', final_dir, parallel]) + print(f'building python {parallel=} in {builddir}...') + if os.path.exists(os.path.join(srcdir, 'Makefile')): + # Out-of-tree builds require a clean srcdir. + _run_quiet([MAKE, '-C', srcdir, 'clean']) + _run_quiet([MAKE, '-C', builddir, parallel]) # Install the build. print(f'installing python into {prefix}...') - _run_quiet([MAKE, '-C', final_dir, 'install']) + _run_quiet([MAKE, '-C', builddir, 'install']) python = os.path.join(prefix, 'bin', 'python3') return outdir, scriptfile, python From 4b72a71bd6b562c3441bef157529b6cd23a44588 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 11 Jul 2023 23:44:28 +0200 Subject: [PATCH 6/6] Revert whitespace change --- Tools/freeze/test/freeze.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index a3a639b5c5e5b8..92e97cb261719c 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -96,6 +96,7 @@ def ignore_non_src(src, names): if os.path.exists(os.path.join(newroot, 'Makefile')): _run_quiet([MAKE, 'clean'], newroot) + def get_makefile_var(builddir, name): regex = re.compile(rf'^{name} *=\s*(.*?)\s*$') filename = os.path.join(builddir, 'Makefile')