From 28364e04ffc475175d4e7eb881a44d0fc0e99b7b Mon Sep 17 00:00:00 2001 From: "Bernhard M. Wiedemann" Date: Sat, 25 Feb 2017 06:42:28 +0100 Subject: [PATCH 1/2] bpo-29708: support SOURCE_DATE_EPOCH env var in py_compile to allow for reproducible builds of python packages See https://reproducible-builds.org/ for why this is good and https://reproducible-builds.org/specs/source-date-epoch/ for the definition of this variable. Background: In some distributions like openSUSE, binary rpms contain precompiled .pyc files. And packages like amqp or twisted dynamically generate .py files at build time so those have the current time and that timestamp gets embedded into the .pyc file header. When we then adapt file timestamps in rpms to be constant, the timestamp in the .pyc header will no more match the .py timestamp in the filesystem. The software will still work, but it will not use the .pyc file as it should. --- Doc/library/py_compile.rst | 4 ++++ Lib/py_compile.py | 8 ++++++++ Lib/test/test_py_compile.py | 12 ++++++++++++ .../Build/2017-09-07-04-20-49.bpo-29708.YCaHEx.rst | 2 ++ 4 files changed, 26 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2017-09-07-04-20-49.bpo-29708.YCaHEx.rst diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index a4f06de597b14e..11c004c6177123 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -57,6 +57,10 @@ byte-code cache files in the directory containing the source code. enum and controls how the generated ``.pyc`` files are invalidated at runtime. + If the SOURCE_DATE_EPOCH environment variable is set, the timestamp entry in + the .pyc file header, will be limited to this value. + See https://reproducible-builds.org/specs/source-date-epoch/ for more info. + .. versionchanged:: 3.2 Changed default value of *cfile* to be :PEP:`3147`-compliant. Previous default was *file* + ``'c'`` (``'o'`` if optimization was enabled). diff --git a/Lib/py_compile.py b/Lib/py_compile.py index a0f4defdce68c9..385f0591fd085e 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -147,6 +147,14 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1, pass if invalidation_mode == PycInvalidationMode.TIMESTAMP: source_stats = loader.path_stats(file) + source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') + if source_date_epoch: + try: + source_date_epoch = int(source_date_epoch) + except ValueError: + raise ValueError("SOURCE_DATE_EPOCH is not a valid integer") + if source_stats['mtime'] > source_date_epoch: + source_stats['mtime'] = source_date_epoch bytecode = importlib._bootstrap_external._code_to_timestamp_pyc( code, source_stats['mtime'], source_stats['size']) else: diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index bcb686c86b5229..5e59187a83f4ab 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -98,6 +98,18 @@ def test_bad_coding(self): self.assertFalse(os.path.exists( importlib.util.cache_from_source(bad_coding))) + def test_source_date_epoch(self): + testtime = 123456789 + with support.EnvironmentVarGuard() as env: + env["SOURCE_DATE_EPOCH"] = str(testtime) + py_compile.compile(self.source_path, self.pyc_path) + self.assertTrue(os.path.exists(self.pyc_path)) + self.assertFalse(os.path.exists(self.cache_path)) + with open(self.pyc_path, "rb") as f: + f.read(4) # Skip the magic number. + timebytes = f.read(4) # Read timestamp. + self.assertEqual(testtime, int.from_bytes(timebytes, 'little')) + @unittest.skipIf(sys.flags.optimize > 0, 'test does not work with -O') def test_double_dot_no_clobber(self): # http://bugs.python.org/issue22966 diff --git a/Misc/NEWS.d/next/Build/2017-09-07-04-20-49.bpo-29708.YCaHEx.rst b/Misc/NEWS.d/next/Build/2017-09-07-04-20-49.bpo-29708.YCaHEx.rst new file mode 100644 index 00000000000000..4166d5d638cf27 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-09-07-04-20-49.bpo-29708.YCaHEx.rst @@ -0,0 +1,2 @@ +If the SOURCE_DATE_EPOCH environment variable is set, +py_compile will use it to override the timestamps it puts into .pyc files. From 3182f490e28224aa8f949c002458e49649706827 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 11 Jan 2018 10:35:00 -0800 Subject: [PATCH 2/2] Add markup and change notice to the docs --- Doc/library/py_compile.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 11c004c6177123..3e1a01b761de3c 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -57,8 +57,8 @@ byte-code cache files in the directory containing the source code. enum and controls how the generated ``.pyc`` files are invalidated at runtime. - If the SOURCE_DATE_EPOCH environment variable is set, the timestamp entry in - the .pyc file header, will be limited to this value. + If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, the timestamp + entry in the ``.pyc`` file header will be set to this value. See https://reproducible-builds.org/specs/source-date-epoch/ for more info. .. versionchanged:: 3.2 @@ -75,6 +75,7 @@ byte-code cache files in the directory containing the source code. .. versionchanged:: 3.7 The *invalidation_mode* parameter was added as specified in :pep:`552`. + Support for :envvar:`SOURCE_DATE_EPOCH` was also added. .. class:: PycInvalidationMode