Skip to content

Commit 7e21823

Browse files
authored
ylib2to3/pgen2/grammar: Fix Grammar.dump for parallel use with Grammar.load (#1243)
Previously, bad timing could make another process run into reading a half-written pickle cache file, and thus fail like this: > Traceback (most recent call last): > File "[..]/bin/yapf", line 5, in <module> > from yapf import run_main > File "[..]/lib/python3.11/site-packages/yapf/__init__.py", line 41, in <module> > from yapf.yapflib import yapf_api > File "[..]/lib/python3.11/site-packages/yapf/yapflib/yapf_api.py", line 38, in <module> > from yapf.pyparser import pyparser > File "[..]/lib/python3.11/site-packages/yapf/pyparser/pyparser.py", line 44, in <module> > from yapf.yapflib import format_token > File "[..]/lib/python3.11/site-packages/yapf/yapflib/format_token.py", line 23, in <module> > from yapf.pytree import pytree_utils > File "[..]/lib/python3.11/site-packages/yapf/pytree/pytree_utils.py", line 30, in <module> > from yapf_third_party._ylib2to3 import pygram > File "[..]/lib/python3.11/site-packages/yapf_third_party/_ylib2to3/pygram.py", line 29, in <module> > python_grammar = driver.load_grammar(_GRAMMAR_FILE) > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ > File "[..]/lib/python3.11/site-packages/yapf_third_party/_ylib2to3/pgen2/driver.py", line 252, in load_grammar > g.load(gp) > File "[..]/lib/python3.11/site-packages/yapf_third_party/_ylib2to3/pgen2/grammar.py", line 95, in load > d = pickle.load(f) > ^^^^^^^^^^^^^^ > EOFError: Ran out of input
1 parent ed70540 commit 7e21823

File tree

1 file changed

+35
-2
lines changed
  • third_party/yapf_third_party/_ylib2to3/pgen2

1 file changed

+35
-2
lines changed

third_party/yapf_third_party/_ylib2to3/pgen2/grammar.py

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
"""
1313

1414
# Python imports
15+
import os
1516
import pickle
17+
import tempfile
1618

1719
# Local imports
1820
from . import token
@@ -86,8 +88,39 @@ def __init__(self):
8688

8789
def dump(self, filename):
8890
"""Dump the grammar tables to a pickle file."""
89-
with open(filename, 'wb') as f:
90-
pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL)
91+
# NOTE:
92+
# - We're writing a tempfile first so that there is no chance
93+
# for someone to read a half-written file from this very spot
94+
# while we're were not done writing.
95+
# - We're using ``os.rename`` to sure not copy data around (which
96+
# would get us back to square one with a reading-half-written file
97+
# race condition).
98+
# - We're making the tempfile go to the same directory as the eventual
99+
# target ``filename`` so that there is no chance of failing from
100+
# cross-file-system renames in ``os.rename``.
101+
# - We're using the same prefix and suffix for the tempfile so if we
102+
# ever have to leave a tempfile around for failure of deletion,
103+
# it will have a reasonable filename extension and its name will help
104+
# explain is nature.
105+
tempfile_dir = os.path.dirname(filename)
106+
tempfile_prefix, tempfile_suffix = os.path.splitext(filename)
107+
with tempfile.NamedTemporaryFile(
108+
mode='wb',
109+
suffix=tempfile_suffix,
110+
prefix=tempfile_prefix,
111+
dir=tempfile_dir,
112+
delete=False) as f:
113+
pickle.dump(self.__dict__, f.file, pickle.HIGHEST_PROTOCOL)
114+
try:
115+
os.rename(f.name, filename)
116+
except OSError:
117+
# This makes sure that we do not leave the tempfile around
118+
# unless we have to...
119+
try:
120+
os.remove(f.name)
121+
except OSError:
122+
pass
123+
raise
91124

92125
def load(self, filename):
93126
"""Load the grammar tables from a pickle file."""

0 commit comments

Comments
 (0)