diff --git a/news/4501.feature b/news/4501.feature new file mode 100644 index 00000000000..475a1317224 --- /dev/null +++ b/news/4501.feature @@ -0,0 +1,3 @@ +Installing from a local directory or a VCS URL now builds a wheel to install, +rather than running ``setup.py install``. Wheels from these sources are not +cached. diff --git a/pip/cache.py b/pip/cache.py index 87fd0e1b595..b3206000aae 100644 --- a/pip/cache.py +++ b/pip/cache.py @@ -10,6 +10,7 @@ import pip.index from pip.compat import expanduser from pip.download import path_to_url +from pip.utils import temp_dir from pip.utils.cache import get_cache_path_for_link from pip.wheel import InvalidWheelFilename, Wheel @@ -27,45 +28,62 @@ def __init__(self, cache_dir, format_control): binaries being read from the cache. """ self._cache_dir = expanduser(cache_dir) if cache_dir else None + # Ephemeral cache: store wheels just for this run + self._ephem_cache_dir = temp_dir.TempDirectory(kind="ephem-cache") + self._ephem_cache_dir.create() self._format_control = format_control def cached_wheel(self, link, package_name): - not_cached = ( - not self._cache_dir or - not link or - link.is_wheel or - not link.is_artifact or - not package_name - ) + orig_link = link + link = cached_wheel( + self._cache_dir, link, self._format_control, package_name) + if link is orig_link: + link = cached_wheel( + self._ephem_cache_dir.path, link, self._format_control, + package_name) + return link - if not_cached: - return link + def cleanup(self): + self._ephem_cache_dir.cleanup() + + +def cached_wheel(cache_dir, link, format_control, package_name): + not_cached = ( + not cache_dir or + not link or + link.is_wheel or + not link.is_artifact or + not package_name + ) - canonical_name = canonicalize_name(package_name) - formats = pip.index.fmt_ctl_formats( - self._format_control, canonical_name - ) - if "binary" not in formats: + if not_cached: + return link + + canonical_name = canonicalize_name(package_name) + formats = pip.index.fmt_ctl_formats( + format_control, canonical_name + ) + if "binary" not in formats: + return link + root = get_cache_path_for_link(cache_dir, link) + try: + wheel_names = os.listdir(root) + except OSError as err: + if err.errno in {errno.ENOENT, errno.ENOTDIR}: return link - root = get_cache_path_for_link(self._cache_dir, link) + raise + candidates = [] + for wheel_name in wheel_names: try: - wheel_names = os.listdir(root) - except OSError as err: - if err.errno in {errno.ENOENT, errno.ENOTDIR}: - return link - raise - candidates = [] - for wheel_name in wheel_names: - try: - wheel = Wheel(wheel_name) - except InvalidWheelFilename: - continue - if not wheel.supported(): - # Built for a different python/arch/etc - continue - candidates.append((wheel.support_index_min(), wheel_name)) - if not candidates: - return link - candidates.sort() - path = os.path.join(root, candidates[0][1]) - return pip.index.Link(path_to_url(path)) + wheel = Wheel(wheel_name) + except InvalidWheelFilename: + continue + if not wheel.supported(): + # Built for a different python/arch/etc + continue + candidates.append((wheel.support_index_min(), wheel_name)) + if not candidates: + return link + candidates.sort() + path = os.path.join(root, candidates[0][1]) + return pip.index.Link(path_to_url(path)) diff --git a/pip/commands/freeze.py b/pip/commands/freeze.py index e02fdcf8799..94500fd4439 100644 --- a/pip/commands/freeze.py +++ b/pip/commands/freeze.py @@ -88,5 +88,8 @@ def run(self, options, args): skip=skip, exclude_editable=options.exclude_editable) - for line in freeze(**freeze_kwargs): - sys.stdout.write(line + '\n') + try: + for line in freeze(**freeze_kwargs): + sys.stdout.write(line + '\n') + finally: + wheel_cache.cleanup() diff --git a/pip/commands/install.py b/pip/commands/install.py index 6c7dab73e5d..b6713b2cb8a 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -247,18 +247,19 @@ def run(self, options, args): use_user_site=options.use_user_site, ) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=None, - progress_bar=options.progress_bar, - ) try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + self.name, wheel_cache + ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=None, + progress_bar=options.progress_bar, + ) + resolver = Resolver( preparer=preparer, finder=finder, @@ -350,6 +351,7 @@ def run(self, options, args): # Clean up if not options.no_clean: requirement_set.cleanup_files() + wheel_cache.cleanup() if options.target_dir: self._handle_target_dir( diff --git a/pip/commands/wheel.py b/pip/commands/wheel.py index cc6476c27a5..644451bf653 100644 --- a/pip/commands/wheel.py +++ b/pip/commands/wheel.py @@ -146,35 +146,35 @@ def run(self, options, args): require_hashes=options.require_hashes, ) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) + try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + self.name, wheel_cache + ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=options.wheel_dir, - progress_bar=options.progress_bar, - ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=options.wheel_dir, + progress_bar=options.progress_bar, + ) - resolver = Resolver( - preparer=preparer, - finder=finder, - session=session, - wheel_cache=wheel_cache, - use_user_site=False, - upgrade_strategy="to-satisfy-only", - force_reinstall=False, - ignore_dependencies=options.ignore_dependencies, - ignore_requires_python=options.ignore_requires_python, - ignore_installed=True, - isolated=options.isolated_mode, - ) - resolver.resolve(requirement_set) + resolver = Resolver( + preparer=preparer, + finder=finder, + session=session, + wheel_cache=wheel_cache, + use_user_site=False, + upgrade_strategy="to-satisfy-only", + force_reinstall=False, + ignore_dependencies=options.ignore_dependencies, + ignore_requires_python=options.ignore_requires_python, + ignore_installed=True, + isolated=options.isolated_mode, + ) + resolver.resolve(requirement_set) - try: # build wheels wb = WheelBuilder( requirement_set, @@ -196,3 +196,4 @@ def run(self, options, args): finally: if not options.no_clean: requirement_set.cleanup_files() + wheel_cache.cleanup() diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 1c8a35e29a5..403bb0f2301 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -44,9 +44,6 @@ def __init__(self, require_hashes=False, target_dir=None, use_user_site=False, pycompile=True): """Create a RequirementSet. - - :param wheel_cache: The pip wheel cache, for passing to - InstallRequirement. """ self.requirements = Requirements() diff --git a/pip/wheel.py b/pip/wheel.py index ca080e24381..499ff6c33c6 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -736,6 +736,7 @@ def build(self, session, autobuilding=False): buildset = [] for req in reqset: + ephem_cache = False if req.constraint: continue if req.is_wheel: @@ -745,7 +746,8 @@ def build(self, session, autobuilding=False): elif autobuilding and req.editable: pass elif autobuilding and req.link and not req.link.is_artifact: - pass + # VCS checkout. Build wheel just for this run. + ephem_cache = True elif autobuilding and not req.source_dir: pass else: @@ -753,17 +755,16 @@ def build(self, session, autobuilding=False): link = req.link base, ext = link.splitext() if pip.index.egg_info_matches(base, None, link) is None: - # Doesn't look like a package - don't autobuild a wheel - # because we'll have no way to lookup the result sanely - continue - if "binary" not in pip.index.fmt_ctl_formats( + # E.g. local directory. Build wheel just for this run. + ephem_cache = True + elif "binary" not in pip.index.fmt_ctl_formats( self.finder.format_control, canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name) continue - buildset.append(req) + buildset.append((req, ephem_cache)) if not buildset: return True @@ -771,18 +772,20 @@ def build(self, session, autobuilding=False): # Build the wheels. logger.info( 'Building wheels for collected packages: %s', - ', '.join([req.name for req in buildset]), + ', '.join([req.name for (req, _) in buildset]), ) with indent_log(): build_success, build_failure = [], [] - for req in buildset: + for req, ephem in buildset: python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag # NOTE: Should move out a method on the cache directly. - output_dir = get_cache_path_for_link( - self.wheel_cache._cache_dir, req.link + cache_root = ( + self.wheel_cache._ephem_cache_dir.path if ephem + else self.wheel_cache._cache_dir ) + output_dir = get_cache_path_for_link(cache_root, req.link) try: ensure_dir(output_dir) except OSError as e: diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 033ad3dc81a..9805f00e8d8 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -983,16 +983,15 @@ def test_install_builds_wheels(script, data): # and built wheels for upper and wheelbroken assert "Running setup.py bdist_wheel for upper" in str(res), str(res) assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res) - # But not requires_wheel... which is a local dir and thus uncachable. - assert "Running setup.py bdist_wheel for requir" not in str(res), str(res) + # Wheels are built for local directories, but not cached. + assert "Running setup.py bdist_wheel for requir" in str(res), str(res) # wheelbroken has to run install # into the cache assert wheels != [], str(res) # and installed from the wheel assert "Running setup.py install for upper" not in str(res), str(res) - # the local tree can't build a wheel (because we can't assume that every - # build will have a suitable unique key to cache on). - assert "Running setup.py install for requires-wheel" in str(res), str(res) + # Wheels are built for local directories, but not cached. + assert "Running setup.py install for requir" not in str(res), str(res) # wheelbroken has to run install assert "Running setup.py install for wheelb" in str(res), str(res) # We want to make sure we used the correct implementation tag @@ -1019,16 +1018,15 @@ def test_install_no_binary_disables_building_wheels(script, data): wheels.extend(files) # and built wheels for wheelbroken only assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res) - # But not requires_wheel... which is a local dir and thus uncachable. - assert "Running setup.py bdist_wheel for requir" not in str(res), str(res) + # Wheels are built for local directories, but not cached + assert "Running setup.py bdist_wheel for requir" in str(res), str(res) # Nor upper, which was blacklisted assert "Running setup.py bdist_wheel for upper" not in str(res), str(res) # wheelbroken has to run install # into the cache assert wheels != [], str(res) - # the local tree can't build a wheel (because we can't assume that every - # build will have a suitable unique key to cache on). - assert "Running setup.py install for requires-wheel" in str(res), str(res) + # Wheels are built for local directories, but not cached + assert "Running setup.py install for requir" not in str(res), str(res) # And these two fell back to sdist based installed. assert "Running setup.py install for wheelb" in str(res), str(res) assert "Running setup.py install for upper" in str(res), str(res) diff --git a/tests/functional/test_install_cleanup.py b/tests/functional/test_install_cleanup.py index 79fff57e293..c1ba9fc3ba0 100644 --- a/tests/functional/test_install_cleanup.py +++ b/tests/functional/test_install_cleanup.py @@ -30,7 +30,7 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data): build = script.base_path / 'pip-build' script.pip( 'install', '--no-clean', '--no-index', '--build', build, - '--find-links=%s' % data.find_links, 'simple', + '--find-links=%s' % data.find_links, 'simple', expect_temp=True, ) assert exists(build) @@ -132,7 +132,7 @@ def test_cleanup_prevented_upon_build_dir_exception(script, data): result = script.pip( 'install', '-f', data.find_links, '--no-index', 'simple', '--build', build, - expect_error=True, + expect_error=True, expect_temp=True, ) assert result.returncode == PREVIOUS_BUILD_DIR_ERROR diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index 6faea6149cb..0c8edfd7241 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -68,11 +68,13 @@ def test_install_subversion_usersite_editable_with_distribute( ) result.assert_installed('INITools', use_user_site=True) + @pytest.mark.network def test_install_curdir_usersite(self, script, virtualenv, data): """ Test installing current directory ('.') into usersite """ virtualenv.system_site_packages = True + script.pip("install", "wheel") run_from = data.packages.join("FSPkg") result = script.pip( 'install', '-vvv', '--user', curdir, @@ -80,12 +82,12 @@ def test_install_curdir_usersite(self, script, virtualenv, data): expect_error=False, ) fspkg_folder = script.user_site / 'fspkg' - egg_info_folder = ( - script.user_site / 'FSPkg-0.1.dev0-py%s.egg-info' % pyversion + dist_info_folder = ( + script.user_site / 'FSPkg-0.1.dev0.dist-info' ) assert fspkg_folder in result.files_created, result.stdout - assert egg_info_folder in result.files_created + assert dist_info_folder in result.files_created def test_install_user_venv_nositepkgs_fails(self, script, data): """ diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index d5e05187adf..b310cf2d069 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -190,7 +190,7 @@ def test_pip_wheel_fail_cause_of_previous_build_dir(script, data): result = script.pip( 'wheel', '--no-index', '--find-links=%s' % data.find_links, '--build', script.venv_path / 'build', - 'simple==3.0', expect_error=True, + 'simple==3.0', expect_error=True, expect_temp=True, ) # Then I see that the error code is the right one