diff --git a/news/4501.feature b/news/4501.feature new file mode 100644 index 00000000000..475a1317224 --- /dev/null +++ b/news/4501.feature @@ -0,0 +1,3 @@ +Installing from a local directory or a VCS URL now builds a wheel to install, +rather than running ``setup.py install``. Wheels from these sources are not +cached. diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index bd4324b0211..1aa17aa96c2 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -11,6 +11,7 @@ from pip._internal import index from pip._internal.compat import expanduser from pip._internal.download import path_to_url +from pip._internal.utils.temp_dir import TempDirectory from pip._internal.wheel import InvalidWheelFilename, Wheel logger = logging.getLogger(__name__) @@ -102,13 +103,18 @@ def _link_for_candidate(self, link, candidate): return index.Link(path_to_url(path)) + def cleanup(self): + pass -class WheelCache(Cache): + +class SimpleWheelCache(Cache): """A cache of wheels for future installs. """ def __init__(self, cache_dir, format_control): - super(WheelCache, self).__init__(cache_dir, format_control, {"binary"}) + super(SimpleWheelCache, self).__init__( + cache_dir, format_control, {"binary"} + ) def get_path_for_link(self, link): """Return a directory to store cached wheels for link @@ -127,8 +133,7 @@ def get_path_for_link(self, link): """ parts = self._get_cache_path_parts(link) - # Inside of the base location for cached wheels, expand our parts and - # join them all together. + # Store wheels within the root cache_dir return os.path.join(self.cache_dir, "wheels", *parts) def get(self, link, package_name): @@ -148,3 +153,50 @@ def get(self, link, package_name): return link return self._link_for_candidate(link, min(candidates)[1]) + + +class EphemWheelCache(SimpleWheelCache): + """A SimpleWheelCache that creates it's own temporary cache directory + """ + + def __init__(self, format_control): + self._temp_dir = TempDirectory(kind="ephem-wheel-cache") + self._temp_dir.create() + + super(EphemWheelCache, self).__init__( + self._temp_dir.path, format_control + ) + + def cleanup(self): + self._temp_dir.cleanup() + + +class WheelCache(Cache): + """Wraps EphemWheelCache and SimpleWheelCache into a single Cache + + This Cache allows for gracefully degradation, using the ephem wheel cache + when a certain link is not found in the simple wheel cache first. + """ + + def __init__(self, cache_dir, format_control): + super(WheelCache, self).__init__( + cache_dir, format_control, {'binary'} + ) + self._wheel_cache = SimpleWheelCache(cache_dir, format_control) + self._ephem_cache = EphemWheelCache(format_control) + + def get_path_for_link(self, link): + return self._wheel_cache.get_path_for_link(link) + + def get_ephem_path_for_link(self, link): + return self._ephem_cache.get_path_for_link(link) + + def get(self, link, package_name): + retval = self._wheel_cache.get(link, package_name) + if retval is link: + retval = self._ephem_cache.get(link, package_name) + return retval + + def cleanup(self): + self._wheel_cache.cleanup() + self._ephem_cache.cleanup() diff --git a/src/pip/_internal/commands/freeze.py b/src/pip/_internal/commands/freeze.py index 5c0e339705d..0d3d4ae2404 100644 --- a/src/pip/_internal/commands/freeze.py +++ b/src/pip/_internal/commands/freeze.py @@ -89,5 +89,8 @@ def run(self, options, args): exclude_editable=options.exclude_editable, ) - for line in freeze(**freeze_kwargs): - sys.stdout.write(line + '\n') + try: + for line in freeze(**freeze_kwargs): + sys.stdout.write(line + '\n') + finally: + wheel_cache.cleanup() diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index cac15aac0e4..2d2656be4eb 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -224,10 +224,10 @@ def run(self, options, args): global_options = options.global_options or [] with self._build_session(options) as session: - finder = self._build_package_finder(options, session) build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) + if options.cache_dir and not check_path_owner(options.cache_dir): logger.warning( "The directory '%s' or its parent directory is not owned " @@ -249,18 +249,19 @@ def run(self, options, args): use_user_site=options.use_user_site, ) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=None, - progress_bar=options.progress_bar, - ) try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + self.name, wheel_cache + ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=None, + progress_bar=options.progress_bar, + ) + resolver = Resolver( preparer=preparer, finder=finder, @@ -349,6 +350,7 @@ def run(self, options, args): # Clean up if not options.no_clean: requirement_set.cleanup_files() + wheel_cache.cleanup() if options.target_dir: self._handle_target_dir( diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 51985b84dab..17c18796f8e 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -146,35 +146,35 @@ def run(self, options, args): require_hashes=options.require_hashes, ) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) + try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + self.name, wheel_cache + ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=options.wheel_dir, - progress_bar=options.progress_bar, - ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=options.wheel_dir, + progress_bar=options.progress_bar, + ) - resolver = Resolver( - preparer=preparer, - finder=finder, - session=session, - wheel_cache=wheel_cache, - use_user_site=False, - upgrade_strategy="to-satisfy-only", - force_reinstall=False, - ignore_dependencies=options.ignore_dependencies, - ignore_requires_python=options.ignore_requires_python, - ignore_installed=True, - isolated=options.isolated_mode, - ) - resolver.resolve(requirement_set) + resolver = Resolver( + preparer=preparer, + finder=finder, + session=session, + wheel_cache=wheel_cache, + use_user_site=False, + upgrade_strategy="to-satisfy-only", + force_reinstall=False, + ignore_dependencies=options.ignore_dependencies, + ignore_requires_python=options.ignore_requires_python, + ignore_installed=True, + isolated=options.isolated_mode, + ) + resolver.resolve(requirement_set) - try: # build wheels wb = WheelBuilder( finder, preparer, wheel_cache, @@ -195,3 +195,4 @@ def run(self, options, args): finally: if not options.no_clean: requirement_set.cleanup_files() + wheel_cache.cleanup() diff --git a/src/pip/_internal/resolve.py b/src/pip/_internal/resolve.py index e84b6dc8511..247c561c549 100644 --- a/src/pip/_internal/resolve.py +++ b/src/pip/_internal/resolve.py @@ -220,7 +220,9 @@ def _get_abstract_dist_for(self, req): if req.satisfied_by: should_modify = ( self.upgrade_strategy != "to-satisfy-only" or - self.force_reinstall or self.ignore_installed + self.force_reinstall or + self.ignore_installed or + req.link.scheme == 'file' ) if should_modify: self._set_req_to_reinstall(req) diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index c8fc21518fd..92a38d72e1e 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -806,6 +806,7 @@ def build(self, requirements, session, autobuilding=False): buildset = [] for req in requirements: + ephem_cache = False if req.constraint: continue if req.is_wheel: @@ -816,7 +817,8 @@ def build(self, requirements, session, autobuilding=False): elif autobuilding and req.editable: pass elif autobuilding and req.link and not req.link.is_artifact: - pass + # VCS checkout. Build wheel just for this run. + ephem_cache = True elif autobuilding and not req.source_dir: pass else: @@ -824,9 +826,8 @@ def build(self, requirements, session, autobuilding=False): link = req.link base, ext = link.splitext() if index.egg_info_matches(base, None, link) is None: - # Doesn't look like a package - don't autobuild a wheel - # because we'll have no way to lookup the result sanely - continue + # E.g. local directory. Build wheel just for this run. + ephem_cache = True if "binary" not in index.fmt_ctl_formats( self.finder.format_control, canonicalize_name(req.name)): @@ -835,7 +836,7 @@ def build(self, requirements, session, autobuilding=False): "being disabled for it.", req.name, ) continue - buildset.append(req) + buildset.append((req, ephem_cache)) if not buildset: return True @@ -843,15 +844,19 @@ def build(self, requirements, session, autobuilding=False): # Build the wheels. logger.info( 'Building wheels for collected packages: %s', - ', '.join([req.name for req in buildset]), + ', '.join([req.name for (req, _) in buildset]), ) + _cache = self.wheel_cache # shorter name with indent_log(): build_success, build_failure = [], [] - for req in buildset: + for req, ephem in buildset: python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag - output_dir = self.wheel_cache.get_path_for_link(req.link) + if ephem: + output_dir = _cache.get_ephem_path_for_link(req.link) + else: + output_dir = _cache.get_path_for_link(req.link) try: ensure_dir(output_dir) except OSError as e: diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 1c7ba487c98..7777cde892f 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -994,16 +994,15 @@ def test_install_builds_wheels(script, data, common_wheels): # and built wheels for upper and wheelbroken assert "Running setup.py bdist_wheel for upper" in str(res), str(res) assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res) - # But not requires_wheel... which is a local dir and thus uncachable. - assert "Running setup.py bdist_wheel for requir" not in str(res), str(res) + # Wheels are built for local directories, but not cached. + assert "Running setup.py bdist_wheel for requir" in str(res), str(res) # wheelbroken has to run install # into the cache assert wheels != [], str(res) # and installed from the wheel assert "Running setup.py install for upper" not in str(res), str(res) - # the local tree can't build a wheel (because we can't assume that every - # build will have a suitable unique key to cache on). - assert "Running setup.py install for requires-wheel" in str(res), str(res) + # Wheels are built for local directories, but not cached. + assert "Running setup.py install for requir" not in str(res), str(res) # wheelbroken has to run install assert "Running setup.py install for wheelb" in str(res), str(res) # We want to make sure we used the correct implementation tag @@ -1027,13 +1026,12 @@ def test_install_no_binary_disables_building_wheels( assert expected in str(res), str(res) # and built wheels for wheelbroken only assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res) - # But not requires_wheel... which is a local dir and thus uncachable. - assert "Running setup.py bdist_wheel for requir" not in str(res), str(res) - # Nor upper, which was blacklisted + # Wheels are built for local directories, but not cached across runs + assert "Running setup.py bdist_wheel for requir" in str(res), str(res) + # Don't build wheel for upper which was blacklisted assert "Running setup.py bdist_wheel for upper" not in str(res), str(res) - # the local tree can't build a wheel (because we can't assume that every - # build will have a suitable unique key to cache on). - assert "Running setup.py install for requires-wheel" in str(res), str(res) + # Wheels are built for local directories, but not cached across runs + assert "Running setup.py install for requir" not in str(res), str(res) # And these two fell back to sdist based installed. assert "Running setup.py install for wheelb" in str(res), str(res) assert "Running setup.py install for upper" in str(res), str(res) diff --git a/tests/functional/test_install_cleanup.py b/tests/functional/test_install_cleanup.py index 7424cb41c68..c7ff18c70ab 100644 --- a/tests/functional/test_install_cleanup.py +++ b/tests/functional/test_install_cleanup.py @@ -31,7 +31,7 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data): build = script.base_path / 'pip-build' script.pip( 'install', '--no-clean', '--no-index', '--build', build, - '--find-links=%s' % data.find_links, 'simple', + '--find-links=%s' % data.find_links, 'simple', expect_temp=True, ) assert exists(build) @@ -134,7 +134,7 @@ def test_cleanup_prevented_upon_build_dir_exception(script, data): result = script.pip( 'install', '-f', data.find_links, '--no-index', 'simple', '--build', build, - expect_error=True, + expect_error=True, expect_temp=True, ) assert result.returncode == PREVIOUS_BUILD_DIR_ERROR diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index cf0cb513773..35bfedafa88 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -68,24 +68,29 @@ def test_install_subversion_usersite_editable_with_distribute( ) result.assert_installed('INITools', use_user_site=True) - def test_install_curdir_usersite(self, script, virtualenv, data): + @pytest.mark.network + def test_install_from_current_directory_into_usersite( + self, script, virtualenv, data, common_wheels): """ Test installing current directory ('.') into usersite """ virtualenv.system_site_packages = True + script.pip("install", "wheel", '--no-index', '-f', common_wheels) + run_from = data.packages.join("FSPkg") result = script.pip( 'install', '-vvv', '--user', curdir, cwd=run_from, expect_error=False, ) + fspkg_folder = script.user_site / 'fspkg' - egg_info_folder = ( - script.user_site / 'FSPkg-0.1.dev0-py%s.egg-info' % pyversion - ) assert fspkg_folder in result.files_created, result.stdout - assert egg_info_folder in result.files_created + dist_info_folder = ( + script.user_site / 'FSPkg-0.1.dev0.dist-info' + ) + assert dist_info_folder in result.files_created def test_install_user_venv_nositepkgs_fails(self, script, data): """ diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index 69827069982..32d1df95e1c 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -193,7 +193,7 @@ def test_pip_wheel_fail_cause_of_previous_build_dir( result = script.pip( 'wheel', '--no-index', '--find-links=%s' % data.find_links, '--build', script.venv_path / 'build', - 'simple==3.0', expect_error=True, + 'simple==3.0', expect_error=True, expect_temp=True, ) # Then I see that the error code is the right one