resolve.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354
  1. """Dependency Resolution
  2. The dependency resolution in pip is performed as follows:
  3. for top-level requirements:
  4. a. only one spec allowed per project, regardless of conflicts or not.
  5. otherwise a "double requirement" exception is raised
  6. b. they override sub-dependency requirements.
  7. for sub-dependencies
  8. a. "first found, wins" (where the order is breadth first)
  9. """
  10. import logging
  11. from collections import defaultdict
  12. from itertools import chain
  13. from pip._internal.exceptions import (
  14. BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors,
  15. UnsupportedPythonVersion,
  16. )
  17. from pip._internal.req.req_install import InstallRequirement
  18. from pip._internal.utils.logging import indent_log
  19. from pip._internal.utils.misc import dist_in_usersite, ensure_dir
  20. from pip._internal.utils.packaging import check_dist_requires_python
  21. logger = logging.getLogger(__name__)
  22. class Resolver(object):
  23. """Resolves which packages need to be installed/uninstalled to perform \
  24. the requested operation without breaking the requirements of any package.
  25. """
  26. _allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"}
  27. def __init__(self, preparer, session, finder, wheel_cache, use_user_site,
  28. ignore_dependencies, ignore_installed, ignore_requires_python,
  29. force_reinstall, isolated, upgrade_strategy):
  30. super(Resolver, self).__init__()
  31. assert upgrade_strategy in self._allowed_strategies
  32. self.preparer = preparer
  33. self.finder = finder
  34. self.session = session
  35. # NOTE: This would eventually be replaced with a cache that can give
  36. # information about both sdist and wheels transparently.
  37. self.wheel_cache = wheel_cache
  38. self.require_hashes = None # This is set in resolve
  39. self.upgrade_strategy = upgrade_strategy
  40. self.force_reinstall = force_reinstall
  41. self.isolated = isolated
  42. self.ignore_dependencies = ignore_dependencies
  43. self.ignore_installed = ignore_installed
  44. self.ignore_requires_python = ignore_requires_python
  45. self.use_user_site = use_user_site
  46. self._discovered_dependencies = defaultdict(list)
  47. def resolve(self, requirement_set):
  48. """Resolve what operations need to be done
  49. As a side-effect of this method, the packages (and their dependencies)
  50. are downloaded, unpacked and prepared for installation. This
  51. preparation is done by ``pip.operations.prepare``.
  52. Once PyPI has static dependency metadata available, it would be
  53. possible to move the preparation to become a step separated from
  54. dependency resolution.
  55. """
  56. # make the wheelhouse
  57. if self.preparer.wheel_download_dir:
  58. ensure_dir(self.preparer.wheel_download_dir)
  59. # If any top-level requirement has a hash specified, enter
  60. # hash-checking mode, which requires hashes from all.
  61. root_reqs = (
  62. requirement_set.unnamed_requirements +
  63. list(requirement_set.requirements.values())
  64. )
  65. self.require_hashes = (
  66. requirement_set.require_hashes or
  67. any(req.has_hash_options for req in root_reqs)
  68. )
  69. # Display where finder is looking for packages
  70. locations = self.finder.get_formatted_locations()
  71. if locations:
  72. logger.info(locations)
  73. # Actually prepare the files, and collect any exceptions. Most hash
  74. # exceptions cannot be checked ahead of time, because
  75. # req.populate_link() needs to be called before we can make decisions
  76. # based on link type.
  77. discovered_reqs = []
  78. hash_errors = HashErrors()
  79. for req in chain(root_reqs, discovered_reqs):
  80. try:
  81. discovered_reqs.extend(
  82. self._resolve_one(requirement_set, req)
  83. )
  84. except HashError as exc:
  85. exc.req = req
  86. hash_errors.append(exc)
  87. if hash_errors:
  88. raise hash_errors
  89. def _is_upgrade_allowed(self, req):
  90. if self.upgrade_strategy == "to-satisfy-only":
  91. return False
  92. elif self.upgrade_strategy == "eager":
  93. return True
  94. else:
  95. assert self.upgrade_strategy == "only-if-needed"
  96. return req.is_direct
  97. def _set_req_to_reinstall(self, req):
  98. """
  99. Set a requirement to be installed.
  100. """
  101. # Don't uninstall the conflict if doing a user install and the
  102. # conflict is not a user install.
  103. if not self.use_user_site or dist_in_usersite(req.satisfied_by):
  104. req.conflicts_with = req.satisfied_by
  105. req.satisfied_by = None
  106. # XXX: Stop passing requirement_set for options
  107. def _check_skip_installed(self, req_to_install):
  108. """Check if req_to_install should be skipped.
  109. This will check if the req is installed, and whether we should upgrade
  110. or reinstall it, taking into account all the relevant user options.
  111. After calling this req_to_install will only have satisfied_by set to
  112. None if the req_to_install is to be upgraded/reinstalled etc. Any
  113. other value will be a dist recording the current thing installed that
  114. satisfies the requirement.
  115. Note that for vcs urls and the like we can't assess skipping in this
  116. routine - we simply identify that we need to pull the thing down,
  117. then later on it is pulled down and introspected to assess upgrade/
  118. reinstalls etc.
  119. :return: A text reason for why it was skipped, or None.
  120. """
  121. if self.ignore_installed:
  122. return None
  123. req_to_install.check_if_exists(self.use_user_site)
  124. if not req_to_install.satisfied_by:
  125. return None
  126. if self.force_reinstall:
  127. self._set_req_to_reinstall(req_to_install)
  128. return None
  129. if not self._is_upgrade_allowed(req_to_install):
  130. if self.upgrade_strategy == "only-if-needed":
  131. return 'not upgraded as not directly required'
  132. return 'already satisfied'
  133. # Check for the possibility of an upgrade. For link-based
  134. # requirements we have to pull the tree down and inspect to assess
  135. # the version #, so it's handled way down.
  136. if not req_to_install.link:
  137. try:
  138. self.finder.find_requirement(req_to_install, upgrade=True)
  139. except BestVersionAlreadyInstalled:
  140. # Then the best version is installed.
  141. return 'already up-to-date'
  142. except DistributionNotFound:
  143. # No distribution found, so we squash the error. It will
  144. # be raised later when we re-try later to do the install.
  145. # Why don't we just raise here?
  146. pass
  147. self._set_req_to_reinstall(req_to_install)
  148. return None
  149. def _get_abstract_dist_for(self, req):
  150. """Takes a InstallRequirement and returns a single AbstractDist \
  151. representing a prepared variant of the same.
  152. """
  153. assert self.require_hashes is not None, (
  154. "require_hashes should have been set in Resolver.resolve()"
  155. )
  156. if req.editable:
  157. return self.preparer.prepare_editable_requirement(
  158. req, self.require_hashes, self.use_user_site, self.finder,
  159. )
  160. # satisfied_by is only evaluated by calling _check_skip_installed,
  161. # so it must be None here.
  162. assert req.satisfied_by is None
  163. skip_reason = self._check_skip_installed(req)
  164. if req.satisfied_by:
  165. return self.preparer.prepare_installed_requirement(
  166. req, self.require_hashes, skip_reason
  167. )
  168. upgrade_allowed = self._is_upgrade_allowed(req)
  169. abstract_dist = self.preparer.prepare_linked_requirement(
  170. req, self.session, self.finder, upgrade_allowed,
  171. self.require_hashes
  172. )
  173. # NOTE
  174. # The following portion is for determining if a certain package is
  175. # going to be re-installed/upgraded or not and reporting to the user.
  176. # This should probably get cleaned up in a future refactor.
  177. # req.req is only avail after unpack for URL
  178. # pkgs repeat check_if_exists to uninstall-on-upgrade
  179. # (#14)
  180. if not self.ignore_installed:
  181. req.check_if_exists(self.use_user_site)
  182. if req.satisfied_by:
  183. should_modify = (
  184. self.upgrade_strategy != "to-satisfy-only" or
  185. self.force_reinstall or
  186. self.ignore_installed or
  187. req.link.scheme == 'file'
  188. )
  189. if should_modify:
  190. self._set_req_to_reinstall(req)
  191. else:
  192. logger.info(
  193. 'Requirement already satisfied (use --upgrade to upgrade):'
  194. ' %s', req,
  195. )
  196. return abstract_dist
  197. def _resolve_one(self, requirement_set, req_to_install):
  198. """Prepare a single requirements file.
  199. :return: A list of additional InstallRequirements to also install.
  200. """
  201. # Tell user what we are doing for this requirement:
  202. # obtain (editable), skipping, processing (local url), collecting
  203. # (remote url or package name)
  204. if req_to_install.constraint or req_to_install.prepared:
  205. return []
  206. req_to_install.prepared = True
  207. # register tmp src for cleanup in case something goes wrong
  208. requirement_set.reqs_to_cleanup.append(req_to_install)
  209. abstract_dist = self._get_abstract_dist_for(req_to_install)
  210. # Parse and return dependencies
  211. dist = abstract_dist.dist(self.finder)
  212. try:
  213. check_dist_requires_python(dist)
  214. except UnsupportedPythonVersion as err:
  215. if self.ignore_requires_python:
  216. logger.warning(err.args[0])
  217. else:
  218. raise
  219. more_reqs = []
  220. def add_req(subreq, extras_requested):
  221. sub_install_req = InstallRequirement.from_req(
  222. str(subreq),
  223. req_to_install,
  224. isolated=self.isolated,
  225. wheel_cache=self.wheel_cache,
  226. )
  227. parent_req_name = req_to_install.name
  228. to_scan_again, add_to_parent = requirement_set.add_requirement(
  229. sub_install_req,
  230. parent_req_name=parent_req_name,
  231. extras_requested=extras_requested,
  232. )
  233. if parent_req_name and add_to_parent:
  234. self._discovered_dependencies[parent_req_name].append(
  235. add_to_parent
  236. )
  237. more_reqs.extend(to_scan_again)
  238. with indent_log():
  239. # We add req_to_install before its dependencies, so that we
  240. # can refer to it when adding dependencies.
  241. if not requirement_set.has_requirement(req_to_install.name):
  242. # 'unnamed' requirements will get added here
  243. req_to_install.is_direct = True
  244. requirement_set.add_requirement(
  245. req_to_install, parent_req_name=None,
  246. )
  247. if not self.ignore_dependencies:
  248. if req_to_install.extras:
  249. logger.debug(
  250. "Installing extra requirements: %r",
  251. ','.join(req_to_install.extras),
  252. )
  253. missing_requested = sorted(
  254. set(req_to_install.extras) - set(dist.extras)
  255. )
  256. for missing in missing_requested:
  257. logger.warning(
  258. '%s does not provide the extra \'%s\'',
  259. dist, missing
  260. )
  261. available_requested = sorted(
  262. set(dist.extras) & set(req_to_install.extras)
  263. )
  264. for subreq in dist.requires(available_requested):
  265. add_req(subreq, extras_requested=available_requested)
  266. if not req_to_install.editable and not req_to_install.satisfied_by:
  267. # XXX: --no-install leads this to report 'Successfully
  268. # downloaded' for only non-editable reqs, even though we took
  269. # action on them.
  270. requirement_set.successfully_downloaded.append(req_to_install)
  271. return more_reqs
  272. def get_installation_order(self, req_set):
  273. """Create the installation order.
  274. The installation order is topological - requirements are installed
  275. before the requiring thing. We break cycles at an arbitrary point,
  276. and make no other guarantees.
  277. """
  278. # The current implementation, which we may change at any point
  279. # installs the user specified things in the order given, except when
  280. # dependencies must come earlier to achieve topological order.
  281. order = []
  282. ordered_reqs = set()
  283. def schedule(req):
  284. if req.satisfied_by or req in ordered_reqs:
  285. return
  286. if req.constraint:
  287. return
  288. ordered_reqs.add(req)
  289. for dep in self._discovered_dependencies[req.name]:
  290. schedule(dep)
  291. order.append(req)
  292. for install_req in req_set.requirements.values():
  293. schedule(install_req)
  294. return order