wheel.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817
  1. """
  2. Support for installing and building the "wheel" binary package format.
  3. """
  4. from __future__ import absolute_import
  5. import collections
  6. import compileall
  7. import copy
  8. import csv
  9. import hashlib
  10. import logging
  11. import os.path
  12. import re
  13. import shutil
  14. import stat
  15. import sys
  16. import warnings
  17. from base64 import urlsafe_b64encode
  18. from email.parser import Parser
  19. from pip._vendor import pkg_resources
  20. from pip._vendor.distlib.scripts import ScriptMaker
  21. from pip._vendor.packaging.utils import canonicalize_name
  22. from pip._vendor.six import StringIO
  23. from pip._internal import pep425tags
  24. from pip._internal.build_env import BuildEnvironment
  25. from pip._internal.download import path_to_url, unpack_url
  26. from pip._internal.exceptions import (
  27. InstallationError, InvalidWheelFilename, UnsupportedWheel,
  28. )
  29. from pip._internal.locations import (
  30. PIP_DELETE_MARKER_FILENAME, distutils_scheme,
  31. )
  32. from pip._internal.utils.logging import indent_log
  33. from pip._internal.utils.misc import (
  34. call_subprocess, captured_stdout, ensure_dir, read_chunks,
  35. )
  36. from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM
  37. from pip._internal.utils.temp_dir import TempDirectory
  38. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  39. from pip._internal.utils.ui import open_spinner
  40. if MYPY_CHECK_RUNNING:
  41. from typing import Dict, List, Optional
  42. wheel_ext = '.whl'
  43. VERSION_COMPATIBLE = (1, 0)
  44. logger = logging.getLogger(__name__)
  45. def rehash(path, algo='sha256', blocksize=1 << 20):
  46. """Return (hash, length) for path using hashlib.new(algo)"""
  47. h = hashlib.new(algo)
  48. length = 0
  49. with open(path, 'rb') as f:
  50. for block in read_chunks(f, size=blocksize):
  51. length += len(block)
  52. h.update(block)
  53. digest = 'sha256=' + urlsafe_b64encode(
  54. h.digest()
  55. ).decode('latin1').rstrip('=')
  56. return (digest, length)
  57. def open_for_csv(name, mode):
  58. if sys.version_info[0] < 3:
  59. nl = {}
  60. bin = 'b'
  61. else:
  62. nl = {'newline': ''}
  63. bin = ''
  64. return open(name, mode + bin, **nl)
  65. def fix_script(path):
  66. """Replace #!python with #!/path/to/python
  67. Return True if file was changed."""
  68. # XXX RECORD hashes will need to be updated
  69. if os.path.isfile(path):
  70. with open(path, 'rb') as script:
  71. firstline = script.readline()
  72. if not firstline.startswith(b'#!python'):
  73. return False
  74. exename = sys.executable.encode(sys.getfilesystemencoding())
  75. firstline = b'#!' + exename + os.linesep.encode("ascii")
  76. rest = script.read()
  77. with open(path, 'wb') as script:
  78. script.write(firstline)
  79. script.write(rest)
  80. return True
  81. dist_info_re = re.compile(r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>.+?))?)
  82. \.dist-info$""", re.VERBOSE)
  83. def root_is_purelib(name, wheeldir):
  84. """
  85. Return True if the extracted wheel in wheeldir should go into purelib.
  86. """
  87. name_folded = name.replace("-", "_")
  88. for item in os.listdir(wheeldir):
  89. match = dist_info_re.match(item)
  90. if match and match.group('name') == name_folded:
  91. with open(os.path.join(wheeldir, item, 'WHEEL')) as wheel:
  92. for line in wheel:
  93. line = line.lower().rstrip()
  94. if line == "root-is-purelib: true":
  95. return True
  96. return False
  97. def get_entrypoints(filename):
  98. if not os.path.exists(filename):
  99. return {}, {}
  100. # This is done because you can pass a string to entry_points wrappers which
  101. # means that they may or may not be valid INI files. The attempt here is to
  102. # strip leading and trailing whitespace in order to make them valid INI
  103. # files.
  104. with open(filename) as fp:
  105. data = StringIO()
  106. for line in fp:
  107. data.write(line.strip())
  108. data.write("\n")
  109. data.seek(0)
  110. # get the entry points and then the script names
  111. entry_points = pkg_resources.EntryPoint.parse_map(data)
  112. console = entry_points.get('console_scripts', {})
  113. gui = entry_points.get('gui_scripts', {})
  114. def _split_ep(s):
  115. """get the string representation of EntryPoint, remove space and split
  116. on '='"""
  117. return str(s).replace(" ", "").split("=")
  118. # convert the EntryPoint objects into strings with module:function
  119. console = dict(_split_ep(v) for v in console.values())
  120. gui = dict(_split_ep(v) for v in gui.values())
  121. return console, gui
  122. def message_about_scripts_not_on_PATH(scripts):
  123. # type: (List[str]) -> Optional[str]
  124. """Determine if any scripts are not on PATH and format a warning.
  125. Returns a warning message if one or more scripts are not on PATH,
  126. otherwise None.
  127. """
  128. if not scripts:
  129. return None
  130. # Group scripts by the path they were installed in
  131. grouped_by_dir = collections.defaultdict(set) # type: Dict[str, set]
  132. for destfile in scripts:
  133. parent_dir = os.path.dirname(destfile)
  134. script_name = os.path.basename(destfile)
  135. grouped_by_dir[parent_dir].add(script_name)
  136. # We don't want to warn for directories that are on PATH.
  137. not_warn_dirs = [
  138. os.path.normcase(i) for i in os.environ["PATH"].split(os.pathsep)
  139. ]
  140. # If an executable sits with sys.executable, we don't warn for it.
  141. # This covers the case of venv invocations without activating the venv.
  142. not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable)))
  143. warn_for = {
  144. parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items()
  145. if os.path.normcase(parent_dir) not in not_warn_dirs
  146. }
  147. if not warn_for:
  148. return None
  149. # Format a message
  150. msg_lines = []
  151. for parent_dir, scripts in warn_for.items():
  152. scripts = sorted(scripts)
  153. if len(scripts) == 1:
  154. start_text = "script {} is".format(scripts[0])
  155. else:
  156. start_text = "scripts {} are".format(
  157. ", ".join(scripts[:-1]) + " and " + scripts[-1]
  158. )
  159. msg_lines.append(
  160. "The {} installed in '{}' which is not on PATH."
  161. .format(start_text, parent_dir)
  162. )
  163. last_line_fmt = (
  164. "Consider adding {} to PATH or, if you prefer "
  165. "to suppress this warning, use --no-warn-script-location."
  166. )
  167. if len(msg_lines) == 1:
  168. msg_lines.append(last_line_fmt.format("this directory"))
  169. else:
  170. msg_lines.append(last_line_fmt.format("these directories"))
  171. # Returns the formatted multiline message
  172. return "\n".join(msg_lines)
  173. def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None,
  174. pycompile=True, scheme=None, isolated=False, prefix=None,
  175. warn_script_location=True):
  176. """Install a wheel"""
  177. if not scheme:
  178. scheme = distutils_scheme(
  179. name, user=user, home=home, root=root, isolated=isolated,
  180. prefix=prefix,
  181. )
  182. if root_is_purelib(name, wheeldir):
  183. lib_dir = scheme['purelib']
  184. else:
  185. lib_dir = scheme['platlib']
  186. info_dir = []
  187. data_dirs = []
  188. source = wheeldir.rstrip(os.path.sep) + os.path.sep
  189. # Record details of the files moved
  190. # installed = files copied from the wheel to the destination
  191. # changed = files changed while installing (scripts #! line typically)
  192. # generated = files newly generated during the install (script wrappers)
  193. installed = {}
  194. changed = set()
  195. generated = []
  196. # Compile all of the pyc files that we're going to be installing
  197. if pycompile:
  198. with captured_stdout() as stdout:
  199. with warnings.catch_warnings():
  200. warnings.filterwarnings('ignore')
  201. compileall.compile_dir(source, force=True, quiet=True)
  202. logger.debug(stdout.getvalue())
  203. def normpath(src, p):
  204. return os.path.relpath(src, p).replace(os.path.sep, '/')
  205. def record_installed(srcfile, destfile, modified=False):
  206. """Map archive RECORD paths to installation RECORD paths."""
  207. oldpath = normpath(srcfile, wheeldir)
  208. newpath = normpath(destfile, lib_dir)
  209. installed[oldpath] = newpath
  210. if modified:
  211. changed.add(destfile)
  212. def clobber(source, dest, is_base, fixer=None, filter=None):
  213. ensure_dir(dest) # common for the 'include' path
  214. for dir, subdirs, files in os.walk(source):
  215. basedir = dir[len(source):].lstrip(os.path.sep)
  216. destdir = os.path.join(dest, basedir)
  217. if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'):
  218. continue
  219. for s in subdirs:
  220. destsubdir = os.path.join(dest, basedir, s)
  221. if is_base and basedir == '' and destsubdir.endswith('.data'):
  222. data_dirs.append(s)
  223. continue
  224. elif (is_base and
  225. s.endswith('.dist-info') and
  226. canonicalize_name(s).startswith(
  227. canonicalize_name(req.name))):
  228. assert not info_dir, ('Multiple .dist-info directories: ' +
  229. destsubdir + ', ' +
  230. ', '.join(info_dir))
  231. info_dir.append(destsubdir)
  232. for f in files:
  233. # Skip unwanted files
  234. if filter and filter(f):
  235. continue
  236. srcfile = os.path.join(dir, f)
  237. destfile = os.path.join(dest, basedir, f)
  238. # directory creation is lazy and after the file filtering above
  239. # to ensure we don't install empty dirs; empty dirs can't be
  240. # uninstalled.
  241. ensure_dir(destdir)
  242. # We use copyfile (not move, copy, or copy2) to be extra sure
  243. # that we are not moving directories over (copyfile fails for
  244. # directories) as well as to ensure that we are not copying
  245. # over any metadata because we want more control over what
  246. # metadata we actually copy over.
  247. shutil.copyfile(srcfile, destfile)
  248. # Copy over the metadata for the file, currently this only
  249. # includes the atime and mtime.
  250. st = os.stat(srcfile)
  251. if hasattr(os, "utime"):
  252. os.utime(destfile, (st.st_atime, st.st_mtime))
  253. # If our file is executable, then make our destination file
  254. # executable.
  255. if os.access(srcfile, os.X_OK):
  256. st = os.stat(srcfile)
  257. permissions = (
  258. st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
  259. )
  260. os.chmod(destfile, permissions)
  261. changed = False
  262. if fixer:
  263. changed = fixer(destfile)
  264. record_installed(srcfile, destfile, changed)
  265. clobber(source, lib_dir, True)
  266. assert info_dir, "%s .dist-info directory not found" % req
  267. # Get the defined entry points
  268. ep_file = os.path.join(info_dir[0], 'entry_points.txt')
  269. console, gui = get_entrypoints(ep_file)
  270. def is_entrypoint_wrapper(name):
  271. # EP, EP.exe and EP-script.py are scripts generated for
  272. # entry point EP by setuptools
  273. if name.lower().endswith('.exe'):
  274. matchname = name[:-4]
  275. elif name.lower().endswith('-script.py'):
  276. matchname = name[:-10]
  277. elif name.lower().endswith(".pya"):
  278. matchname = name[:-4]
  279. else:
  280. matchname = name
  281. # Ignore setuptools-generated scripts
  282. return (matchname in console or matchname in gui)
  283. for datadir in data_dirs:
  284. fixer = None
  285. filter = None
  286. for subdir in os.listdir(os.path.join(wheeldir, datadir)):
  287. fixer = None
  288. if subdir == 'scripts':
  289. fixer = fix_script
  290. filter = is_entrypoint_wrapper
  291. source = os.path.join(wheeldir, datadir, subdir)
  292. dest = scheme[subdir]
  293. clobber(source, dest, False, fixer=fixer, filter=filter)
  294. maker = ScriptMaker(None, scheme['scripts'])
  295. # Ensure old scripts are overwritten.
  296. # See https://github.com/pypa/pip/issues/1800
  297. maker.clobber = True
  298. # Ensure we don't generate any variants for scripts because this is almost
  299. # never what somebody wants.
  300. # See https://bitbucket.org/pypa/distlib/issue/35/
  301. maker.variants = {''}
  302. # This is required because otherwise distlib creates scripts that are not
  303. # executable.
  304. # See https://bitbucket.org/pypa/distlib/issue/32/
  305. maker.set_mode = True
  306. # Simplify the script and fix the fact that the default script swallows
  307. # every single stack trace.
  308. # See https://bitbucket.org/pypa/distlib/issue/34/
  309. # See https://bitbucket.org/pypa/distlib/issue/33/
  310. def _get_script_text(entry):
  311. if entry.suffix is None:
  312. raise InstallationError(
  313. "Invalid script entry point: %s for req: %s - A callable "
  314. "suffix is required. Cf https://packaging.python.org/en/"
  315. "latest/distributing.html#console-scripts for more "
  316. "information." % (entry, req)
  317. )
  318. return maker.script_template % {
  319. "module": entry.prefix,
  320. "import_name": entry.suffix.split(".")[0],
  321. "func": entry.suffix,
  322. }
  323. maker._get_script_text = _get_script_text
  324. maker.script_template = r"""# -*- coding: utf-8 -*-
  325. import re
  326. import sys
  327. from %(module)s import %(import_name)s
  328. if __name__ == '__main__':
  329. sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
  330. sys.exit(%(func)s())
  331. """
  332. # Special case pip and setuptools to generate versioned wrappers
  333. #
  334. # The issue is that some projects (specifically, pip and setuptools) use
  335. # code in setup.py to create "versioned" entry points - pip2.7 on Python
  336. # 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into
  337. # the wheel metadata at build time, and so if the wheel is installed with
  338. # a *different* version of Python the entry points will be wrong. The
  339. # correct fix for this is to enhance the metadata to be able to describe
  340. # such versioned entry points, but that won't happen till Metadata 2.0 is
  341. # available.
  342. # In the meantime, projects using versioned entry points will either have
  343. # incorrect versioned entry points, or they will not be able to distribute
  344. # "universal" wheels (i.e., they will need a wheel per Python version).
  345. #
  346. # Because setuptools and pip are bundled with _ensurepip and virtualenv,
  347. # we need to use universal wheels. So, as a stopgap until Metadata 2.0, we
  348. # override the versioned entry points in the wheel and generate the
  349. # correct ones. This code is purely a short-term measure until Metadata 2.0
  350. # is available.
  351. #
  352. # To add the level of hack in this section of code, in order to support
  353. # ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment
  354. # variable which will control which version scripts get installed.
  355. #
  356. # ENSUREPIP_OPTIONS=altinstall
  357. # - Only pipX.Y and easy_install-X.Y will be generated and installed
  358. # ENSUREPIP_OPTIONS=install
  359. # - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note
  360. # that this option is technically if ENSUREPIP_OPTIONS is set and is
  361. # not altinstall
  362. # DEFAULT
  363. # - The default behavior is to install pip, pipX, pipX.Y, easy_install
  364. # and easy_install-X.Y.
  365. pip_script = console.pop('pip', None)
  366. if pip_script:
  367. if "ENSUREPIP_OPTIONS" not in os.environ:
  368. spec = 'pip = ' + pip_script
  369. generated.extend(maker.make(spec))
  370. if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall":
  371. spec = 'pip%s = %s' % (sys.version[:1], pip_script)
  372. generated.extend(maker.make(spec))
  373. spec = 'pip%s = %s' % (sys.version[:3], pip_script)
  374. generated.extend(maker.make(spec))
  375. # Delete any other versioned pip entry points
  376. pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)]
  377. for k in pip_ep:
  378. del console[k]
  379. easy_install_script = console.pop('easy_install', None)
  380. if easy_install_script:
  381. if "ENSUREPIP_OPTIONS" not in os.environ:
  382. spec = 'easy_install = ' + easy_install_script
  383. generated.extend(maker.make(spec))
  384. spec = 'easy_install-%s = %s' % (sys.version[:3], easy_install_script)
  385. generated.extend(maker.make(spec))
  386. # Delete any other versioned easy_install entry points
  387. easy_install_ep = [
  388. k for k in console if re.match(r'easy_install(-\d\.\d)?$', k)
  389. ]
  390. for k in easy_install_ep:
  391. del console[k]
  392. # Generate the console and GUI entry points specified in the wheel
  393. if len(console) > 0:
  394. generated_console_scripts = maker.make_multiple(
  395. ['%s = %s' % kv for kv in console.items()]
  396. )
  397. generated.extend(generated_console_scripts)
  398. if warn_script_location:
  399. msg = message_about_scripts_not_on_PATH(generated_console_scripts)
  400. if msg is not None:
  401. logger.warn(msg)
  402. if len(gui) > 0:
  403. generated.extend(
  404. maker.make_multiple(
  405. ['%s = %s' % kv for kv in gui.items()],
  406. {'gui': True}
  407. )
  408. )
  409. # Record pip as the installer
  410. installer = os.path.join(info_dir[0], 'INSTALLER')
  411. temp_installer = os.path.join(info_dir[0], 'INSTALLER.pip')
  412. with open(temp_installer, 'wb') as installer_file:
  413. installer_file.write(b'pip\n')
  414. shutil.move(temp_installer, installer)
  415. generated.append(installer)
  416. # Record details of all files installed
  417. record = os.path.join(info_dir[0], 'RECORD')
  418. temp_record = os.path.join(info_dir[0], 'RECORD.pip')
  419. with open_for_csv(record, 'r') as record_in:
  420. with open_for_csv(temp_record, 'w+') as record_out:
  421. reader = csv.reader(record_in)
  422. writer = csv.writer(record_out)
  423. for row in reader:
  424. row[0] = installed.pop(row[0], row[0])
  425. if row[0] in changed:
  426. row[1], row[2] = rehash(row[0])
  427. writer.writerow(row)
  428. for f in generated:
  429. h, l = rehash(f)
  430. writer.writerow((normpath(f, lib_dir), h, l))
  431. for f in installed:
  432. writer.writerow((installed[f], '', ''))
  433. shutil.move(temp_record, record)
  434. def wheel_version(source_dir):
  435. """
  436. Return the Wheel-Version of an extracted wheel, if possible.
  437. Otherwise, return False if we couldn't parse / extract it.
  438. """
  439. try:
  440. dist = [d for d in pkg_resources.find_on_path(None, source_dir)][0]
  441. wheel_data = dist.get_metadata('WHEEL')
  442. wheel_data = Parser().parsestr(wheel_data)
  443. version = wheel_data['Wheel-Version'].strip()
  444. version = tuple(map(int, version.split('.')))
  445. return version
  446. except:
  447. return False
  448. def check_compatibility(version, name):
  449. """
  450. Raises errors or warns if called with an incompatible Wheel-Version.
  451. Pip should refuse to install a Wheel-Version that's a major series
  452. ahead of what it's compatible with (e.g 2.0 > 1.1); and warn when
  453. installing a version only minor version ahead (e.g 1.2 > 1.1).
  454. version: a 2-tuple representing a Wheel-Version (Major, Minor)
  455. name: name of wheel or package to raise exception about
  456. :raises UnsupportedWheel: when an incompatible Wheel-Version is given
  457. """
  458. if not version:
  459. raise UnsupportedWheel(
  460. "%s is in an unsupported or invalid wheel" % name
  461. )
  462. if version[0] > VERSION_COMPATIBLE[0]:
  463. raise UnsupportedWheel(
  464. "%s's Wheel-Version (%s) is not compatible with this version "
  465. "of pip" % (name, '.'.join(map(str, version)))
  466. )
  467. elif version > VERSION_COMPATIBLE:
  468. logger.warning(
  469. 'Installing from a newer Wheel-Version (%s)',
  470. '.'.join(map(str, version)),
  471. )
  472. class Wheel(object):
  473. """A wheel file"""
  474. # TODO: maybe move the install code into this class
  475. wheel_file_re = re.compile(
  476. r"""^(?P<namever>(?P<name>.+?)-(?P<ver>.*?))
  477. ((-(?P<build>\d[^-]*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
  478. \.whl|\.dist-info)$""",
  479. re.VERBOSE
  480. )
  481. def __init__(self, filename):
  482. """
  483. :raises InvalidWheelFilename: when the filename is invalid for a wheel
  484. """
  485. wheel_info = self.wheel_file_re.match(filename)
  486. if not wheel_info:
  487. raise InvalidWheelFilename(
  488. "%s is not a valid wheel filename." % filename
  489. )
  490. self.filename = filename
  491. self.name = wheel_info.group('name').replace('_', '-')
  492. # we'll assume "_" means "-" due to wheel naming scheme
  493. # (https://github.com/pypa/pip/issues/1150)
  494. self.version = wheel_info.group('ver').replace('_', '-')
  495. self.build_tag = wheel_info.group('build')
  496. self.pyversions = wheel_info.group('pyver').split('.')
  497. self.abis = wheel_info.group('abi').split('.')
  498. self.plats = wheel_info.group('plat').split('.')
  499. # All the tag combinations from this file
  500. self.file_tags = {
  501. (x, y, z) for x in self.pyversions
  502. for y in self.abis for z in self.plats
  503. }
  504. def support_index_min(self, tags=None):
  505. """
  506. Return the lowest index that one of the wheel's file_tag combinations
  507. achieves in the supported_tags list e.g. if there are 8 supported tags,
  508. and one of the file tags is first in the list, then return 0. Returns
  509. None is the wheel is not supported.
  510. """
  511. if tags is None: # for mock
  512. tags = pep425tags.get_supported()
  513. indexes = [tags.index(c) for c in self.file_tags if c in tags]
  514. return min(indexes) if indexes else None
  515. def supported(self, tags=None):
  516. """Is this wheel supported on this system?"""
  517. if tags is None: # for mock
  518. tags = pep425tags.get_supported()
  519. return bool(set(tags).intersection(self.file_tags))
  520. class WheelBuilder(object):
  521. """Build wheels from a RequirementSet."""
  522. def __init__(self, finder, preparer, wheel_cache,
  523. build_options=None, global_options=None, no_clean=False):
  524. self.finder = finder
  525. self.preparer = preparer
  526. self.wheel_cache = wheel_cache
  527. self._wheel_dir = preparer.wheel_download_dir
  528. self.build_options = build_options or []
  529. self.global_options = global_options or []
  530. self.no_clean = no_clean
  531. def _build_one(self, req, output_dir, python_tag=None):
  532. """Build one wheel.
  533. :return: The filename of the built wheel, or None if the build failed.
  534. """
  535. # Install build deps into temporary directory (PEP 518)
  536. with req.build_env:
  537. return self._build_one_inside_env(req, output_dir,
  538. python_tag=python_tag)
  539. def _build_one_inside_env(self, req, output_dir, python_tag=None):
  540. with TempDirectory(kind="wheel") as temp_dir:
  541. if self.__build_one(req, temp_dir.path, python_tag=python_tag):
  542. try:
  543. wheel_name = os.listdir(temp_dir.path)[0]
  544. wheel_path = os.path.join(output_dir, wheel_name)
  545. shutil.move(
  546. os.path.join(temp_dir.path, wheel_name), wheel_path
  547. )
  548. logger.info('Stored in directory: %s', output_dir)
  549. return wheel_path
  550. except:
  551. pass
  552. # Ignore return, we can't do anything else useful.
  553. self._clean_one(req)
  554. return None
  555. def _base_setup_args(self, req):
  556. # NOTE: Eventually, we'd want to also -S to the flags here, when we're
  557. # isolating. Currently, it breaks Python in virtualenvs, because it
  558. # relies on site.py to find parts of the standard library outside the
  559. # virtualenv.
  560. return [
  561. sys.executable, '-u', '-c',
  562. SETUPTOOLS_SHIM % req.setup_py
  563. ] + list(self.global_options)
  564. def __build_one(self, req, tempd, python_tag=None):
  565. base_args = self._base_setup_args(req)
  566. spin_message = 'Running setup.py bdist_wheel for %s' % (req.name,)
  567. with open_spinner(spin_message) as spinner:
  568. logger.debug('Destination directory: %s', tempd)
  569. wheel_args = base_args + ['bdist_wheel', '-d', tempd] \
  570. + self.build_options
  571. if python_tag is not None:
  572. wheel_args += ["--python-tag", python_tag]
  573. try:
  574. call_subprocess(wheel_args, cwd=req.setup_py_dir,
  575. show_stdout=False, spinner=spinner)
  576. return True
  577. except:
  578. spinner.finish("error")
  579. logger.error('Failed building wheel for %s', req.name)
  580. return False
  581. def _clean_one(self, req):
  582. base_args = self._base_setup_args(req)
  583. logger.info('Running setup.py clean for %s', req.name)
  584. clean_args = base_args + ['clean', '--all']
  585. try:
  586. call_subprocess(clean_args, cwd=req.source_dir, show_stdout=False)
  587. return True
  588. except:
  589. logger.error('Failed cleaning build dir for %s', req.name)
  590. return False
  591. def build(self, requirements, session, autobuilding=False):
  592. """Build wheels.
  593. :param unpack: If True, replace the sdist we built from with the
  594. newly built wheel, in preparation for installation.
  595. :return: True if all the wheels built correctly.
  596. """
  597. from pip._internal import index
  598. building_is_possible = self._wheel_dir or (
  599. autobuilding and self.wheel_cache.cache_dir
  600. )
  601. assert building_is_possible
  602. buildset = []
  603. for req in requirements:
  604. if req.constraint:
  605. continue
  606. if req.is_wheel:
  607. if not autobuilding:
  608. logger.info(
  609. 'Skipping %s, due to already being wheel.', req.name,
  610. )
  611. elif autobuilding and req.editable:
  612. pass
  613. elif autobuilding and not req.source_dir:
  614. pass
  615. elif autobuilding and req.link and not req.link.is_artifact:
  616. # VCS checkout. Build wheel just for this run.
  617. buildset.append((req, True))
  618. else:
  619. ephem_cache = False
  620. if autobuilding:
  621. link = req.link
  622. base, ext = link.splitext()
  623. if index.egg_info_matches(base, None, link) is None:
  624. # E.g. local directory. Build wheel just for this run.
  625. ephem_cache = True
  626. if "binary" not in index.fmt_ctl_formats(
  627. self.finder.format_control,
  628. canonicalize_name(req.name)):
  629. logger.info(
  630. "Skipping bdist_wheel for %s, due to binaries "
  631. "being disabled for it.", req.name,
  632. )
  633. continue
  634. buildset.append((req, ephem_cache))
  635. if not buildset:
  636. return True
  637. # Build the wheels.
  638. logger.info(
  639. 'Building wheels for collected packages: %s',
  640. ', '.join([req.name for (req, _) in buildset]),
  641. )
  642. _cache = self.wheel_cache # shorter name
  643. with indent_log():
  644. build_success, build_failure = [], []
  645. for req, ephem in buildset:
  646. python_tag = None
  647. if autobuilding:
  648. python_tag = pep425tags.implementation_tag
  649. if ephem:
  650. output_dir = _cache.get_ephem_path_for_link(req.link)
  651. else:
  652. output_dir = _cache.get_path_for_link(req.link)
  653. try:
  654. ensure_dir(output_dir)
  655. except OSError as e:
  656. logger.warning("Building wheel for %s failed: %s",
  657. req.name, e)
  658. build_failure.append(req)
  659. continue
  660. else:
  661. output_dir = self._wheel_dir
  662. wheel_file = self._build_one(
  663. req, output_dir,
  664. python_tag=python_tag,
  665. )
  666. if wheel_file:
  667. build_success.append(req)
  668. if autobuilding:
  669. # XXX: This is mildly duplicative with prepare_files,
  670. # but not close enough to pull out to a single common
  671. # method.
  672. # The code below assumes temporary source dirs -
  673. # prevent it doing bad things.
  674. if req.source_dir and not os.path.exists(os.path.join(
  675. req.source_dir, PIP_DELETE_MARKER_FILENAME)):
  676. raise AssertionError(
  677. "bad source dir - missing marker")
  678. # Delete the source we built the wheel from
  679. req.remove_temporary_source()
  680. # set the build directory again - name is known from
  681. # the work prepare_files did.
  682. req.source_dir = req.build_location(
  683. self.preparer.build_dir
  684. )
  685. # Update the link for this.
  686. req.link = index.Link(path_to_url(wheel_file))
  687. assert req.link.is_wheel
  688. # extract the wheel into the dir
  689. unpack_url(
  690. req.link, req.source_dir, None, False,
  691. session=session,
  692. )
  693. else:
  694. build_failure.append(req)
  695. # notify success/failure
  696. if build_success:
  697. logger.info(
  698. 'Successfully built %s',
  699. ' '.join([req.name for req in build_success]),
  700. )
  701. if build_failure:
  702. logger.info(
  703. 'Failed to build %s',
  704. ' '.join([req.name for req in build_failure]),
  705. )
  706. # Return True if all builds were successful
  707. return len(build_failure) == 0