config.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556
  1. from __future__ import absolute_import, unicode_literals
  2. import io
  3. import os
  4. import sys
  5. from collections import defaultdict
  6. from functools import partial
  7. from importlib import import_module
  8. from distutils.errors import DistutilsOptionError, DistutilsFileError
  9. from setuptools.extern.six import string_types
  10. def read_configuration(
  11. filepath, find_others=False, ignore_option_errors=False):
  12. """Read given configuration file and returns options from it as a dict.
  13. :param str|unicode filepath: Path to configuration file
  14. to get options from.
  15. :param bool find_others: Whether to search for other configuration files
  16. which could be on in various places.
  17. :param bool ignore_option_errors: Whether to silently ignore
  18. options, values of which could not be resolved (e.g. due to exceptions
  19. in directives such as file:, attr:, etc.).
  20. If False exceptions are propagated as expected.
  21. :rtype: dict
  22. """
  23. from setuptools.dist import Distribution, _Distribution
  24. filepath = os.path.abspath(filepath)
  25. if not os.path.isfile(filepath):
  26. raise DistutilsFileError(
  27. 'Configuration file %s does not exist.' % filepath)
  28. current_directory = os.getcwd()
  29. os.chdir(os.path.dirname(filepath))
  30. try:
  31. dist = Distribution()
  32. filenames = dist.find_config_files() if find_others else []
  33. if filepath not in filenames:
  34. filenames.append(filepath)
  35. _Distribution.parse_config_files(dist, filenames=filenames)
  36. handlers = parse_configuration(
  37. dist, dist.command_options,
  38. ignore_option_errors=ignore_option_errors)
  39. finally:
  40. os.chdir(current_directory)
  41. return configuration_to_dict(handlers)
  42. def configuration_to_dict(handlers):
  43. """Returns configuration data gathered by given handlers as a dict.
  44. :param list[ConfigHandler] handlers: Handlers list,
  45. usually from parse_configuration()
  46. :rtype: dict
  47. """
  48. config_dict = defaultdict(dict)
  49. for handler in handlers:
  50. obj_alias = handler.section_prefix
  51. target_obj = handler.target_obj
  52. for option in handler.set_options:
  53. getter = getattr(target_obj, 'get_%s' % option, None)
  54. if getter is None:
  55. value = getattr(target_obj, option)
  56. else:
  57. value = getter()
  58. config_dict[obj_alias][option] = value
  59. return config_dict
  60. def parse_configuration(
  61. distribution, command_options, ignore_option_errors=False):
  62. """Performs additional parsing of configuration options
  63. for a distribution.
  64. Returns a list of used option handlers.
  65. :param Distribution distribution:
  66. :param dict command_options:
  67. :param bool ignore_option_errors: Whether to silently ignore
  68. options, values of which could not be resolved (e.g. due to exceptions
  69. in directives such as file:, attr:, etc.).
  70. If False exceptions are propagated as expected.
  71. :rtype: list
  72. """
  73. meta = ConfigMetadataHandler(
  74. distribution.metadata, command_options, ignore_option_errors)
  75. meta.parse()
  76. options = ConfigOptionsHandler(
  77. distribution, command_options, ignore_option_errors)
  78. options.parse()
  79. return meta, options
  80. class ConfigHandler(object):
  81. """Handles metadata supplied in configuration files."""
  82. section_prefix = None
  83. """Prefix for config sections handled by this handler.
  84. Must be provided by class heirs.
  85. """
  86. aliases = {}
  87. """Options aliases.
  88. For compatibility with various packages. E.g.: d2to1 and pbr.
  89. Note: `-` in keys is replaced with `_` by config parser.
  90. """
  91. def __init__(self, target_obj, options, ignore_option_errors=False):
  92. sections = {}
  93. section_prefix = self.section_prefix
  94. for section_name, section_options in options.items():
  95. if not section_name.startswith(section_prefix):
  96. continue
  97. section_name = section_name.replace(section_prefix, '').strip('.')
  98. sections[section_name] = section_options
  99. self.ignore_option_errors = ignore_option_errors
  100. self.target_obj = target_obj
  101. self.sections = sections
  102. self.set_options = []
  103. @property
  104. def parsers(self):
  105. """Metadata item name to parser function mapping."""
  106. raise NotImplementedError(
  107. '%s must provide .parsers property' % self.__class__.__name__)
  108. def __setitem__(self, option_name, value):
  109. unknown = tuple()
  110. target_obj = self.target_obj
  111. # Translate alias into real name.
  112. option_name = self.aliases.get(option_name, option_name)
  113. current_value = getattr(target_obj, option_name, unknown)
  114. if current_value is unknown:
  115. raise KeyError(option_name)
  116. if current_value:
  117. # Already inhabited. Skipping.
  118. return
  119. skip_option = False
  120. parser = self.parsers.get(option_name)
  121. if parser:
  122. try:
  123. value = parser(value)
  124. except Exception:
  125. skip_option = True
  126. if not self.ignore_option_errors:
  127. raise
  128. if skip_option:
  129. return
  130. setter = getattr(target_obj, 'set_%s' % option_name, None)
  131. if setter is None:
  132. setattr(target_obj, option_name, value)
  133. else:
  134. setter(value)
  135. self.set_options.append(option_name)
  136. @classmethod
  137. def _parse_list(cls, value, separator=','):
  138. """Represents value as a list.
  139. Value is split either by separator (defaults to comma) or by lines.
  140. :param value:
  141. :param separator: List items separator character.
  142. :rtype: list
  143. """
  144. if isinstance(value, list): # _get_parser_compound case
  145. return value
  146. if '\n' in value:
  147. value = value.splitlines()
  148. else:
  149. value = value.split(separator)
  150. return [chunk.strip() for chunk in value if chunk.strip()]
  151. @classmethod
  152. def _parse_dict(cls, value):
  153. """Represents value as a dict.
  154. :param value:
  155. :rtype: dict
  156. """
  157. separator = '='
  158. result = {}
  159. for line in cls._parse_list(value):
  160. key, sep, val = line.partition(separator)
  161. if sep != separator:
  162. raise DistutilsOptionError(
  163. 'Unable to parse option value to dict: %s' % value)
  164. result[key.strip()] = val.strip()
  165. return result
  166. @classmethod
  167. def _parse_bool(cls, value):
  168. """Represents value as boolean.
  169. :param value:
  170. :rtype: bool
  171. """
  172. value = value.lower()
  173. return value in ('1', 'true', 'yes')
  174. @classmethod
  175. def _parse_file(cls, value):
  176. """Represents value as a string, allowing including text
  177. from nearest files using `file:` directive.
  178. Directive is sandboxed and won't reach anything outside
  179. directory with setup.py.
  180. Examples:
  181. file: LICENSE
  182. file: README.rst, CHANGELOG.md, src/file.txt
  183. :param str value:
  184. :rtype: str
  185. """
  186. include_directive = 'file:'
  187. if not isinstance(value, string_types):
  188. return value
  189. if not value.startswith(include_directive):
  190. return value
  191. spec = value[len(include_directive):]
  192. filepaths = (os.path.abspath(path.strip()) for path in spec.split(','))
  193. return '\n'.join(
  194. cls._read_file(path)
  195. for path in filepaths
  196. if (cls._assert_local(path) or True)
  197. and os.path.isfile(path)
  198. )
  199. @staticmethod
  200. def _assert_local(filepath):
  201. if not filepath.startswith(os.getcwd()):
  202. raise DistutilsOptionError(
  203. '`file:` directive can not access %s' % filepath)
  204. @staticmethod
  205. def _read_file(filepath):
  206. with io.open(filepath, encoding='utf-8') as f:
  207. return f.read()
  208. @classmethod
  209. def _parse_attr(cls, value):
  210. """Represents value as a module attribute.
  211. Examples:
  212. attr: package.attr
  213. attr: package.module.attr
  214. :param str value:
  215. :rtype: str
  216. """
  217. attr_directive = 'attr:'
  218. if not value.startswith(attr_directive):
  219. return value
  220. attrs_path = value.replace(attr_directive, '').strip().split('.')
  221. attr_name = attrs_path.pop()
  222. module_name = '.'.join(attrs_path)
  223. module_name = module_name or '__init__'
  224. sys.path.insert(0, os.getcwd())
  225. try:
  226. module = import_module(module_name)
  227. value = getattr(module, attr_name)
  228. finally:
  229. sys.path = sys.path[1:]
  230. return value
  231. @classmethod
  232. def _get_parser_compound(cls, *parse_methods):
  233. """Returns parser function to represents value as a list.
  234. Parses a value applying given methods one after another.
  235. :param parse_methods:
  236. :rtype: callable
  237. """
  238. def parse(value):
  239. parsed = value
  240. for method in parse_methods:
  241. parsed = method(parsed)
  242. return parsed
  243. return parse
  244. @classmethod
  245. def _parse_section_to_dict(cls, section_options, values_parser=None):
  246. """Parses section options into a dictionary.
  247. Optionally applies a given parser to values.
  248. :param dict section_options:
  249. :param callable values_parser:
  250. :rtype: dict
  251. """
  252. value = {}
  253. values_parser = values_parser or (lambda val: val)
  254. for key, (_, val) in section_options.items():
  255. value[key] = values_parser(val)
  256. return value
  257. def parse_section(self, section_options):
  258. """Parses configuration file section.
  259. :param dict section_options:
  260. """
  261. for (name, (_, value)) in section_options.items():
  262. try:
  263. self[name] = value
  264. except KeyError:
  265. pass # Keep silent for a new option may appear anytime.
  266. def parse(self):
  267. """Parses configuration file items from one
  268. or more related sections.
  269. """
  270. for section_name, section_options in self.sections.items():
  271. method_postfix = ''
  272. if section_name: # [section.option] variant
  273. method_postfix = '_%s' % section_name
  274. section_parser_method = getattr(
  275. self,
  276. # Dots in section names are tranlsated into dunderscores.
  277. ('parse_section%s' % method_postfix).replace('.', '__'),
  278. None)
  279. if section_parser_method is None:
  280. raise DistutilsOptionError(
  281. 'Unsupported distribution option section: [%s.%s]' % (
  282. self.section_prefix, section_name))
  283. section_parser_method(section_options)
  284. class ConfigMetadataHandler(ConfigHandler):
  285. section_prefix = 'metadata'
  286. aliases = {
  287. 'home_page': 'url',
  288. 'summary': 'description',
  289. 'classifier': 'classifiers',
  290. 'platform': 'platforms',
  291. }
  292. strict_mode = False
  293. """We need to keep it loose, to be partially compatible with
  294. `pbr` and `d2to1` packages which also uses `metadata` section.
  295. """
  296. @property
  297. def parsers(self):
  298. """Metadata item name to parser function mapping."""
  299. parse_list = self._parse_list
  300. parse_file = self._parse_file
  301. parse_dict = self._parse_dict
  302. return {
  303. 'platforms': parse_list,
  304. 'keywords': parse_list,
  305. 'provides': parse_list,
  306. 'requires': parse_list,
  307. 'obsoletes': parse_list,
  308. 'classifiers': self._get_parser_compound(parse_file, parse_list),
  309. 'license': parse_file,
  310. 'description': parse_file,
  311. 'long_description': parse_file,
  312. 'version': self._parse_version,
  313. 'project_urls': parse_dict,
  314. }
  315. def _parse_version(self, value):
  316. """Parses `version` option value.
  317. :param value:
  318. :rtype: str
  319. """
  320. version = self._parse_attr(value)
  321. if callable(version):
  322. version = version()
  323. if not isinstance(version, string_types):
  324. if hasattr(version, '__iter__'):
  325. version = '.'.join(map(str, version))
  326. else:
  327. version = '%s' % version
  328. return version
  329. class ConfigOptionsHandler(ConfigHandler):
  330. section_prefix = 'options'
  331. @property
  332. def parsers(self):
  333. """Metadata item name to parser function mapping."""
  334. parse_list = self._parse_list
  335. parse_list_semicolon = partial(self._parse_list, separator=';')
  336. parse_bool = self._parse_bool
  337. parse_dict = self._parse_dict
  338. return {
  339. 'zip_safe': parse_bool,
  340. 'use_2to3': parse_bool,
  341. 'include_package_data': parse_bool,
  342. 'package_dir': parse_dict,
  343. 'use_2to3_fixers': parse_list,
  344. 'use_2to3_exclude_fixers': parse_list,
  345. 'convert_2to3_doctests': parse_list,
  346. 'scripts': parse_list,
  347. 'eager_resources': parse_list,
  348. 'dependency_links': parse_list,
  349. 'namespace_packages': parse_list,
  350. 'install_requires': parse_list_semicolon,
  351. 'setup_requires': parse_list_semicolon,
  352. 'tests_require': parse_list_semicolon,
  353. 'packages': self._parse_packages,
  354. 'entry_points': self._parse_file,
  355. 'py_modules': parse_list,
  356. }
  357. def _parse_packages(self, value):
  358. """Parses `packages` option value.
  359. :param value:
  360. :rtype: list
  361. """
  362. find_directive = 'find:'
  363. if not value.startswith(find_directive):
  364. return self._parse_list(value)
  365. # Read function arguments from a dedicated section.
  366. find_kwargs = self.parse_section_packages__find(
  367. self.sections.get('packages.find', {}))
  368. from setuptools import find_packages
  369. return find_packages(**find_kwargs)
  370. def parse_section_packages__find(self, section_options):
  371. """Parses `packages.find` configuration file section.
  372. To be used in conjunction with _parse_packages().
  373. :param dict section_options:
  374. """
  375. section_data = self._parse_section_to_dict(
  376. section_options, self._parse_list)
  377. valid_keys = ['where', 'include', 'exclude']
  378. find_kwargs = dict(
  379. [(k, v) for k, v in section_data.items() if k in valid_keys and v])
  380. where = find_kwargs.get('where')
  381. if where is not None:
  382. find_kwargs['where'] = where[0] # cast list to single val
  383. return find_kwargs
  384. def parse_section_entry_points(self, section_options):
  385. """Parses `entry_points` configuration file section.
  386. :param dict section_options:
  387. """
  388. parsed = self._parse_section_to_dict(section_options, self._parse_list)
  389. self['entry_points'] = parsed
  390. def _parse_package_data(self, section_options):
  391. parsed = self._parse_section_to_dict(section_options, self._parse_list)
  392. root = parsed.get('*')
  393. if root:
  394. parsed[''] = root
  395. del parsed['*']
  396. return parsed
  397. def parse_section_package_data(self, section_options):
  398. """Parses `package_data` configuration file section.
  399. :param dict section_options:
  400. """
  401. self['package_data'] = self._parse_package_data(section_options)
  402. def parse_section_exclude_package_data(self, section_options):
  403. """Parses `exclude_package_data` configuration file section.
  404. :param dict section_options:
  405. """
  406. self['exclude_package_data'] = self._parse_package_data(
  407. section_options)
  408. def parse_section_extras_require(self, section_options):
  409. """Parses `extras_require` configuration file section.
  410. :param dict section_options:
  411. """
  412. parse_list = partial(self._parse_list, separator=';')
  413. self['extras_require'] = self._parse_section_to_dict(
  414. section_options, parse_list)