build_py.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298
  1. from functools import partial
  2. from glob import glob
  3. from distutils.util import convert_path
  4. import distutils.command.build_py as orig
  5. import os
  6. import fnmatch
  7. import textwrap
  8. import io
  9. import distutils.errors
  10. import itertools
  11. import stat
  12. import warnings
  13. from pathlib import Path
  14. from setuptools._deprecation_warning import SetuptoolsDeprecationWarning
  15. from setuptools.extern.more_itertools import unique_everseen
  16. def make_writable(target):
  17. os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE)
  18. class build_py(orig.build_py):
  19. """Enhanced 'build_py' command that includes data files with packages
  20. The data files are specified via a 'package_data' argument to 'setup()'.
  21. See 'setuptools.dist.Distribution' for more details.
  22. Also, this version of the 'build_py' command allows you to specify both
  23. 'py_modules' and 'packages' in the same setup operation.
  24. """
  25. def finalize_options(self):
  26. orig.build_py.finalize_options(self)
  27. self.package_data = self.distribution.package_data
  28. self.exclude_package_data = self.distribution.exclude_package_data or {}
  29. if 'data_files' in self.__dict__:
  30. del self.__dict__['data_files']
  31. self.__updated_files = []
  32. def run(self):
  33. """Build modules, packages, and copy data files to build directory"""
  34. if not self.py_modules and not self.packages:
  35. return
  36. if self.py_modules:
  37. self.build_modules()
  38. if self.packages:
  39. self.build_packages()
  40. self.build_package_data()
  41. # Only compile actual .py files, using our base class' idea of what our
  42. # output files are.
  43. self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))
  44. def __getattr__(self, attr):
  45. "lazily compute data files"
  46. if attr == 'data_files':
  47. self.data_files = self._get_data_files()
  48. return self.data_files
  49. return orig.build_py.__getattr__(self, attr)
  50. def build_module(self, module, module_file, package):
  51. outfile, copied = orig.build_py.build_module(self, module, module_file, package)
  52. if copied:
  53. self.__updated_files.append(outfile)
  54. return outfile, copied
  55. def _get_data_files(self):
  56. """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
  57. self.analyze_manifest()
  58. return list(map(self._get_pkg_data_files, self.packages or ()))
  59. def get_data_files_without_manifest(self):
  60. """
  61. Generate list of ``(package,src_dir,build_dir,filenames)`` tuples,
  62. but without triggering any attempt to analyze or build the manifest.
  63. """
  64. # Prevent eventual errors from unset `manifest_files`
  65. # (that would otherwise be set by `analyze_manifest`)
  66. self.__dict__.setdefault('manifest_files', {})
  67. return list(map(self._get_pkg_data_files, self.packages or ()))
  68. def _get_pkg_data_files(self, package):
  69. # Locate package source directory
  70. src_dir = self.get_package_dir(package)
  71. # Compute package build directory
  72. build_dir = os.path.join(*([self.build_lib] + package.split('.')))
  73. # Strip directory from globbed filenames
  74. filenames = [
  75. os.path.relpath(file, src_dir)
  76. for file in self.find_data_files(package, src_dir)
  77. ]
  78. return package, src_dir, build_dir, filenames
  79. def find_data_files(self, package, src_dir):
  80. """Return filenames for package's data files in 'src_dir'"""
  81. patterns = self._get_platform_patterns(
  82. self.package_data,
  83. package,
  84. src_dir,
  85. )
  86. globs_expanded = map(partial(glob, recursive=True), patterns)
  87. # flatten the expanded globs into an iterable of matches
  88. globs_matches = itertools.chain.from_iterable(globs_expanded)
  89. glob_files = filter(os.path.isfile, globs_matches)
  90. files = itertools.chain(
  91. self.manifest_files.get(package, []),
  92. glob_files,
  93. )
  94. return self.exclude_data_files(package, src_dir, files)
  95. def build_package_data(self):
  96. """Copy data files into build directory"""
  97. for package, src_dir, build_dir, filenames in self.data_files:
  98. for filename in filenames:
  99. target = os.path.join(build_dir, filename)
  100. self.mkpath(os.path.dirname(target))
  101. srcfile = os.path.join(src_dir, filename)
  102. outf, copied = self.copy_file(srcfile, target)
  103. make_writable(target)
  104. srcfile = os.path.abspath(srcfile)
  105. def analyze_manifest(self):
  106. self.manifest_files = mf = {}
  107. if not self.distribution.include_package_data:
  108. return
  109. src_dirs = {}
  110. for package in self.packages or ():
  111. # Locate package source directory
  112. src_dirs[assert_relative(self.get_package_dir(package))] = package
  113. self.run_command('egg_info')
  114. check = _IncludePackageDataAbuse()
  115. ei_cmd = self.get_finalized_command('egg_info')
  116. for path in ei_cmd.filelist.files:
  117. d, f = os.path.split(assert_relative(path))
  118. prev = None
  119. oldf = f
  120. while d and d != prev and d not in src_dirs:
  121. prev = d
  122. d, df = os.path.split(d)
  123. f = os.path.join(df, f)
  124. if d in src_dirs:
  125. if f == oldf:
  126. if check.is_module(f):
  127. continue # it's a module, not data
  128. else:
  129. importable = check.importable_subpackage(src_dirs[d], f)
  130. if importable:
  131. check.warn(importable)
  132. mf.setdefault(src_dirs[d], []).append(path)
  133. def get_data_files(self):
  134. pass # Lazily compute data files in _get_data_files() function.
  135. def check_package(self, package, package_dir):
  136. """Check namespace packages' __init__ for declare_namespace"""
  137. try:
  138. return self.packages_checked[package]
  139. except KeyError:
  140. pass
  141. init_py = orig.build_py.check_package(self, package, package_dir)
  142. self.packages_checked[package] = init_py
  143. if not init_py or not self.distribution.namespace_packages:
  144. return init_py
  145. for pkg in self.distribution.namespace_packages:
  146. if pkg == package or pkg.startswith(package + '.'):
  147. break
  148. else:
  149. return init_py
  150. with io.open(init_py, 'rb') as f:
  151. contents = f.read()
  152. if b'declare_namespace' not in contents:
  153. raise distutils.errors.DistutilsError(
  154. "Namespace package problem: %s is a namespace package, but "
  155. "its\n__init__.py does not call declare_namespace()! Please "
  156. 'fix it.\n(See the setuptools manual under '
  157. '"Namespace Packages" for details.)\n"' % (package,)
  158. )
  159. return init_py
  160. def initialize_options(self):
  161. self.packages_checked = {}
  162. orig.build_py.initialize_options(self)
  163. def get_package_dir(self, package):
  164. res = orig.build_py.get_package_dir(self, package)
  165. if self.distribution.src_root is not None:
  166. return os.path.join(self.distribution.src_root, res)
  167. return res
  168. def exclude_data_files(self, package, src_dir, files):
  169. """Filter filenames for package's data files in 'src_dir'"""
  170. files = list(files)
  171. patterns = self._get_platform_patterns(
  172. self.exclude_package_data,
  173. package,
  174. src_dir,
  175. )
  176. match_groups = (fnmatch.filter(files, pattern) for pattern in patterns)
  177. # flatten the groups of matches into an iterable of matches
  178. matches = itertools.chain.from_iterable(match_groups)
  179. bad = set(matches)
  180. keepers = (fn for fn in files if fn not in bad)
  181. # ditch dupes
  182. return list(unique_everseen(keepers))
  183. @staticmethod
  184. def _get_platform_patterns(spec, package, src_dir):
  185. """
  186. yield platform-specific path patterns (suitable for glob
  187. or fn_match) from a glob-based spec (such as
  188. self.package_data or self.exclude_package_data)
  189. matching package in src_dir.
  190. """
  191. raw_patterns = itertools.chain(
  192. spec.get('', []),
  193. spec.get(package, []),
  194. )
  195. return (
  196. # Each pattern has to be converted to a platform-specific path
  197. os.path.join(src_dir, convert_path(pattern))
  198. for pattern in raw_patterns
  199. )
  200. def assert_relative(path):
  201. if not os.path.isabs(path):
  202. return path
  203. from distutils.errors import DistutilsSetupError
  204. msg = (
  205. textwrap.dedent(
  206. """
  207. Error: setup script specifies an absolute path:
  208. %s
  209. setup() arguments must *always* be /-separated paths relative to the
  210. setup.py directory, *never* absolute paths.
  211. """
  212. ).lstrip()
  213. % path
  214. )
  215. raise DistutilsSetupError(msg)
  216. class _IncludePackageDataAbuse:
  217. """Inform users that package or module is included as 'data file'"""
  218. MESSAGE = """\
  219. Installing {importable!r} as data is deprecated, please list it in `packages`.
  220. !!\n\n
  221. ############################
  222. # Package would be ignored #
  223. ############################
  224. Python recognizes {importable!r} as an importable package,
  225. but it is not listed in the `packages` configuration of setuptools.
  226. {importable!r} has been automatically added to the distribution only
  227. because it may contain data files, but this behavior is likely to change
  228. in future versions of setuptools (and therefore is considered deprecated).
  229. Please make sure that {importable!r} is included as a package by using
  230. the `packages` configuration field or the proper discovery methods
  231. (for example by using `find_namespace_packages(...)`/`find_namespace:`
  232. instead of `find_packages(...)`/`find:`).
  233. You can read more about "package discovery" and "data files" on setuptools
  234. documentation page.
  235. \n\n!!
  236. """
  237. def __init__(self):
  238. self._already_warned = set()
  239. def is_module(self, file):
  240. return file.endswith(".py") and file[:-len(".py")].isidentifier()
  241. def importable_subpackage(self, parent, file):
  242. pkg = Path(file).parent
  243. parts = list(itertools.takewhile(str.isidentifier, pkg.parts))
  244. if parts:
  245. return ".".join([parent, *parts])
  246. return None
  247. def warn(self, importable):
  248. if importable not in self._already_warned:
  249. msg = textwrap.dedent(self.MESSAGE).format(importable=importable)
  250. warnings.warn(msg, SetuptoolsDeprecationWarning, stacklevel=2)
  251. self._already_warned.add(importable)