build_env.py 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290
  1. """Build Environment used for isolation during sdist building
  2. """
  3. import logging
  4. import os
  5. import pathlib
  6. import sys
  7. import textwrap
  8. from collections import OrderedDict
  9. from sysconfig import get_paths
  10. from types import TracebackType
  11. from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Tuple, Type
  12. from pip._vendor.certifi import where
  13. from pip._vendor.packaging.requirements import Requirement
  14. from pip._vendor.packaging.version import Version
  15. from pip import __file__ as pip_location
  16. from pip._internal.cli.spinners import open_spinner
  17. from pip._internal.locations import get_platlib, get_prefixed_libs, get_purelib
  18. from pip._internal.metadata import get_default_environment, get_environment
  19. from pip._internal.utils.subprocess import call_subprocess
  20. from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
  21. if TYPE_CHECKING:
  22. from pip._internal.index.package_finder import PackageFinder
  23. logger = logging.getLogger(__name__)
  24. class _Prefix:
  25. def __init__(self, path: str) -> None:
  26. self.path = path
  27. self.setup = False
  28. self.bin_dir = get_paths(
  29. "nt" if os.name == "nt" else "posix_prefix",
  30. vars={"base": path, "platbase": path},
  31. )["scripts"]
  32. self.lib_dirs = get_prefixed_libs(path)
  33. def _get_runnable_pip() -> str:
  34. """Get a file to pass to a Python executable, to run the currently-running pip.
  35. This is used to run a pip subprocess, for installing requirements into the build
  36. environment.
  37. """
  38. source = pathlib.Path(pip_location).resolve().parent
  39. if not source.is_dir():
  40. # This would happen if someone is using pip from inside a zip file. In that
  41. # case, we can use that directly.
  42. return str(source)
  43. return os.fsdecode(source / "__pip-runner__.py")
  44. class BuildEnvironment:
  45. """Creates and manages an isolated environment to install build deps"""
  46. def __init__(self) -> None:
  47. temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True)
  48. self._prefixes = OrderedDict(
  49. (name, _Prefix(os.path.join(temp_dir.path, name)))
  50. for name in ("normal", "overlay")
  51. )
  52. self._bin_dirs: List[str] = []
  53. self._lib_dirs: List[str] = []
  54. for prefix in reversed(list(self._prefixes.values())):
  55. self._bin_dirs.append(prefix.bin_dir)
  56. self._lib_dirs.extend(prefix.lib_dirs)
  57. # Customize site to:
  58. # - ensure .pth files are honored
  59. # - prevent access to system site packages
  60. system_sites = {
  61. os.path.normcase(site) for site in (get_purelib(), get_platlib())
  62. }
  63. self._site_dir = os.path.join(temp_dir.path, "site")
  64. if not os.path.exists(self._site_dir):
  65. os.mkdir(self._site_dir)
  66. with open(
  67. os.path.join(self._site_dir, "sitecustomize.py"), "w", encoding="utf-8"
  68. ) as fp:
  69. fp.write(
  70. textwrap.dedent(
  71. """
  72. import os, site, sys
  73. # First, drop system-sites related paths.
  74. original_sys_path = sys.path[:]
  75. known_paths = set()
  76. for path in {system_sites!r}:
  77. site.addsitedir(path, known_paths=known_paths)
  78. system_paths = set(
  79. os.path.normcase(path)
  80. for path in sys.path[len(original_sys_path):]
  81. )
  82. original_sys_path = [
  83. path for path in original_sys_path
  84. if os.path.normcase(path) not in system_paths
  85. ]
  86. sys.path = original_sys_path
  87. # Second, add lib directories.
  88. # ensuring .pth file are processed.
  89. for path in {lib_dirs!r}:
  90. assert not path in sys.path
  91. site.addsitedir(path)
  92. """
  93. ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)
  94. )
  95. def __enter__(self) -> None:
  96. self._save_env = {
  97. name: os.environ.get(name, None)
  98. for name in ("PATH", "PYTHONNOUSERSITE", "PYTHONPATH")
  99. }
  100. path = self._bin_dirs[:]
  101. old_path = self._save_env["PATH"]
  102. if old_path:
  103. path.extend(old_path.split(os.pathsep))
  104. pythonpath = [self._site_dir]
  105. os.environ.update(
  106. {
  107. "PATH": os.pathsep.join(path),
  108. "PYTHONNOUSERSITE": "1",
  109. "PYTHONPATH": os.pathsep.join(pythonpath),
  110. }
  111. )
  112. def __exit__(
  113. self,
  114. exc_type: Optional[Type[BaseException]],
  115. exc_val: Optional[BaseException],
  116. exc_tb: Optional[TracebackType],
  117. ) -> None:
  118. for varname, old_value in self._save_env.items():
  119. if old_value is None:
  120. os.environ.pop(varname, None)
  121. else:
  122. os.environ[varname] = old_value
  123. def check_requirements(
  124. self, reqs: Iterable[str]
  125. ) -> Tuple[Set[Tuple[str, str]], Set[str]]:
  126. """Return 2 sets:
  127. - conflicting requirements: set of (installed, wanted) reqs tuples
  128. - missing requirements: set of reqs
  129. """
  130. missing = set()
  131. conflicting = set()
  132. if reqs:
  133. env = (
  134. get_environment(self._lib_dirs)
  135. if hasattr(self, "_lib_dirs")
  136. else get_default_environment()
  137. )
  138. for req_str in reqs:
  139. req = Requirement(req_str)
  140. # We're explicitly evaluating with an empty extra value, since build
  141. # environments are not provided any mechanism to select specific extras.
  142. if req.marker is not None and not req.marker.evaluate({"extra": ""}):
  143. continue
  144. dist = env.get_distribution(req.name)
  145. if not dist:
  146. missing.add(req_str)
  147. continue
  148. if isinstance(dist.version, Version):
  149. installed_req_str = f"{req.name}=={dist.version}"
  150. else:
  151. installed_req_str = f"{req.name}==={dist.version}"
  152. if not req.specifier.contains(dist.version, prereleases=True):
  153. conflicting.add((installed_req_str, req_str))
  154. # FIXME: Consider direct URL?
  155. return conflicting, missing
  156. def install_requirements(
  157. self,
  158. finder: "PackageFinder",
  159. requirements: Iterable[str],
  160. prefix_as_string: str,
  161. *,
  162. kind: str,
  163. ) -> None:
  164. prefix = self._prefixes[prefix_as_string]
  165. assert not prefix.setup
  166. prefix.setup = True
  167. if not requirements:
  168. return
  169. self._install_requirements(
  170. _get_runnable_pip(),
  171. finder,
  172. requirements,
  173. prefix,
  174. kind=kind,
  175. )
  176. @staticmethod
  177. def _install_requirements(
  178. pip_runnable: str,
  179. finder: "PackageFinder",
  180. requirements: Iterable[str],
  181. prefix: _Prefix,
  182. *,
  183. kind: str,
  184. ) -> None:
  185. args: List[str] = [
  186. sys.executable,
  187. pip_runnable,
  188. "install",
  189. "--ignore-installed",
  190. "--no-user",
  191. "--prefix",
  192. prefix.path,
  193. "--no-warn-script-location",
  194. ]
  195. if logger.getEffectiveLevel() <= logging.DEBUG:
  196. args.append("-v")
  197. for format_control in ("no_binary", "only_binary"):
  198. formats = getattr(finder.format_control, format_control)
  199. args.extend(
  200. (
  201. "--" + format_control.replace("_", "-"),
  202. ",".join(sorted(formats or {":none:"})),
  203. )
  204. )
  205. index_urls = finder.index_urls
  206. if index_urls:
  207. args.extend(["-i", index_urls[0]])
  208. for extra_index in index_urls[1:]:
  209. args.extend(["--extra-index-url", extra_index])
  210. else:
  211. args.append("--no-index")
  212. for link in finder.find_links:
  213. args.extend(["--find-links", link])
  214. for host in finder.trusted_hosts:
  215. args.extend(["--trusted-host", host])
  216. if finder.allow_all_prereleases:
  217. args.append("--pre")
  218. if finder.prefer_binary:
  219. args.append("--prefer-binary")
  220. args.append("--")
  221. args.extend(requirements)
  222. extra_environ = {"_PIP_STANDALONE_CERT": where()}
  223. with open_spinner(f"Installing {kind}") as spinner:
  224. call_subprocess(
  225. args,
  226. command_desc=f"pip subprocess to install {kind}",
  227. spinner=spinner,
  228. extra_environ=extra_environ,
  229. )
  230. class NoOpBuildEnvironment(BuildEnvironment):
  231. """A no-op drop-in replacement for BuildEnvironment"""
  232. def __init__(self) -> None:
  233. pass
  234. def __enter__(self) -> None:
  235. pass
  236. def __exit__(
  237. self,
  238. exc_type: Optional[Type[BaseException]],
  239. exc_val: Optional[BaseException],
  240. exc_tb: Optional[TracebackType],
  241. ) -> None:
  242. pass
  243. def cleanup(self) -> None:
  244. pass
  245. def install_requirements(
  246. self,
  247. finder: "PackageFinder",
  248. requirements: Iterable[str],
  249. prefix_as_string: str,
  250. *,
  251. kind: str,
  252. ) -> None:
  253. raise NotImplementedError()