editable_wheel.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908
  1. """
  2. Create a wheel that, when installed, will make the source package 'editable'
  3. (add it to the interpreter's path, including metadata) per PEP 660. Replaces
  4. 'setup.py develop'.
  5. .. note::
  6. One of the mechanisms briefly mentioned in PEP 660 to implement editable installs is
  7. to create a separated directory inside ``build`` and use a .pth file to point to that
  8. directory. In the context of this file such directory is referred as
  9. *auxiliary build directory* or ``auxiliary_dir``.
  10. """
  11. from __future__ import annotations
  12. import io
  13. import logging
  14. import os
  15. import shutil
  16. import traceback
  17. from collections.abc import Iterable, Iterator, Mapping
  18. from contextlib import suppress
  19. from enum import Enum
  20. from inspect import cleandoc
  21. from itertools import chain, starmap
  22. from pathlib import Path
  23. from tempfile import TemporaryDirectory
  24. from types import TracebackType
  25. from typing import TYPE_CHECKING, Protocol, TypeVar, cast
  26. from .. import Command, _normalization, _path, _shutil, errors, namespaces
  27. from .._path import StrPath
  28. from ..compat import py310, py312
  29. from ..discovery import find_package_path
  30. from ..dist import Distribution
  31. from ..warnings import InformationOnly, SetuptoolsDeprecationWarning
  32. from .build import build as build_cls
  33. from .build_py import build_py as build_py_cls
  34. from .dist_info import dist_info as dist_info_cls
  35. from .egg_info import egg_info as egg_info_cls
  36. from .install import install as install_cls
  37. from .install_scripts import install_scripts as install_scripts_cls
  38. if TYPE_CHECKING:
  39. from typing_extensions import Self
  40. from .._vendor.wheel.wheelfile import WheelFile
  41. _P = TypeVar("_P", bound=StrPath)
  42. _logger = logging.getLogger(__name__)
  43. class _EditableMode(Enum):
  44. """
  45. Possible editable installation modes:
  46. `lenient` (new files automatically added to the package - DEFAULT);
  47. `strict` (requires a new installation when files are added/removed); or
  48. `compat` (attempts to emulate `python setup.py develop` - DEPRECATED).
  49. """
  50. STRICT = "strict"
  51. LENIENT = "lenient"
  52. COMPAT = "compat" # TODO: Remove `compat` after Dec/2022.
  53. @classmethod
  54. def convert(cls, mode: str | None) -> _EditableMode:
  55. if not mode:
  56. return _EditableMode.LENIENT # default
  57. _mode = mode.upper()
  58. if _mode not in _EditableMode.__members__:
  59. raise errors.OptionError(f"Invalid editable mode: {mode!r}. Try: 'strict'.")
  60. if _mode == "COMPAT":
  61. SetuptoolsDeprecationWarning.emit(
  62. "Compat editable installs",
  63. """
  64. The 'compat' editable mode is transitional and will be removed
  65. in future versions of `setuptools`.
  66. Please adapt your code accordingly to use either the 'strict' or the
  67. 'lenient' modes.
  68. """,
  69. see_docs="userguide/development_mode.html",
  70. # TODO: define due_date
  71. # There is a series of shortcomings with the available editable install
  72. # methods, and they are very controversial. This is something that still
  73. # needs work.
  74. # Moreover, `pip` is still hiding this warning, so users are not aware.
  75. )
  76. return _EditableMode[_mode]
  77. _STRICT_WARNING = """
  78. New or renamed files may not be automatically picked up without a new installation.
  79. """
  80. _LENIENT_WARNING = """
  81. Options like `package-data`, `include/exclude-package-data` or
  82. `packages.find.exclude/include` may have no effect.
  83. """
  84. class editable_wheel(Command):
  85. """Build 'editable' wheel for development.
  86. This command is private and reserved for internal use of setuptools,
  87. users should rely on ``setuptools.build_meta`` APIs.
  88. """
  89. description = "DO NOT CALL DIRECTLY, INTERNAL ONLY: create PEP 660 editable wheel"
  90. user_options = [
  91. ("dist-dir=", "d", "directory to put final built distributions in"),
  92. ("dist-info-dir=", "I", "path to a pre-build .dist-info directory"),
  93. ("mode=", None, cleandoc(_EditableMode.__doc__ or "")),
  94. ]
  95. def initialize_options(self):
  96. self.dist_dir = None
  97. self.dist_info_dir = None
  98. self.project_dir = None
  99. self.mode = None
  100. def finalize_options(self) -> None:
  101. dist = self.distribution
  102. self.project_dir = dist.src_root or os.curdir
  103. self.package_dir = dist.package_dir or {}
  104. self.dist_dir = Path(self.dist_dir or os.path.join(self.project_dir, "dist"))
  105. def run(self) -> None:
  106. try:
  107. self.dist_dir.mkdir(exist_ok=True)
  108. self._ensure_dist_info()
  109. # Add missing dist_info files
  110. self.reinitialize_command("bdist_wheel")
  111. bdist_wheel = self.get_finalized_command("bdist_wheel")
  112. bdist_wheel.write_wheelfile(self.dist_info_dir)
  113. self._create_wheel_file(bdist_wheel)
  114. except Exception as ex:
  115. project = self.distribution.name or self.distribution.get_name()
  116. py310.add_note(
  117. ex,
  118. f"An error occurred when building editable wheel for {project}.\n"
  119. "See debugging tips in: "
  120. "https://setuptools.pypa.io/en/latest/userguide/development_mode.html#debugging-tips",
  121. )
  122. raise
  123. def _ensure_dist_info(self):
  124. if self.dist_info_dir is None:
  125. dist_info = cast(dist_info_cls, self.reinitialize_command("dist_info"))
  126. dist_info.output_dir = self.dist_dir
  127. dist_info.ensure_finalized()
  128. dist_info.run()
  129. self.dist_info_dir = dist_info.dist_info_dir
  130. else:
  131. assert str(self.dist_info_dir).endswith(".dist-info")
  132. assert Path(self.dist_info_dir, "METADATA").exists()
  133. def _install_namespaces(self, installation_dir, pth_prefix):
  134. # XXX: Only required to support the deprecated namespace practice
  135. dist = self.distribution
  136. if not dist.namespace_packages:
  137. return
  138. src_root = Path(self.project_dir, self.package_dir.get("", ".")).resolve()
  139. installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, src_root)
  140. installer.install_namespaces()
  141. def _find_egg_info_dir(self) -> str | None:
  142. parent_dir = Path(self.dist_info_dir).parent if self.dist_info_dir else Path()
  143. candidates = map(str, parent_dir.glob("*.egg-info"))
  144. return next(candidates, None)
  145. def _configure_build(
  146. self, name: str, unpacked_wheel: StrPath, build_lib: StrPath, tmp_dir: StrPath
  147. ):
  148. """Configure commands to behave in the following ways:
  149. - Build commands can write to ``build_lib`` if they really want to...
  150. (but this folder is expected to be ignored and modules are expected to live
  151. in the project directory...)
  152. - Binary extensions should be built in-place (editable_mode = True)
  153. - Data/header/script files are not part of the "editable" specification
  154. so they are written directly to the unpacked_wheel directory.
  155. """
  156. # Non-editable files (data, headers, scripts) are written directly to the
  157. # unpacked_wheel
  158. dist = self.distribution
  159. wheel = str(unpacked_wheel)
  160. build_lib = str(build_lib)
  161. data = str(Path(unpacked_wheel, f"{name}.data", "data"))
  162. headers = str(Path(unpacked_wheel, f"{name}.data", "headers"))
  163. scripts = str(Path(unpacked_wheel, f"{name}.data", "scripts"))
  164. # egg-info may be generated again to create a manifest (used for package data)
  165. egg_info = cast(
  166. egg_info_cls, dist.reinitialize_command("egg_info", reinit_subcommands=True)
  167. )
  168. egg_info.egg_base = str(tmp_dir)
  169. egg_info.ignore_egg_info_in_manifest = True
  170. build = cast(
  171. build_cls, dist.reinitialize_command("build", reinit_subcommands=True)
  172. )
  173. install = cast(
  174. install_cls, dist.reinitialize_command("install", reinit_subcommands=True)
  175. )
  176. build.build_platlib = build.build_purelib = build.build_lib = build_lib
  177. install.install_purelib = install.install_platlib = install.install_lib = wheel
  178. install.install_scripts = build.build_scripts = scripts
  179. install.install_headers = headers
  180. install.install_data = data
  181. # For portability, ensure scripts are built with #!python shebang
  182. # pypa/setuptools#4863
  183. build_scripts = dist.get_command_obj("build_scripts")
  184. build_scripts.executable = 'python'
  185. install_scripts = cast(
  186. install_scripts_cls, dist.get_command_obj("install_scripts")
  187. )
  188. install_scripts.no_ep = True
  189. build.build_temp = str(tmp_dir)
  190. build_py = cast(build_py_cls, dist.get_command_obj("build_py"))
  191. build_py.compile = False
  192. build_py.existing_egg_info_dir = self._find_egg_info_dir()
  193. self._set_editable_mode()
  194. build.ensure_finalized()
  195. install.ensure_finalized()
  196. def _set_editable_mode(self):
  197. """Set the ``editable_mode`` flag in the build sub-commands"""
  198. dist = self.distribution
  199. build = dist.get_command_obj("build")
  200. for cmd_name in build.get_sub_commands():
  201. cmd = dist.get_command_obj(cmd_name)
  202. if hasattr(cmd, "editable_mode"):
  203. cmd.editable_mode = True
  204. elif hasattr(cmd, "inplace"):
  205. cmd.inplace = True # backward compatibility with distutils
  206. def _collect_build_outputs(self) -> tuple[list[str], dict[str, str]]:
  207. files: list[str] = []
  208. mapping: dict[str, str] = {}
  209. build = self.get_finalized_command("build")
  210. for cmd_name in build.get_sub_commands():
  211. cmd = self.get_finalized_command(cmd_name)
  212. if hasattr(cmd, "get_outputs"):
  213. files.extend(cmd.get_outputs() or [])
  214. if hasattr(cmd, "get_output_mapping"):
  215. mapping.update(cmd.get_output_mapping() or {})
  216. return files, mapping
  217. def _run_build_commands(
  218. self,
  219. dist_name: str,
  220. unpacked_wheel: StrPath,
  221. build_lib: StrPath,
  222. tmp_dir: StrPath,
  223. ) -> tuple[list[str], dict[str, str]]:
  224. self._configure_build(dist_name, unpacked_wheel, build_lib, tmp_dir)
  225. self._run_build_subcommands()
  226. files, mapping = self._collect_build_outputs()
  227. self._run_install("headers")
  228. self._run_install("scripts")
  229. self._run_install("data")
  230. return files, mapping
  231. def _run_build_subcommands(self) -> None:
  232. """
  233. Issue #3501 indicates that some plugins/customizations might rely on:
  234. 1. ``build_py`` not running
  235. 2. ``build_py`` always copying files to ``build_lib``
  236. However both these assumptions may be false in editable_wheel.
  237. This method implements a temporary workaround to support the ecosystem
  238. while the implementations catch up.
  239. """
  240. # TODO: Once plugins/customizations had the chance to catch up, replace
  241. # `self._run_build_subcommands()` with `self.run_command("build")`.
  242. # Also remove _safely_run, TestCustomBuildPy. Suggested date: Aug/2023.
  243. build = self.get_finalized_command("build")
  244. for name in build.get_sub_commands():
  245. cmd = self.get_finalized_command(name)
  246. if name == "build_py" and type(cmd) is not build_py_cls:
  247. self._safely_run(name)
  248. else:
  249. self.run_command(name)
  250. def _safely_run(self, cmd_name: str):
  251. try:
  252. return self.run_command(cmd_name)
  253. except Exception:
  254. SetuptoolsDeprecationWarning.emit(
  255. "Customization incompatible with editable install",
  256. f"""
  257. {traceback.format_exc()}
  258. If you are seeing this warning it is very likely that a setuptools
  259. plugin or customization overrides the `{cmd_name}` command, without
  260. taking into consideration how editable installs run build steps
  261. starting from setuptools v64.0.0.
  262. Plugin authors and developers relying on custom build steps are
  263. encouraged to update their `{cmd_name}` implementation considering the
  264. information about editable installs in
  265. https://setuptools.pypa.io/en/latest/userguide/extension.html.
  266. For the time being `setuptools` will silence this error and ignore
  267. the faulty command, but this behavior will change in future versions.
  268. """,
  269. # TODO: define due_date
  270. # There is a series of shortcomings with the available editable install
  271. # methods, and they are very controversial. This is something that still
  272. # needs work.
  273. )
  274. def _create_wheel_file(self, bdist_wheel):
  275. from wheel.wheelfile import WheelFile
  276. dist_info = self.get_finalized_command("dist_info")
  277. dist_name = dist_info.name
  278. tag = "-".join(bdist_wheel.get_tag())
  279. build_tag = "0.editable" # According to PEP 427 needs to start with digit
  280. archive_name = f"{dist_name}-{build_tag}-{tag}.whl"
  281. wheel_path = Path(self.dist_dir, archive_name)
  282. if wheel_path.exists():
  283. wheel_path.unlink()
  284. unpacked_wheel = TemporaryDirectory(suffix=archive_name)
  285. build_lib = TemporaryDirectory(suffix=".build-lib")
  286. build_tmp = TemporaryDirectory(suffix=".build-temp")
  287. with unpacked_wheel as unpacked, build_lib as lib, build_tmp as tmp:
  288. unpacked_dist_info = Path(unpacked, Path(self.dist_info_dir).name)
  289. shutil.copytree(self.dist_info_dir, unpacked_dist_info)
  290. self._install_namespaces(unpacked, dist_name)
  291. files, mapping = self._run_build_commands(dist_name, unpacked, lib, tmp)
  292. strategy = self._select_strategy(dist_name, tag, lib)
  293. with strategy, WheelFile(wheel_path, "w") as wheel_obj:
  294. strategy(wheel_obj, files, mapping)
  295. wheel_obj.write_files(unpacked)
  296. return wheel_path
  297. def _run_install(self, category: str):
  298. has_category = getattr(self.distribution, f"has_{category}", None)
  299. if has_category and has_category():
  300. _logger.info(f"Installing {category} as non editable")
  301. self.run_command(f"install_{category}")
  302. def _select_strategy(
  303. self,
  304. name: str,
  305. tag: str,
  306. build_lib: StrPath,
  307. ) -> EditableStrategy:
  308. """Decides which strategy to use to implement an editable installation."""
  309. build_name = f"__editable__.{name}-{tag}"
  310. project_dir = Path(self.project_dir)
  311. mode = _EditableMode.convert(self.mode)
  312. if mode is _EditableMode.STRICT:
  313. auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name))
  314. return _LinkTree(self.distribution, name, auxiliary_dir, build_lib)
  315. packages = _find_packages(self.distribution)
  316. has_simple_layout = _simple_layout(packages, self.package_dir, project_dir)
  317. is_compat_mode = mode is _EditableMode.COMPAT
  318. if set(self.package_dir) == {""} and has_simple_layout or is_compat_mode:
  319. # src-layout(ish) is relatively safe for a simple pth file
  320. src_dir = self.package_dir.get("", ".")
  321. return _StaticPth(self.distribution, name, [Path(project_dir, src_dir)])
  322. # Use a MetaPathFinder to avoid adding accidental top-level packages/modules
  323. return _TopLevelFinder(self.distribution, name)
  324. class EditableStrategy(Protocol):
  325. def __call__(
  326. self, wheel: WheelFile, files: list[str], mapping: Mapping[str, str]
  327. ) -> object: ...
  328. def __enter__(self) -> Self: ...
  329. def __exit__(
  330. self,
  331. _exc_type: type[BaseException] | None,
  332. _exc_value: BaseException | None,
  333. _traceback: TracebackType | None,
  334. ) -> object: ...
  335. class _StaticPth:
  336. def __init__(self, dist: Distribution, name: str, path_entries: list[Path]) -> None:
  337. self.dist = dist
  338. self.name = name
  339. self.path_entries = path_entries
  340. def __call__(self, wheel: WheelFile, files: list[str], mapping: Mapping[str, str]):
  341. entries = "\n".join(str(p.resolve()) for p in self.path_entries)
  342. contents = _encode_pth(f"{entries}\n")
  343. wheel.writestr(f"__editable__.{self.name}.pth", contents)
  344. def __enter__(self) -> Self:
  345. msg = f"""
  346. Editable install will be performed using .pth file to extend `sys.path` with:
  347. {list(map(os.fspath, self.path_entries))!r}
  348. """
  349. _logger.warning(msg + _LENIENT_WARNING)
  350. return self
  351. def __exit__(
  352. self,
  353. _exc_type: object,
  354. _exc_value: object,
  355. _traceback: object,
  356. ) -> None:
  357. pass
  358. class _LinkTree(_StaticPth):
  359. """
  360. Creates a ``.pth`` file that points to a link tree in the ``auxiliary_dir``.
  361. This strategy will only link files (not dirs), so it can be implemented in
  362. any OS, even if that means using hardlinks instead of symlinks.
  363. By collocating ``auxiliary_dir`` and the original source code, limitations
  364. with hardlinks should be avoided.
  365. """
  366. def __init__(
  367. self,
  368. dist: Distribution,
  369. name: str,
  370. auxiliary_dir: StrPath,
  371. build_lib: StrPath,
  372. ) -> None:
  373. self.auxiliary_dir = Path(auxiliary_dir)
  374. self.build_lib = Path(build_lib).resolve()
  375. self._file = dist.get_command_obj("build_py").copy_file
  376. super().__init__(dist, name, [self.auxiliary_dir])
  377. def __call__(self, wheel: WheelFile, files: list[str], mapping: Mapping[str, str]):
  378. self._create_links(files, mapping)
  379. super().__call__(wheel, files, mapping)
  380. def _normalize_output(self, file: str) -> str | None:
  381. # Files relative to build_lib will be normalized to None
  382. with suppress(ValueError):
  383. path = Path(file).resolve().relative_to(self.build_lib)
  384. return str(path).replace(os.sep, '/')
  385. return None
  386. def _create_file(self, relative_output: str, src_file: str, link=None):
  387. dest = self.auxiliary_dir / relative_output
  388. if not dest.parent.is_dir():
  389. dest.parent.mkdir(parents=True)
  390. self._file(src_file, dest, link=link)
  391. def _create_links(self, outputs, output_mapping: Mapping[str, str]):
  392. self.auxiliary_dir.mkdir(parents=True, exist_ok=True)
  393. link_type = "sym" if _can_symlink_files(self.auxiliary_dir) else "hard"
  394. normalised = ((self._normalize_output(k), v) for k, v in output_mapping.items())
  395. # remove files that are not relative to build_lib
  396. mappings = {k: v for k, v in normalised if k is not None}
  397. for output in outputs:
  398. relative = self._normalize_output(output)
  399. if relative and relative not in mappings:
  400. self._create_file(relative, output)
  401. for relative, src in mappings.items():
  402. self._create_file(relative, src, link=link_type)
  403. def __enter__(self) -> Self:
  404. msg = "Strict editable install will be performed using a link tree.\n"
  405. _logger.warning(msg + _STRICT_WARNING)
  406. return self
  407. def __exit__(
  408. self,
  409. _exc_type: object,
  410. _exc_value: object,
  411. _traceback: object,
  412. ) -> None:
  413. msg = f"""\n
  414. Strict editable installation performed using the auxiliary directory:
  415. {self.auxiliary_dir}
  416. Please be careful to not remove this directory, otherwise you might not be able
  417. to import/use your package.
  418. """
  419. InformationOnly.emit("Editable installation.", msg)
  420. class _TopLevelFinder:
  421. def __init__(self, dist: Distribution, name: str) -> None:
  422. self.dist = dist
  423. self.name = name
  424. def template_vars(self) -> tuple[str, str, dict[str, str], dict[str, list[str]]]:
  425. src_root = self.dist.src_root or os.curdir
  426. top_level = chain(_find_packages(self.dist), _find_top_level_modules(self.dist))
  427. package_dir = self.dist.package_dir or {}
  428. roots = _find_package_roots(top_level, package_dir, src_root)
  429. namespaces_ = dict(
  430. chain(
  431. _find_namespaces(self.dist.packages or [], roots),
  432. ((ns, []) for ns in _find_virtual_namespaces(roots)),
  433. )
  434. )
  435. legacy_namespaces = {
  436. pkg: find_package_path(pkg, roots, self.dist.src_root or "")
  437. for pkg in self.dist.namespace_packages or []
  438. }
  439. mapping = {**roots, **legacy_namespaces}
  440. # ^-- We need to explicitly add the legacy_namespaces to the mapping to be
  441. # able to import their modules even if another package sharing the same
  442. # namespace is installed in a conventional (non-editable) way.
  443. name = f"__editable__.{self.name}.finder"
  444. finder = _normalization.safe_identifier(name)
  445. return finder, name, mapping, namespaces_
  446. def get_implementation(self) -> Iterator[tuple[str, bytes]]:
  447. finder, name, mapping, namespaces_ = self.template_vars()
  448. content = bytes(_finder_template(name, mapping, namespaces_), "utf-8")
  449. yield (f"{finder}.py", content)
  450. content = _encode_pth(f"import {finder}; {finder}.install()")
  451. yield (f"__editable__.{self.name}.pth", content)
  452. def __call__(self, wheel: WheelFile, files: list[str], mapping: Mapping[str, str]):
  453. for file, content in self.get_implementation():
  454. wheel.writestr(file, content)
  455. def __enter__(self) -> Self:
  456. msg = "Editable install will be performed using a meta path finder.\n"
  457. _logger.warning(msg + _LENIENT_WARNING)
  458. return self
  459. def __exit__(
  460. self,
  461. _exc_type: object,
  462. _exc_value: object,
  463. _traceback: object,
  464. ) -> None:
  465. msg = """\n
  466. Please be careful with folders in your working directory with the same
  467. name as your package as they may take precedence during imports.
  468. """
  469. InformationOnly.emit("Editable installation.", msg)
  470. def _encode_pth(content: str) -> bytes:
  471. """
  472. Prior to Python 3.13 (see https://github.com/python/cpython/issues/77102),
  473. .pth files are always read with 'locale' encoding, the recommendation
  474. from the cpython core developers is to write them as ``open(path, "w")``
  475. and ignore warnings (see python/cpython#77102, pypa/setuptools#3937).
  476. This function tries to simulate this behavior without having to create an
  477. actual file, in a way that supports a range of active Python versions.
  478. (There seems to be some variety in the way different version of Python handle
  479. ``encoding=None``, not all of them use ``locale.getpreferredencoding(False)``
  480. or ``locale.getencoding()``).
  481. """
  482. with io.BytesIO() as buffer:
  483. wrapper = io.TextIOWrapper(buffer, encoding=py312.PTH_ENCODING)
  484. # TODO: Python 3.13 replace the whole function with `bytes(content, "utf-8")`
  485. wrapper.write(content)
  486. wrapper.flush()
  487. buffer.seek(0)
  488. return buffer.read()
  489. def _can_symlink_files(base_dir: Path) -> bool:
  490. with TemporaryDirectory(dir=str(base_dir.resolve())) as tmp:
  491. path1, path2 = Path(tmp, "file1.txt"), Path(tmp, "file2.txt")
  492. path1.write_text("file1", encoding="utf-8")
  493. with suppress(AttributeError, NotImplementedError, OSError):
  494. os.symlink(path1, path2)
  495. if path2.is_symlink() and path2.read_text(encoding="utf-8") == "file1":
  496. return True
  497. try:
  498. os.link(path1, path2) # Ensure hard links can be created
  499. except Exception as ex:
  500. msg = (
  501. "File system does not seem to support either symlinks or hard links. "
  502. "Strict editable installs require one of them to be supported."
  503. )
  504. raise LinksNotSupported(msg) from ex
  505. return False
  506. def _simple_layout(
  507. packages: Iterable[str], package_dir: dict[str, str], project_dir: StrPath
  508. ) -> bool:
  509. """Return ``True`` if:
  510. - all packages are contained by the same parent directory, **and**
  511. - all packages become importable if the parent directory is added to ``sys.path``.
  512. >>> _simple_layout(['a'], {"": "src"}, "/tmp/myproj")
  513. True
  514. >>> _simple_layout(['a', 'a.b'], {"": "src"}, "/tmp/myproj")
  515. True
  516. >>> _simple_layout(['a', 'a.b'], {}, "/tmp/myproj")
  517. True
  518. >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"": "src"}, "/tmp/myproj")
  519. True
  520. >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "a", "b": "b"}, ".")
  521. True
  522. >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "_a", "b": "_b"}, ".")
  523. False
  524. >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "_a"}, "/tmp/myproj")
  525. False
  526. >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a.a1.a2": "_a2"}, ".")
  527. False
  528. >>> _simple_layout(['a', 'a.b'], {"": "src", "a.b": "_ab"}, "/tmp/myproj")
  529. False
  530. >>> # Special cases, no packages yet:
  531. >>> _simple_layout([], {"": "src"}, "/tmp/myproj")
  532. True
  533. >>> _simple_layout([], {"a": "_a", "": "src"}, "/tmp/myproj")
  534. False
  535. """
  536. layout = {pkg: find_package_path(pkg, package_dir, project_dir) for pkg in packages}
  537. if not layout:
  538. return set(package_dir) in ({}, {""})
  539. parent = os.path.commonpath(starmap(_parent_path, layout.items()))
  540. return all(
  541. _path.same_path(Path(parent, *key.split('.')), value)
  542. for key, value in layout.items()
  543. )
  544. def _parent_path(pkg, pkg_path):
  545. """Infer the parent path containing a package, that if added to ``sys.path`` would
  546. allow importing that package.
  547. When ``pkg`` is directly mapped into a directory with a different name, return its
  548. own path.
  549. >>> _parent_path("a", "src/a")
  550. 'src'
  551. >>> _parent_path("b", "src/c")
  552. 'src/c'
  553. """
  554. parent = pkg_path[: -len(pkg)] if pkg_path.endswith(pkg) else pkg_path
  555. return parent.rstrip("/" + os.sep)
  556. def _find_packages(dist: Distribution) -> Iterator[str]:
  557. yield from iter(dist.packages or [])
  558. py_modules = dist.py_modules or []
  559. nested_modules = [mod for mod in py_modules if "." in mod]
  560. if dist.ext_package:
  561. yield dist.ext_package
  562. else:
  563. ext_modules = dist.ext_modules or []
  564. nested_modules += [x.name for x in ext_modules if "." in x.name]
  565. for module in nested_modules:
  566. package, _, _ = module.rpartition(".")
  567. yield package
  568. def _find_top_level_modules(dist: Distribution) -> Iterator[str]:
  569. py_modules = dist.py_modules or []
  570. yield from (mod for mod in py_modules if "." not in mod)
  571. if not dist.ext_package:
  572. ext_modules = dist.ext_modules or []
  573. yield from (x.name for x in ext_modules if "." not in x.name)
  574. def _find_package_roots(
  575. packages: Iterable[str],
  576. package_dir: Mapping[str, str],
  577. src_root: StrPath,
  578. ) -> dict[str, str]:
  579. pkg_roots: dict[str, str] = {
  580. pkg: _absolute_root(find_package_path(pkg, package_dir, src_root))
  581. for pkg in sorted(packages)
  582. }
  583. return _remove_nested(pkg_roots)
  584. def _absolute_root(path: StrPath) -> str:
  585. """Works for packages and top-level modules"""
  586. path_ = Path(path)
  587. parent = path_.parent
  588. if path_.exists():
  589. return str(path_.resolve())
  590. else:
  591. return str(parent.resolve() / path_.name)
  592. def _find_virtual_namespaces(pkg_roots: dict[str, str]) -> Iterator[str]:
  593. """By carefully designing ``package_dir``, it is possible to implement the logical
  594. structure of PEP 420 in a package without the corresponding directories.
  595. Moreover a parent package can be purposefully/accidentally skipped in the discovery
  596. phase (e.g. ``find_packages(include=["mypkg.*"])``, when ``mypkg.foo`` is included
  597. by ``mypkg`` itself is not).
  598. We consider this case to also be a virtual namespace (ignoring the original
  599. directory) to emulate a non-editable installation.
  600. This function will try to find these kinds of namespaces.
  601. """
  602. for pkg in pkg_roots:
  603. if "." not in pkg:
  604. continue
  605. parts = pkg.split(".")
  606. for i in range(len(parts) - 1, 0, -1):
  607. partial_name = ".".join(parts[:i])
  608. path = Path(find_package_path(partial_name, pkg_roots, ""))
  609. if not path.exists() or partial_name not in pkg_roots:
  610. # partial_name not in pkg_roots ==> purposefully/accidentally skipped
  611. yield partial_name
  612. def _find_namespaces(
  613. packages: list[str], pkg_roots: dict[str, str]
  614. ) -> Iterator[tuple[str, list[str]]]:
  615. for pkg in packages:
  616. path = find_package_path(pkg, pkg_roots, "")
  617. if Path(path).exists() and not Path(path, "__init__.py").exists():
  618. yield (pkg, [path])
  619. def _remove_nested(pkg_roots: dict[str, str]) -> dict[str, str]:
  620. output = dict(pkg_roots.copy())
  621. for pkg, path in reversed(list(pkg_roots.items())):
  622. if any(
  623. pkg != other and _is_nested(pkg, path, other, other_path)
  624. for other, other_path in pkg_roots.items()
  625. ):
  626. output.pop(pkg)
  627. return output
  628. def _is_nested(pkg: str, pkg_path: str, parent: str, parent_path: str) -> bool:
  629. """
  630. Return ``True`` if ``pkg`` is nested inside ``parent`` both logically and in the
  631. file system.
  632. >>> _is_nested("a.b", "path/a/b", "a", "path/a")
  633. True
  634. >>> _is_nested("a.b", "path/a/b", "a", "otherpath/a")
  635. False
  636. >>> _is_nested("a.b", "path/a/b", "c", "path/c")
  637. False
  638. >>> _is_nested("a.a", "path/a/a", "a", "path/a")
  639. True
  640. >>> _is_nested("b.a", "path/b/a", "a", "path/a")
  641. False
  642. """
  643. norm_pkg_path = _path.normpath(pkg_path)
  644. rest = pkg.replace(parent, "", 1).strip(".").split(".")
  645. return pkg.startswith(parent) and norm_pkg_path == _path.normpath(
  646. Path(parent_path, *rest)
  647. )
  648. def _empty_dir(dir_: _P) -> _P:
  649. """Create a directory ensured to be empty. Existing files may be removed."""
  650. _shutil.rmtree(dir_, ignore_errors=True)
  651. os.makedirs(dir_)
  652. return dir_
  653. class _NamespaceInstaller(namespaces.Installer):
  654. def __init__(self, distribution, installation_dir, editable_name, src_root) -> None:
  655. self.distribution = distribution
  656. self.src_root = src_root
  657. self.installation_dir = installation_dir
  658. self.editable_name = editable_name
  659. self.outputs: list[str] = []
  660. self.dry_run = False
  661. def _get_nspkg_file(self):
  662. """Installation target."""
  663. return os.path.join(self.installation_dir, self.editable_name + self.nspkg_ext)
  664. def _get_root(self):
  665. """Where the modules/packages should be loaded from."""
  666. return repr(str(self.src_root))
  667. _FINDER_TEMPLATE = """\
  668. from __future__ import annotations
  669. import sys
  670. from importlib.machinery import ModuleSpec, PathFinder
  671. from importlib.machinery import all_suffixes as module_suffixes
  672. from importlib.util import spec_from_file_location
  673. from itertools import chain
  674. from pathlib import Path
  675. MAPPING: dict[str, str] = {mapping!r}
  676. NAMESPACES: dict[str, list[str]] = {namespaces!r}
  677. PATH_PLACEHOLDER = {name!r} + ".__path_hook__"
  678. class _EditableFinder: # MetaPathFinder
  679. @classmethod
  680. def find_spec(cls, fullname: str, path=None, target=None) -> ModuleSpec | None: # type: ignore
  681. # Top-level packages and modules (we know these exist in the FS)
  682. if fullname in MAPPING:
  683. pkg_path = MAPPING[fullname]
  684. return cls._find_spec(fullname, Path(pkg_path))
  685. # Handle immediate children modules (required for namespaces to work)
  686. # To avoid problems with case sensitivity in the file system we delegate
  687. # to the importlib.machinery implementation.
  688. parent, _, child = fullname.rpartition(".")
  689. if parent and parent in MAPPING:
  690. return PathFinder.find_spec(fullname, path=[MAPPING[parent]])
  691. # Other levels of nesting should be handled automatically by importlib
  692. # using the parent path.
  693. return None
  694. @classmethod
  695. def _find_spec(cls, fullname: str, candidate_path: Path) -> ModuleSpec | None:
  696. init = candidate_path / "__init__.py"
  697. candidates = (candidate_path.with_suffix(x) for x in module_suffixes())
  698. for candidate in chain([init], candidates):
  699. if candidate.exists():
  700. return spec_from_file_location(fullname, candidate)
  701. return None
  702. class _EditableNamespaceFinder: # PathEntryFinder
  703. @classmethod
  704. def _path_hook(cls, path) -> type[_EditableNamespaceFinder]:
  705. if path == PATH_PLACEHOLDER:
  706. return cls
  707. raise ImportError
  708. @classmethod
  709. def _paths(cls, fullname: str) -> list[str]:
  710. paths = NAMESPACES[fullname]
  711. if not paths and fullname in MAPPING:
  712. paths = [MAPPING[fullname]]
  713. # Always add placeholder, for 2 reasons:
  714. # 1. __path__ cannot be empty for the spec to be considered namespace.
  715. # 2. In the case of nested namespaces, we need to force
  716. # import machinery to query _EditableNamespaceFinder again.
  717. return [*paths, PATH_PLACEHOLDER]
  718. @classmethod
  719. def find_spec(cls, fullname: str, target=None) -> ModuleSpec | None: # type: ignore
  720. if fullname in NAMESPACES:
  721. spec = ModuleSpec(fullname, None, is_package=True)
  722. spec.submodule_search_locations = cls._paths(fullname)
  723. return spec
  724. return None
  725. @classmethod
  726. def find_module(cls, _fullname) -> None:
  727. return None
  728. def install():
  729. if not any(finder == _EditableFinder for finder in sys.meta_path):
  730. sys.meta_path.append(_EditableFinder)
  731. if not NAMESPACES:
  732. return
  733. if not any(hook == _EditableNamespaceFinder._path_hook for hook in sys.path_hooks):
  734. # PathEntryFinder is needed to create NamespaceSpec without private APIS
  735. sys.path_hooks.append(_EditableNamespaceFinder._path_hook)
  736. if PATH_PLACEHOLDER not in sys.path:
  737. sys.path.append(PATH_PLACEHOLDER) # Used just to trigger the path hook
  738. """
  739. def _finder_template(
  740. name: str, mapping: Mapping[str, str], namespaces: dict[str, list[str]]
  741. ) -> str:
  742. """Create a string containing the code for the``MetaPathFinder`` and
  743. ``PathEntryFinder``.
  744. """
  745. mapping = dict(sorted(mapping.items(), key=lambda p: p[0]))
  746. return _FINDER_TEMPLATE.format(name=name, mapping=mapping, namespaces=namespaces)
  747. class LinksNotSupported(errors.FileError):
  748. """File system does not seem to support either symlinks or hard links."""