freeze.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256
  1. import collections
  2. import logging
  3. import os
  4. from dataclasses import dataclass, field
  5. from typing import Container, Dict, Generator, Iterable, List, NamedTuple, Optional, Set
  6. from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
  7. from pip._vendor.packaging.version import InvalidVersion
  8. from pip._internal.exceptions import BadCommand, InstallationError
  9. from pip._internal.metadata import BaseDistribution, get_environment
  10. from pip._internal.req.constructors import (
  11. install_req_from_editable,
  12. install_req_from_line,
  13. )
  14. from pip._internal.req.req_file import COMMENT_RE
  15. from pip._internal.utils.direct_url_helpers import direct_url_as_pep440_direct_reference
  16. logger = logging.getLogger(__name__)
  17. class _EditableInfo(NamedTuple):
  18. requirement: str
  19. comments: List[str]
  20. def freeze(
  21. requirement: Optional[List[str]] = None,
  22. local_only: bool = False,
  23. user_only: bool = False,
  24. paths: Optional[List[str]] = None,
  25. isolated: bool = False,
  26. exclude_editable: bool = False,
  27. skip: Container[str] = (),
  28. ) -> Generator[str, None, None]:
  29. installations: Dict[str, FrozenRequirement] = {}
  30. dists = get_environment(paths).iter_installed_distributions(
  31. local_only=local_only,
  32. skip=(),
  33. user_only=user_only,
  34. )
  35. for dist in dists:
  36. req = FrozenRequirement.from_dist(dist)
  37. if exclude_editable and req.editable:
  38. continue
  39. installations[req.canonical_name] = req
  40. if requirement:
  41. # the options that don't get turned into an InstallRequirement
  42. # should only be emitted once, even if the same option is in multiple
  43. # requirements files, so we need to keep track of what has been emitted
  44. # so that we don't emit it again if it's seen again
  45. emitted_options: Set[str] = set()
  46. # keep track of which files a requirement is in so that we can
  47. # give an accurate warning if a requirement appears multiple times.
  48. req_files: Dict[str, List[str]] = collections.defaultdict(list)
  49. for req_file_path in requirement:
  50. with open(req_file_path) as req_file:
  51. for line in req_file:
  52. if (
  53. not line.strip()
  54. or line.strip().startswith("#")
  55. or line.startswith(
  56. (
  57. "-r",
  58. "--requirement",
  59. "-f",
  60. "--find-links",
  61. "-i",
  62. "--index-url",
  63. "--pre",
  64. "--trusted-host",
  65. "--process-dependency-links",
  66. "--extra-index-url",
  67. "--use-feature",
  68. )
  69. )
  70. ):
  71. line = line.rstrip()
  72. if line not in emitted_options:
  73. emitted_options.add(line)
  74. yield line
  75. continue
  76. if line.startswith("-e") or line.startswith("--editable"):
  77. if line.startswith("-e"):
  78. line = line[2:].strip()
  79. else:
  80. line = line[len("--editable") :].strip().lstrip("=")
  81. line_req = install_req_from_editable(
  82. line,
  83. isolated=isolated,
  84. )
  85. else:
  86. line_req = install_req_from_line(
  87. COMMENT_RE.sub("", line).strip(),
  88. isolated=isolated,
  89. )
  90. if not line_req.name:
  91. logger.info(
  92. "Skipping line in requirement file [%s] because "
  93. "it's not clear what it would install: %s",
  94. req_file_path,
  95. line.strip(),
  96. )
  97. logger.info(
  98. " (add #egg=PackageName to the URL to avoid"
  99. " this warning)"
  100. )
  101. else:
  102. line_req_canonical_name = canonicalize_name(line_req.name)
  103. if line_req_canonical_name not in installations:
  104. # either it's not installed, or it is installed
  105. # but has been processed already
  106. if not req_files[line_req.name]:
  107. logger.warning(
  108. "Requirement file [%s] contains %s, but "
  109. "package %r is not installed",
  110. req_file_path,
  111. COMMENT_RE.sub("", line).strip(),
  112. line_req.name,
  113. )
  114. else:
  115. req_files[line_req.name].append(req_file_path)
  116. else:
  117. yield str(installations[line_req_canonical_name]).rstrip()
  118. del installations[line_req_canonical_name]
  119. req_files[line_req.name].append(req_file_path)
  120. # Warn about requirements that were included multiple times (in a
  121. # single requirements file or in different requirements files).
  122. for name, files in req_files.items():
  123. if len(files) > 1:
  124. logger.warning(
  125. "Requirement %s included multiple times [%s]",
  126. name,
  127. ", ".join(sorted(set(files))),
  128. )
  129. yield ("## The following requirements were added by pip freeze:")
  130. for installation in sorted(installations.values(), key=lambda x: x.name.lower()):
  131. if installation.canonical_name not in skip:
  132. yield str(installation).rstrip()
  133. def _format_as_name_version(dist: BaseDistribution) -> str:
  134. try:
  135. dist_version = dist.version
  136. except InvalidVersion:
  137. # legacy version
  138. return f"{dist.raw_name}==={dist.raw_version}"
  139. else:
  140. return f"{dist.raw_name}=={dist_version}"
  141. def _get_editable_info(dist: BaseDistribution) -> _EditableInfo:
  142. """
  143. Compute and return values (req, comments) for use in
  144. FrozenRequirement.from_dist().
  145. """
  146. editable_project_location = dist.editable_project_location
  147. assert editable_project_location
  148. location = os.path.normcase(os.path.abspath(editable_project_location))
  149. from pip._internal.vcs import RemoteNotFoundError, RemoteNotValidError, vcs
  150. vcs_backend = vcs.get_backend_for_dir(location)
  151. if vcs_backend is None:
  152. display = _format_as_name_version(dist)
  153. logger.debug(
  154. 'No VCS found for editable requirement "%s" in: %r',
  155. display,
  156. location,
  157. )
  158. return _EditableInfo(
  159. requirement=location,
  160. comments=[f"# Editable install with no version control ({display})"],
  161. )
  162. vcs_name = type(vcs_backend).__name__
  163. try:
  164. req = vcs_backend.get_src_requirement(location, dist.raw_name)
  165. except RemoteNotFoundError:
  166. display = _format_as_name_version(dist)
  167. return _EditableInfo(
  168. requirement=location,
  169. comments=[f"# Editable {vcs_name} install with no remote ({display})"],
  170. )
  171. except RemoteNotValidError as ex:
  172. display = _format_as_name_version(dist)
  173. return _EditableInfo(
  174. requirement=location,
  175. comments=[
  176. f"# Editable {vcs_name} install ({display}) with either a deleted "
  177. f"local remote or invalid URI:",
  178. f"# '{ex.url}'",
  179. ],
  180. )
  181. except BadCommand:
  182. logger.warning(
  183. "cannot determine version of editable source in %s "
  184. "(%s command not found in path)",
  185. location,
  186. vcs_backend.name,
  187. )
  188. return _EditableInfo(requirement=location, comments=[])
  189. except InstallationError as exc:
  190. logger.warning("Error when trying to get requirement for VCS system %s", exc)
  191. else:
  192. return _EditableInfo(requirement=req, comments=[])
  193. logger.warning("Could not determine repository location of %s", location)
  194. return _EditableInfo(
  195. requirement=location,
  196. comments=["## !! Could not determine repository location"],
  197. )
  198. @dataclass(frozen=True)
  199. class FrozenRequirement:
  200. name: str
  201. req: str
  202. editable: bool
  203. comments: Iterable[str] = field(default_factory=tuple)
  204. @property
  205. def canonical_name(self) -> NormalizedName:
  206. return canonicalize_name(self.name)
  207. @classmethod
  208. def from_dist(cls, dist: BaseDistribution) -> "FrozenRequirement":
  209. editable = dist.editable
  210. if editable:
  211. req, comments = _get_editable_info(dist)
  212. else:
  213. comments = []
  214. direct_url = dist.direct_url
  215. if direct_url:
  216. # if PEP 610 metadata is present, use it
  217. req = direct_url_as_pep440_direct_reference(direct_url, dist.raw_name)
  218. else:
  219. # name==version requirement
  220. req = _format_as_name_version(dist)
  221. return cls(dist.raw_name, req, editable, comments=comments)
  222. def __str__(self) -> str:
  223. req = self.req
  224. if self.editable:
  225. req = f"-e {req}"
  226. return "\n".join(list(self.comments) + [str(req)]) + "\n"