archive_util.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294
  1. """distutils.archive_util
  2. Utility functions for creating archive files (tarballs, zip files,
  3. that sort of thing)."""
  4. from __future__ import annotations
  5. import os
  6. from typing import Literal, overload
  7. try:
  8. import zipfile
  9. except ImportError:
  10. zipfile = None
  11. from ._log import log
  12. from .dir_util import mkpath
  13. from .errors import DistutilsExecError
  14. from .spawn import spawn
  15. try:
  16. from pwd import getpwnam
  17. except ImportError:
  18. getpwnam = None
  19. try:
  20. from grp import getgrnam
  21. except ImportError:
  22. getgrnam = None
  23. def _get_gid(name):
  24. """Returns a gid, given a group name."""
  25. if getgrnam is None or name is None:
  26. return None
  27. try:
  28. result = getgrnam(name)
  29. except KeyError:
  30. result = None
  31. if result is not None:
  32. return result[2]
  33. return None
  34. def _get_uid(name):
  35. """Returns an uid, given a user name."""
  36. if getpwnam is None or name is None:
  37. return None
  38. try:
  39. result = getpwnam(name)
  40. except KeyError:
  41. result = None
  42. if result is not None:
  43. return result[2]
  44. return None
  45. def make_tarball(
  46. base_name: str,
  47. base_dir: str | os.PathLike[str],
  48. compress: Literal["gzip", "bzip2", "xz"] | None = "gzip",
  49. verbose: bool = False,
  50. dry_run: bool = False,
  51. owner: str | None = None,
  52. group: str | None = None,
  53. ) -> str:
  54. """Create a (possibly compressed) tar file from all the files under
  55. 'base_dir'.
  56. 'compress' must be "gzip" (the default), "bzip2", "xz", or None.
  57. 'owner' and 'group' can be used to define an owner and a group for the
  58. archive that is being built. If not provided, the current owner and group
  59. will be used.
  60. The output tar file will be named 'base_dir' + ".tar", possibly plus
  61. the appropriate compression extension (".gz", ".bz2", ".xz" or ".Z").
  62. Returns the output filename.
  63. """
  64. tar_compression = {
  65. 'gzip': 'gz',
  66. 'bzip2': 'bz2',
  67. 'xz': 'xz',
  68. None: '',
  69. }
  70. compress_ext = {'gzip': '.gz', 'bzip2': '.bz2', 'xz': '.xz'}
  71. # flags for compression program, each element of list will be an argument
  72. if compress is not None and compress not in compress_ext.keys():
  73. raise ValueError(
  74. "bad value for 'compress': must be None, 'gzip', 'bzip2', 'xz'"
  75. )
  76. archive_name = base_name + '.tar'
  77. archive_name += compress_ext.get(compress, '')
  78. mkpath(os.path.dirname(archive_name), dry_run=dry_run)
  79. # creating the tarball
  80. import tarfile # late import so Python build itself doesn't break
  81. log.info('Creating tar archive')
  82. uid = _get_uid(owner)
  83. gid = _get_gid(group)
  84. def _set_uid_gid(tarinfo):
  85. if gid is not None:
  86. tarinfo.gid = gid
  87. tarinfo.gname = group
  88. if uid is not None:
  89. tarinfo.uid = uid
  90. tarinfo.uname = owner
  91. return tarinfo
  92. if not dry_run:
  93. tar = tarfile.open(archive_name, f'w|{tar_compression[compress]}')
  94. try:
  95. tar.add(base_dir, filter=_set_uid_gid)
  96. finally:
  97. tar.close()
  98. return archive_name
  99. def make_zipfile( # noqa: C901
  100. base_name: str,
  101. base_dir: str | os.PathLike[str],
  102. verbose: bool = False,
  103. dry_run: bool = False,
  104. ) -> str:
  105. """Create a zip file from all the files under 'base_dir'.
  106. The output zip file will be named 'base_name' + ".zip". Uses either the
  107. "zipfile" Python module (if available) or the InfoZIP "zip" utility
  108. (if installed and found on the default search path). If neither tool is
  109. available, raises DistutilsExecError. Returns the name of the output zip
  110. file.
  111. """
  112. zip_filename = base_name + ".zip"
  113. mkpath(os.path.dirname(zip_filename), dry_run=dry_run)
  114. # If zipfile module is not available, try spawning an external
  115. # 'zip' command.
  116. if zipfile is None:
  117. if verbose:
  118. zipoptions = "-r"
  119. else:
  120. zipoptions = "-rq"
  121. try:
  122. spawn(["zip", zipoptions, zip_filename, base_dir], dry_run=dry_run)
  123. except DistutilsExecError:
  124. # XXX really should distinguish between "couldn't find
  125. # external 'zip' command" and "zip failed".
  126. raise DistutilsExecError(
  127. f"unable to create zip file '{zip_filename}': "
  128. "could neither import the 'zipfile' module nor "
  129. "find a standalone zip utility"
  130. )
  131. else:
  132. log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)
  133. if not dry_run:
  134. try:
  135. zip = zipfile.ZipFile(
  136. zip_filename, "w", compression=zipfile.ZIP_DEFLATED
  137. )
  138. except RuntimeError:
  139. zip = zipfile.ZipFile(zip_filename, "w", compression=zipfile.ZIP_STORED)
  140. with zip:
  141. if base_dir != os.curdir:
  142. path = os.path.normpath(os.path.join(base_dir, ''))
  143. zip.write(path, path)
  144. log.info("adding '%s'", path)
  145. for dirpath, dirnames, filenames in os.walk(base_dir):
  146. for name in dirnames:
  147. path = os.path.normpath(os.path.join(dirpath, name, ''))
  148. zip.write(path, path)
  149. log.info("adding '%s'", path)
  150. for name in filenames:
  151. path = os.path.normpath(os.path.join(dirpath, name))
  152. if os.path.isfile(path):
  153. zip.write(path, path)
  154. log.info("adding '%s'", path)
  155. return zip_filename
  156. ARCHIVE_FORMATS = {
  157. 'gztar': (make_tarball, [('compress', 'gzip')], "gzip'ed tar-file"),
  158. 'bztar': (make_tarball, [('compress', 'bzip2')], "bzip2'ed tar-file"),
  159. 'xztar': (make_tarball, [('compress', 'xz')], "xz'ed tar-file"),
  160. 'ztar': (make_tarball, [('compress', 'compress')], "compressed tar file"),
  161. 'tar': (make_tarball, [('compress', None)], "uncompressed tar file"),
  162. 'zip': (make_zipfile, [], "ZIP file"),
  163. }
  164. def check_archive_formats(formats):
  165. """Returns the first format from the 'format' list that is unknown.
  166. If all formats are known, returns None
  167. """
  168. for format in formats:
  169. if format not in ARCHIVE_FORMATS:
  170. return format
  171. return None
  172. @overload
  173. def make_archive(
  174. base_name: str,
  175. format: str,
  176. root_dir: str | os.PathLike[str] | bytes | os.PathLike[bytes] | None = None,
  177. base_dir: str | None = None,
  178. verbose: bool = False,
  179. dry_run: bool = False,
  180. owner: str | None = None,
  181. group: str | None = None,
  182. ) -> str: ...
  183. @overload
  184. def make_archive(
  185. base_name: str | os.PathLike[str],
  186. format: str,
  187. root_dir: str | os.PathLike[str] | bytes | os.PathLike[bytes],
  188. base_dir: str | None = None,
  189. verbose: bool = False,
  190. dry_run: bool = False,
  191. owner: str | None = None,
  192. group: str | None = None,
  193. ) -> str: ...
  194. def make_archive(
  195. base_name: str | os.PathLike[str],
  196. format: str,
  197. root_dir: str | os.PathLike[str] | bytes | os.PathLike[bytes] | None = None,
  198. base_dir: str | None = None,
  199. verbose: bool = False,
  200. dry_run: bool = False,
  201. owner: str | None = None,
  202. group: str | None = None,
  203. ) -> str:
  204. """Create an archive file (eg. zip or tar).
  205. 'base_name' is the name of the file to create, minus any format-specific
  206. extension; 'format' is the archive format: one of "zip", "tar", "gztar",
  207. "bztar", "xztar", or "ztar".
  208. 'root_dir' is a directory that will be the root directory of the
  209. archive; ie. we typically chdir into 'root_dir' before creating the
  210. archive. 'base_dir' is the directory where we start archiving from;
  211. ie. 'base_dir' will be the common prefix of all files and
  212. directories in the archive. 'root_dir' and 'base_dir' both default
  213. to the current directory. Returns the name of the archive file.
  214. 'owner' and 'group' are used when creating a tar archive. By default,
  215. uses the current owner and group.
  216. """
  217. save_cwd = os.getcwd()
  218. if root_dir is not None:
  219. log.debug("changing into '%s'", root_dir)
  220. base_name = os.path.abspath(base_name)
  221. if not dry_run:
  222. os.chdir(root_dir)
  223. if base_dir is None:
  224. base_dir = os.curdir
  225. kwargs = {'dry_run': dry_run}
  226. try:
  227. format_info = ARCHIVE_FORMATS[format]
  228. except KeyError:
  229. raise ValueError(f"unknown archive format '{format}'")
  230. func = format_info[0]
  231. kwargs.update(format_info[1])
  232. if format != 'zip':
  233. kwargs['owner'] = owner
  234. kwargs['group'] = group
  235. try:
  236. filename = func(base_name, base_dir, **kwargs)
  237. finally:
  238. if root_dir is not None:
  239. log.debug("changing back to '%s'", save_cwd)
  240. os.chdir(save_cwd)
  241. return filename