build_ext.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470
  1. from __future__ import annotations
  2. import itertools
  3. import os
  4. import sys
  5. import textwrap
  6. from collections.abc import Iterator
  7. from importlib.machinery import EXTENSION_SUFFIXES
  8. from importlib.util import cache_from_source as _compiled_file_name
  9. from pathlib import Path
  10. from typing import TYPE_CHECKING
  11. from setuptools.dist import Distribution
  12. from setuptools.errors import BaseError
  13. from setuptools.extension import Extension, Library
  14. from distutils import log
  15. from distutils.ccompiler import new_compiler
  16. from distutils.sysconfig import customize_compiler, get_config_var
  17. if TYPE_CHECKING:
  18. # Cython not installed on CI tests, causing _build_ext to be `Any`
  19. from distutils.command.build_ext import build_ext as _build_ext
  20. else:
  21. try:
  22. # Attempt to use Cython for building extensions, if available
  23. from Cython.Distutils.build_ext import build_ext as _build_ext
  24. # Additionally, assert that the compiler module will load
  25. # also. Ref #1229.
  26. __import__('Cython.Compiler.Main')
  27. except ImportError:
  28. from distutils.command.build_ext import build_ext as _build_ext
  29. # make sure _config_vars is initialized
  30. get_config_var("LDSHARED")
  31. # Not publicly exposed in typeshed distutils stubs, but this is done on purpose
  32. # See https://github.com/pypa/setuptools/pull/4228#issuecomment-1959856400
  33. from distutils.sysconfig import _config_vars as _CONFIG_VARS # noqa: E402
  34. def _customize_compiler_for_shlib(compiler):
  35. if sys.platform == "darwin":
  36. # building .dylib requires additional compiler flags on OSX; here we
  37. # temporarily substitute the pyconfig.h variables so that distutils'
  38. # 'customize_compiler' uses them before we build the shared libraries.
  39. tmp = _CONFIG_VARS.copy()
  40. try:
  41. # XXX Help! I don't have any idea whether these are right...
  42. _CONFIG_VARS['LDSHARED'] = (
  43. "gcc -Wl,-x -dynamiclib -undefined dynamic_lookup"
  44. )
  45. _CONFIG_VARS['CCSHARED'] = " -dynamiclib"
  46. _CONFIG_VARS['SO'] = ".dylib"
  47. customize_compiler(compiler)
  48. finally:
  49. _CONFIG_VARS.clear()
  50. _CONFIG_VARS.update(tmp)
  51. else:
  52. customize_compiler(compiler)
  53. have_rtld = False
  54. use_stubs = False
  55. libtype = 'shared'
  56. if sys.platform == "darwin":
  57. use_stubs = True
  58. elif os.name != 'nt':
  59. try:
  60. import dl # type: ignore[import-not-found] # https://github.com/python/mypy/issues/13002
  61. use_stubs = have_rtld = hasattr(dl, 'RTLD_NOW')
  62. except ImportError:
  63. pass
  64. def get_abi3_suffix():
  65. """Return the file extension for an abi3-compliant Extension()"""
  66. for suffix in EXTENSION_SUFFIXES:
  67. if '.abi3' in suffix: # Unix
  68. return suffix
  69. elif suffix == '.pyd': # Windows
  70. return suffix
  71. return None
  72. class build_ext(_build_ext):
  73. distribution: Distribution # override distutils.dist.Distribution with setuptools.dist.Distribution
  74. editable_mode = False
  75. inplace = False
  76. def run(self):
  77. """Build extensions in build directory, then copy if --inplace"""
  78. old_inplace, self.inplace = self.inplace, False
  79. _build_ext.run(self)
  80. self.inplace = old_inplace
  81. if old_inplace:
  82. self.copy_extensions_to_source()
  83. def _get_inplace_equivalent(self, build_py, ext: Extension) -> tuple[str, str]:
  84. fullname = self.get_ext_fullname(ext.name)
  85. filename = self.get_ext_filename(fullname)
  86. modpath = fullname.split('.')
  87. package = '.'.join(modpath[:-1])
  88. package_dir = build_py.get_package_dir(package)
  89. inplace_file = os.path.join(package_dir, os.path.basename(filename))
  90. regular_file = os.path.join(self.build_lib, filename)
  91. return (inplace_file, regular_file)
  92. def copy_extensions_to_source(self) -> None:
  93. build_py = self.get_finalized_command('build_py')
  94. for ext in self.extensions:
  95. inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext)
  96. # Always copy, even if source is older than destination, to ensure
  97. # that the right extensions for the current Python/platform are
  98. # used.
  99. if os.path.exists(regular_file) or not ext.optional:
  100. self.copy_file(regular_file, inplace_file, level=self.verbose)
  101. if ext._needs_stub:
  102. inplace_stub = self._get_equivalent_stub(ext, inplace_file)
  103. self._write_stub_file(inplace_stub, ext, compile=True)
  104. # Always compile stub and remove the original (leave the cache behind)
  105. # (this behaviour was observed in previous iterations of the code)
  106. def _get_equivalent_stub(self, ext: Extension, output_file: str) -> str:
  107. dir_ = os.path.dirname(output_file)
  108. _, _, name = ext.name.rpartition(".")
  109. return f"{os.path.join(dir_, name)}.py"
  110. def _get_output_mapping(self) -> Iterator[tuple[str, str]]:
  111. if not self.inplace:
  112. return
  113. build_py = self.get_finalized_command('build_py')
  114. opt = self.get_finalized_command('install_lib').optimize or ""
  115. for ext in self.extensions:
  116. inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext)
  117. yield (regular_file, inplace_file)
  118. if ext._needs_stub:
  119. # This version of `build_ext` always builds artifacts in another dir,
  120. # when "inplace=True" is given it just copies them back.
  121. # This is done in the `copy_extensions_to_source` function, which
  122. # always compile stub files via `_compile_and_remove_stub`.
  123. # At the end of the process, a `.pyc` stub file is created without the
  124. # corresponding `.py`.
  125. inplace_stub = self._get_equivalent_stub(ext, inplace_file)
  126. regular_stub = self._get_equivalent_stub(ext, regular_file)
  127. inplace_cache = _compiled_file_name(inplace_stub, optimization=opt)
  128. output_cache = _compiled_file_name(regular_stub, optimization=opt)
  129. yield (output_cache, inplace_cache)
  130. def get_ext_filename(self, fullname: str) -> str:
  131. so_ext = os.getenv('SETUPTOOLS_EXT_SUFFIX')
  132. if so_ext:
  133. filename = os.path.join(*fullname.split('.')) + so_ext
  134. else:
  135. filename = _build_ext.get_ext_filename(self, fullname)
  136. ext_suffix = get_config_var('EXT_SUFFIX')
  137. if not isinstance(ext_suffix, str):
  138. raise OSError(
  139. "Configuration variable EXT_SUFFIX not found for this platform "
  140. "and environment variable SETUPTOOLS_EXT_SUFFIX is missing"
  141. )
  142. so_ext = ext_suffix
  143. if fullname in self.ext_map:
  144. ext = self.ext_map[fullname]
  145. abi3_suffix = get_abi3_suffix()
  146. if ext.py_limited_api and abi3_suffix: # Use abi3
  147. filename = filename[: -len(so_ext)] + abi3_suffix
  148. if isinstance(ext, Library):
  149. fn, ext = os.path.splitext(filename)
  150. return self.shlib_compiler.library_filename(fn, libtype)
  151. elif use_stubs and ext._links_to_dynamic:
  152. d, fn = os.path.split(filename)
  153. return os.path.join(d, 'dl-' + fn)
  154. return filename
  155. def initialize_options(self):
  156. _build_ext.initialize_options(self)
  157. self.shlib_compiler = None
  158. self.shlibs = []
  159. self.ext_map = {}
  160. self.editable_mode = False
  161. def finalize_options(self) -> None:
  162. _build_ext.finalize_options(self)
  163. self.extensions = self.extensions or []
  164. self.check_extensions_list(self.extensions)
  165. self.shlibs = [ext for ext in self.extensions if isinstance(ext, Library)]
  166. if self.shlibs:
  167. self.setup_shlib_compiler()
  168. for ext in self.extensions:
  169. ext._full_name = self.get_ext_fullname(ext.name)
  170. for ext in self.extensions:
  171. fullname = ext._full_name
  172. self.ext_map[fullname] = ext
  173. # distutils 3.1 will also ask for module names
  174. # XXX what to do with conflicts?
  175. self.ext_map[fullname.split('.')[-1]] = ext
  176. ltd = self.shlibs and self.links_to_dynamic(ext) or False
  177. ns = ltd and use_stubs and not isinstance(ext, Library)
  178. ext._links_to_dynamic = ltd
  179. ext._needs_stub = ns
  180. filename = ext._file_name = self.get_ext_filename(fullname)
  181. libdir = os.path.dirname(os.path.join(self.build_lib, filename))
  182. if ltd and libdir not in ext.library_dirs:
  183. ext.library_dirs.append(libdir)
  184. if ltd and use_stubs and os.curdir not in ext.runtime_library_dirs:
  185. ext.runtime_library_dirs.append(os.curdir)
  186. if self.editable_mode:
  187. self.inplace = True
  188. def setup_shlib_compiler(self):
  189. compiler = self.shlib_compiler = new_compiler(
  190. compiler=self.compiler, dry_run=self.dry_run, force=self.force
  191. )
  192. _customize_compiler_for_shlib(compiler)
  193. if self.include_dirs is not None:
  194. compiler.set_include_dirs(self.include_dirs)
  195. if self.define is not None:
  196. # 'define' option is a list of (name,value) tuples
  197. for name, value in self.define:
  198. compiler.define_macro(name, value)
  199. if self.undef is not None:
  200. for macro in self.undef:
  201. compiler.undefine_macro(macro)
  202. if self.libraries is not None:
  203. compiler.set_libraries(self.libraries)
  204. if self.library_dirs is not None:
  205. compiler.set_library_dirs(self.library_dirs)
  206. if self.rpath is not None:
  207. compiler.set_runtime_library_dirs(self.rpath)
  208. if self.link_objects is not None:
  209. compiler.set_link_objects(self.link_objects)
  210. # hack so distutils' build_extension() builds a library instead
  211. compiler.link_shared_object = link_shared_object.__get__(compiler) # type: ignore[method-assign]
  212. def get_export_symbols(self, ext):
  213. if isinstance(ext, Library):
  214. return ext.export_symbols
  215. return _build_ext.get_export_symbols(self, ext)
  216. def build_extension(self, ext) -> None:
  217. ext._convert_pyx_sources_to_lang()
  218. _compiler = self.compiler
  219. try:
  220. if isinstance(ext, Library):
  221. self.compiler = self.shlib_compiler
  222. _build_ext.build_extension(self, ext)
  223. if ext._needs_stub:
  224. build_lib = self.get_finalized_command('build_py').build_lib
  225. self.write_stub(build_lib, ext)
  226. finally:
  227. self.compiler = _compiler
  228. def links_to_dynamic(self, ext):
  229. """Return true if 'ext' links to a dynamic lib in the same package"""
  230. # XXX this should check to ensure the lib is actually being built
  231. # XXX as dynamic, and not just using a locally-found version or a
  232. # XXX static-compiled version
  233. libnames = dict.fromkeys([lib._full_name for lib in self.shlibs])
  234. pkg = '.'.join(ext._full_name.split('.')[:-1] + [''])
  235. return any(pkg + libname in libnames for libname in ext.libraries)
  236. def get_source_files(self) -> list[str]:
  237. return [*_build_ext.get_source_files(self), *self._get_internal_depends()]
  238. def _get_internal_depends(self) -> Iterator[str]:
  239. """Yield ``ext.depends`` that are contained by the project directory"""
  240. project_root = Path(self.distribution.src_root or os.curdir).resolve()
  241. depends = (dep for ext in self.extensions for dep in ext.depends)
  242. def skip(orig_path: str, reason: str) -> None:
  243. log.info(
  244. "dependency %s won't be automatically "
  245. "included in the manifest: the path %s",
  246. orig_path,
  247. reason,
  248. )
  249. for dep in depends:
  250. path = Path(dep)
  251. if path.is_absolute():
  252. skip(dep, "must be relative")
  253. continue
  254. if ".." in path.parts:
  255. skip(dep, "can't have `..` segments")
  256. continue
  257. try:
  258. resolved = (project_root / path).resolve(strict=True)
  259. except OSError:
  260. skip(dep, "doesn't exist")
  261. continue
  262. try:
  263. resolved.relative_to(project_root)
  264. except ValueError:
  265. skip(dep, "must be inside the project root")
  266. continue
  267. yield path.as_posix()
  268. def get_outputs(self) -> list[str]:
  269. if self.inplace:
  270. return list(self.get_output_mapping().keys())
  271. return sorted(_build_ext.get_outputs(self) + self.__get_stubs_outputs())
  272. def get_output_mapping(self) -> dict[str, str]:
  273. """See :class:`setuptools.commands.build.SubCommand`"""
  274. mapping = self._get_output_mapping()
  275. return dict(sorted(mapping, key=lambda x: x[0]))
  276. def __get_stubs_outputs(self):
  277. # assemble the base name for each extension that needs a stub
  278. ns_ext_bases = (
  279. os.path.join(self.build_lib, *ext._full_name.split('.'))
  280. for ext in self.extensions
  281. if ext._needs_stub
  282. )
  283. # pair each base with the extension
  284. pairs = itertools.product(ns_ext_bases, self.__get_output_extensions())
  285. return list(base + fnext for base, fnext in pairs)
  286. def __get_output_extensions(self):
  287. yield '.py'
  288. yield '.pyc'
  289. if self.get_finalized_command('build_py').optimize:
  290. yield '.pyo'
  291. def write_stub(self, output_dir, ext, compile=False) -> None:
  292. stub_file = os.path.join(output_dir, *ext._full_name.split('.')) + '.py'
  293. self._write_stub_file(stub_file, ext, compile)
  294. def _write_stub_file(self, stub_file: str, ext: Extension, compile=False):
  295. log.info("writing stub loader for %s to %s", ext._full_name, stub_file)
  296. if compile and os.path.exists(stub_file):
  297. raise BaseError(stub_file + " already exists! Please delete.")
  298. if not self.dry_run:
  299. with open(stub_file, 'w', encoding="utf-8") as f:
  300. content = (
  301. textwrap.dedent(f"""
  302. def __bootstrap__():
  303. global __bootstrap__, __file__, __loader__
  304. import sys, os, importlib.resources as irs, importlib.util
  305. #rtld import dl
  306. with irs.files(__name__).joinpath(
  307. {os.path.basename(ext._file_name)!r}) as __file__:
  308. del __bootstrap__
  309. if '__loader__' in globals():
  310. del __loader__
  311. #rtld old_flags = sys.getdlopenflags()
  312. old_dir = os.getcwd()
  313. try:
  314. os.chdir(os.path.dirname(__file__))
  315. #rtld sys.setdlopenflags(dl.RTLD_NOW)
  316. spec = importlib.util.spec_from_file_location(
  317. __name__, __file__)
  318. mod = importlib.util.module_from_spec(spec)
  319. spec.loader.exec_module(mod)
  320. finally:
  321. #rtld sys.setdlopenflags(old_flags)
  322. os.chdir(old_dir)
  323. __bootstrap__()
  324. """)
  325. .lstrip()
  326. .replace('#rtld', '#rtld' * (not have_rtld))
  327. )
  328. f.write(content)
  329. if compile:
  330. self._compile_and_remove_stub(stub_file)
  331. def _compile_and_remove_stub(self, stub_file: str):
  332. from distutils.util import byte_compile
  333. byte_compile([stub_file], optimize=0, force=True, dry_run=self.dry_run)
  334. optimize = self.get_finalized_command('install_lib').optimize
  335. if optimize > 0:
  336. byte_compile(
  337. [stub_file],
  338. optimize=optimize,
  339. force=True,
  340. dry_run=self.dry_run,
  341. )
  342. if os.path.exists(stub_file) and not self.dry_run:
  343. os.unlink(stub_file)
  344. if use_stubs or os.name == 'nt':
  345. # Build shared libraries
  346. #
  347. def link_shared_object(
  348. self,
  349. objects,
  350. output_libname,
  351. output_dir=None,
  352. libraries=None,
  353. library_dirs=None,
  354. runtime_library_dirs=None,
  355. export_symbols=None,
  356. debug: bool = False,
  357. extra_preargs=None,
  358. extra_postargs=None,
  359. build_temp=None,
  360. target_lang=None,
  361. ) -> None:
  362. self.link(
  363. self.SHARED_LIBRARY,
  364. objects,
  365. output_libname,
  366. output_dir,
  367. libraries,
  368. library_dirs,
  369. runtime_library_dirs,
  370. export_symbols,
  371. debug,
  372. extra_preargs,
  373. extra_postargs,
  374. build_temp,
  375. target_lang,
  376. )
  377. else:
  378. # Build static libraries everywhere else
  379. libtype = 'static'
  380. def link_shared_object(
  381. self,
  382. objects,
  383. output_libname,
  384. output_dir=None,
  385. libraries=None,
  386. library_dirs=None,
  387. runtime_library_dirs=None,
  388. export_symbols=None,
  389. debug: bool = False,
  390. extra_preargs=None,
  391. extra_postargs=None,
  392. build_temp=None,
  393. target_lang=None,
  394. ) -> None:
  395. # XXX we need to either disallow these attrs on Library instances,
  396. # or warn/abort here if set, or something...
  397. # libraries=None, library_dirs=None, runtime_library_dirs=None,
  398. # export_symbols=None, extra_preargs=None, extra_postargs=None,
  399. # build_temp=None
  400. assert output_dir is None # distutils build_ext doesn't pass this
  401. output_dir, filename = os.path.split(output_libname)
  402. basename, _ext = os.path.splitext(filename)
  403. if self.library_filename("x").startswith('lib'):
  404. # strip 'lib' prefix; this is kludgy if some platform uses
  405. # a different prefix
  406. basename = basename[3:]
  407. self.create_static_lib(objects, basename, output_dir, debug, target_lang)