extbuild.py 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. """
  2. Build a c-extension module on-the-fly in tests.
  3. See build_and_import_extensions for usage hints
  4. """
  5. import os
  6. import pathlib
  7. import subprocess
  8. import sys
  9. import sysconfig
  10. import textwrap
  11. __all__ = ['build_and_import_extension', 'compile_extension_module']
  12. def build_and_import_extension(
  13. modname, functions, *, prologue="", build_dir=None,
  14. include_dirs=[], more_init=""):
  15. """
  16. Build and imports a c-extension module `modname` from a list of function
  17. fragments `functions`.
  18. Parameters
  19. ----------
  20. functions : list of fragments
  21. Each fragment is a sequence of func_name, calling convention, snippet.
  22. prologue : string
  23. Code to precede the rest, usually extra ``#include`` or ``#define``
  24. macros.
  25. build_dir : pathlib.Path
  26. Where to build the module, usually a temporary directory
  27. include_dirs : list
  28. Extra directories to find include files when compiling
  29. more_init : string
  30. Code to appear in the module PyMODINIT_FUNC
  31. Returns
  32. -------
  33. out: module
  34. The module will have been loaded and is ready for use
  35. Examples
  36. --------
  37. >>> functions = [("test_bytes", "METH_O", \"\"\"
  38. if ( !PyBytesCheck(args)) {
  39. Py_RETURN_FALSE;
  40. }
  41. Py_RETURN_TRUE;
  42. \"\"\")]
  43. >>> mod = build_and_import_extension("testme", functions)
  44. >>> assert not mod.test_bytes(u'abc')
  45. >>> assert mod.test_bytes(b'abc')
  46. """
  47. body = prologue + _make_methods(functions, modname)
  48. init = """PyObject *mod = PyModule_Create(&moduledef);
  49. """
  50. if not build_dir:
  51. build_dir = pathlib.Path('.')
  52. if more_init:
  53. init += """#define INITERROR return NULL
  54. """
  55. init += more_init
  56. init += "\nreturn mod;"
  57. source_string = _make_source(modname, init, body)
  58. try:
  59. mod_so = compile_extension_module(
  60. modname, build_dir, include_dirs, source_string)
  61. except Exception as e:
  62. # shorten the exception chain
  63. raise RuntimeError(f"could not compile in {build_dir}:") from e
  64. import importlib.util
  65. spec = importlib.util.spec_from_file_location(modname, mod_so)
  66. foo = importlib.util.module_from_spec(spec)
  67. spec.loader.exec_module(foo)
  68. return foo
  69. def compile_extension_module(
  70. name, builddir, include_dirs,
  71. source_string, libraries=[], library_dirs=[]):
  72. """
  73. Build an extension module and return the filename of the resulting
  74. native code file.
  75. Parameters
  76. ----------
  77. name : string
  78. name of the module, possibly including dots if it is a module inside a
  79. package.
  80. builddir : pathlib.Path
  81. Where to build the module, usually a temporary directory
  82. include_dirs : list
  83. Extra directories to find include files when compiling
  84. libraries : list
  85. Libraries to link into the extension module
  86. library_dirs: list
  87. Where to find the libraries, ``-L`` passed to the linker
  88. """
  89. modname = name.split('.')[-1]
  90. dirname = builddir / name
  91. dirname.mkdir(exist_ok=True)
  92. cfile = _convert_str_to_file(source_string, dirname)
  93. include_dirs = include_dirs + [sysconfig.get_config_var('INCLUDEPY')]
  94. return _c_compile(
  95. cfile, outputfilename=dirname / modname,
  96. include_dirs=include_dirs, libraries=[], library_dirs=[],
  97. )
  98. def _convert_str_to_file(source, dirname):
  99. """Helper function to create a file ``source.c`` in `dirname` that contains
  100. the string in `source`. Returns the file name
  101. """
  102. filename = dirname / 'source.c'
  103. with filename.open('w') as f:
  104. f.write(str(source))
  105. return filename
  106. def _make_methods(functions, modname):
  107. """ Turns the name, signature, code in functions into complete functions
  108. and lists them in a methods_table. Then turns the methods_table into a
  109. ``PyMethodDef`` structure and returns the resulting code fragment ready
  110. for compilation
  111. """
  112. methods_table = []
  113. codes = []
  114. for funcname, flags, code in functions:
  115. cfuncname = "%s_%s" % (modname, funcname)
  116. if 'METH_KEYWORDS' in flags:
  117. signature = '(PyObject *self, PyObject *args, PyObject *kwargs)'
  118. else:
  119. signature = '(PyObject *self, PyObject *args)'
  120. methods_table.append(
  121. "{\"%s\", (PyCFunction)%s, %s}," % (funcname, cfuncname, flags))
  122. func_code = """
  123. static PyObject* {cfuncname}{signature}
  124. {{
  125. {code}
  126. }}
  127. """.format(cfuncname=cfuncname, signature=signature, code=code)
  128. codes.append(func_code)
  129. body = "\n".join(codes) + """
  130. static PyMethodDef methods[] = {
  131. %(methods)s
  132. { NULL }
  133. };
  134. static struct PyModuleDef moduledef = {
  135. PyModuleDef_HEAD_INIT,
  136. "%(modname)s", /* m_name */
  137. NULL, /* m_doc */
  138. -1, /* m_size */
  139. methods, /* m_methods */
  140. };
  141. """ % dict(methods='\n'.join(methods_table), modname=modname)
  142. return body
  143. def _make_source(name, init, body):
  144. """ Combines the code fragments into source code ready to be compiled
  145. """
  146. code = """
  147. #include <Python.h>
  148. %(body)s
  149. PyMODINIT_FUNC
  150. PyInit_%(name)s(void) {
  151. %(init)s
  152. }
  153. """ % dict(
  154. name=name, init=init, body=body,
  155. )
  156. return code
  157. def _c_compile(cfile, outputfilename, include_dirs=[], libraries=[],
  158. library_dirs=[]):
  159. if sys.platform == 'win32':
  160. compile_extra = ["/we4013"]
  161. link_extra = ["/LIBPATH:" + os.path.join(sys.base_prefix, 'libs')]
  162. elif sys.platform.startswith('linux'):
  163. compile_extra = [
  164. "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
  165. link_extra = []
  166. else:
  167. compile_extra = link_extra = []
  168. pass
  169. if sys.platform == 'win32':
  170. link_extra = link_extra + ['/DEBUG'] # generate .pdb file
  171. if sys.platform == 'darwin':
  172. # support Fink & Darwinports
  173. for s in ('/sw/', '/opt/local/'):
  174. if (s + 'include' not in include_dirs
  175. and os.path.exists(s + 'include')):
  176. include_dirs.append(s + 'include')
  177. if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'):
  178. library_dirs.append(s + 'lib')
  179. outputfilename = outputfilename.with_suffix(get_so_suffix())
  180. build(
  181. cfile, outputfilename,
  182. compile_extra, link_extra,
  183. include_dirs, libraries, library_dirs)
  184. return outputfilename
  185. def build(cfile, outputfilename, compile_extra, link_extra,
  186. include_dirs, libraries, library_dirs):
  187. "use meson to build"
  188. build_dir = cfile.parent / "build"
  189. os.makedirs(build_dir, exist_ok=True)
  190. so_name = outputfilename.parts[-1]
  191. with open(cfile.parent / "meson.build", "wt") as fid:
  192. includes = ['-I' + d for d in include_dirs]
  193. link_dirs = ['-L' + d for d in library_dirs]
  194. fid.write(textwrap.dedent(f"""\
  195. project('foo', 'c')
  196. shared_module('{so_name}', '{cfile.parts[-1]}',
  197. c_args: {includes} + {compile_extra},
  198. link_args: {link_dirs} + {link_extra},
  199. link_with: {libraries},
  200. name_prefix: '',
  201. name_suffix: 'dummy',
  202. )
  203. """))
  204. if sys.platform == "win32":
  205. subprocess.check_call(["meson", "setup",
  206. "--buildtype=release",
  207. "--vsenv", ".."],
  208. cwd=build_dir,
  209. )
  210. else:
  211. subprocess.check_call(["meson", "setup", "--vsenv", ".."],
  212. cwd=build_dir
  213. )
  214. subprocess.check_call(["meson", "compile"], cwd=build_dir)
  215. os.rename(str(build_dir / so_name) + ".dummy", cfile.parent / so_name)
  216. def get_so_suffix():
  217. ret = sysconfig.get_config_var('EXT_SUFFIX')
  218. assert ret
  219. return ret