embed.h 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320
  1. /*
  2. pybind11/embed.h: Support for embedding the interpreter
  3. Copyright (c) 2017 Wenzel Jakob <wenzel.jakob@epfl.ch>
  4. All rights reserved. Use of this source code is governed by a
  5. BSD-style license that can be found in the LICENSE file.
  6. */
  7. #pragma once
  8. #include "pybind11.h"
  9. #include "eval.h"
  10. #include <memory>
  11. #include <vector>
  12. #if defined(PYPY_VERSION)
  13. # error Embedding the interpreter is not supported with PyPy
  14. #endif
  15. #define PYBIND11_EMBEDDED_MODULE_IMPL(name) \
  16. extern "C" PyObject *pybind11_init_impl_##name(); \
  17. extern "C" PyObject *pybind11_init_impl_##name() { return pybind11_init_wrapper_##name(); }
  18. /** \rst
  19. Add a new module to the table of builtins for the interpreter. Must be
  20. defined in global scope. The first macro parameter is the name of the
  21. module (without quotes). The second parameter is the variable which will
  22. be used as the interface to add functions and classes to the module.
  23. .. code-block:: cpp
  24. PYBIND11_EMBEDDED_MODULE(example, m) {
  25. // ... initialize functions and classes here
  26. m.def("foo", []() {
  27. return "Hello, World!";
  28. });
  29. }
  30. The third and subsequent macro arguments are optional, and can be used to
  31. mark the module as supporting various Python features.
  32. - ``mod_gil_not_used()``
  33. - ``multiple_interpreters::per_interpreter_gil()``
  34. - ``multiple_interpreters::shared_gil()``
  35. - ``multiple_interpreters::not_supported()``
  36. .. code-block:: cpp
  37. PYBIND11_EMBEDDED_MODULE(example, m, py::mod_gil_not_used()) {
  38. m.def("foo", []() {
  39. return "Hello, Free-threaded World!";
  40. });
  41. }
  42. \endrst */
  43. PYBIND11_WARNING_PUSH
  44. PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments")
  45. #define PYBIND11_EMBEDDED_MODULE(name, variable, ...) \
  46. PYBIND11_MODULE_PYINIT(name, {}, ##__VA_ARGS__) \
  47. ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)( \
  48. PYBIND11_TOSTRING(name), PYBIND11_CONCAT(PyInit_, name)); \
  49. PYBIND11_MODULE_EXEC(name, variable)
  50. PYBIND11_WARNING_POP
  51. PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
  52. PYBIND11_NAMESPACE_BEGIN(detail)
  53. /// Python 2.7/3.x compatible version of `PyImport_AppendInittab` and error checks.
  54. struct embedded_module {
  55. using init_t = PyObject *(*) ();
  56. embedded_module(const char *name, init_t init) {
  57. if (Py_IsInitialized() != 0) {
  58. pybind11_fail("Can't add new modules after the interpreter has been initialized");
  59. }
  60. auto result = PyImport_AppendInittab(name, init);
  61. if (result == -1) {
  62. pybind11_fail("Insufficient memory to add a new module");
  63. }
  64. }
  65. };
  66. struct wide_char_arg_deleter {
  67. void operator()(wchar_t *ptr) const {
  68. // API docs: https://docs.python.org/3/c-api/sys.html#c.Py_DecodeLocale
  69. PyMem_RawFree(ptr);
  70. }
  71. };
  72. inline wchar_t *widen_chars(const char *safe_arg) {
  73. wchar_t *widened_arg = Py_DecodeLocale(safe_arg, nullptr);
  74. return widened_arg;
  75. }
  76. inline void precheck_interpreter() {
  77. if (Py_IsInitialized() != 0) {
  78. pybind11_fail("The interpreter is already running");
  79. }
  80. }
  81. #if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX)
  82. # define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000)
  83. #endif
  84. #if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
  85. inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers,
  86. int argc,
  87. const char *const *argv,
  88. bool add_program_dir_to_path) {
  89. detail::precheck_interpreter();
  90. Py_InitializeEx(init_signal_handlers ? 1 : 0);
  91. auto argv_size = static_cast<size_t>(argc);
  92. // SetArgv* on python 3 takes wchar_t, so we have to convert.
  93. std::unique_ptr<wchar_t *[]> widened_argv(new wchar_t *[argv_size]);
  94. std::vector<std::unique_ptr<wchar_t[], detail::wide_char_arg_deleter>> widened_argv_entries;
  95. widened_argv_entries.reserve(argv_size);
  96. for (size_t ii = 0; ii < argv_size; ++ii) {
  97. widened_argv_entries.emplace_back(detail::widen_chars(argv[ii]));
  98. if (!widened_argv_entries.back()) {
  99. // A null here indicates a character-encoding failure or the python
  100. // interpreter out of memory. Give up.
  101. return;
  102. }
  103. widened_argv[ii] = widened_argv_entries.back().get();
  104. }
  105. auto *pysys_argv = widened_argv.get();
  106. PySys_SetArgvEx(argc, pysys_argv, static_cast<int>(add_program_dir_to_path));
  107. }
  108. #endif
  109. PYBIND11_NAMESPACE_END(detail)
  110. #if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
  111. inline void initialize_interpreter(PyConfig *config,
  112. int argc = 0,
  113. const char *const *argv = nullptr,
  114. bool add_program_dir_to_path = true) {
  115. detail::precheck_interpreter();
  116. PyStatus status = PyConfig_SetBytesArgv(config, argc, const_cast<char *const *>(argv));
  117. if (PyStatus_Exception(status) != 0) {
  118. // A failure here indicates a character-encoding failure or the python
  119. // interpreter out of memory. Give up.
  120. PyConfig_Clear(config);
  121. throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
  122. : "Failed to prepare CPython");
  123. }
  124. status = Py_InitializeFromConfig(config);
  125. if (PyStatus_Exception(status) != 0) {
  126. PyConfig_Clear(config);
  127. throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
  128. : "Failed to init CPython");
  129. }
  130. if (add_program_dir_to_path) {
  131. PyRun_SimpleString("import sys, os.path; "
  132. "sys.path.insert(0, "
  133. "os.path.abspath(os.path.dirname(sys.argv[0])) "
  134. "if sys.argv and os.path.exists(sys.argv[0]) else '')");
  135. }
  136. PyConfig_Clear(config);
  137. }
  138. #endif
  139. /** \rst
  140. Initialize the Python interpreter. No other pybind11 or CPython API functions can be
  141. called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The
  142. optional `init_signal_handlers` parameter can be used to skip the registration of
  143. signal handlers (see the `Python documentation`_ for details). Calling this function
  144. again after the interpreter has already been initialized is a fatal error.
  145. If initializing the Python interpreter fails, then the program is terminated. (This
  146. is controlled by the CPython runtime and is an exception to pybind11's normal behavior
  147. of throwing exceptions on errors.)
  148. The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are
  149. used to populate ``sys.argv`` and ``sys.path``.
  150. See the |PySys_SetArgvEx documentation|_ for details.
  151. .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx
  152. .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation
  153. .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx
  154. \endrst */
  155. inline void initialize_interpreter(bool init_signal_handlers = true,
  156. int argc = 0,
  157. const char *const *argv = nullptr,
  158. bool add_program_dir_to_path = true) {
  159. #if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
  160. detail::initialize_interpreter_pre_pyconfig(
  161. init_signal_handlers, argc, argv, add_program_dir_to_path);
  162. #else
  163. PyConfig config;
  164. PyConfig_InitPythonConfig(&config);
  165. // See PR #4473 for background
  166. config.parse_argv = 0;
  167. config.install_signal_handlers = init_signal_handlers ? 1 : 0;
  168. initialize_interpreter(&config, argc, argv, add_program_dir_to_path);
  169. #endif
  170. // There is exactly one interpreter alive currently.
  171. detail::get_num_interpreters_seen() = 1;
  172. }
  173. /** \rst
  174. Shut down the Python interpreter. No pybind11 or CPython API functions can be called
  175. after this. In addition, pybind11 objects must not outlive the interpreter:
  176. .. code-block:: cpp
  177. { // BAD
  178. py::initialize_interpreter();
  179. auto hello = py::str("Hello, World!");
  180. py::finalize_interpreter();
  181. } // <-- BOOM, hello's destructor is called after interpreter shutdown
  182. { // GOOD
  183. py::initialize_interpreter();
  184. { // scoped
  185. auto hello = py::str("Hello, World!");
  186. } // <-- OK, hello is cleaned up properly
  187. py::finalize_interpreter();
  188. }
  189. { // BETTER
  190. py::scoped_interpreter guard{};
  191. auto hello = py::str("Hello, World!");
  192. }
  193. .. warning::
  194. The interpreter can be restarted by calling `initialize_interpreter` again.
  195. Modules created using pybind11 can be safely re-initialized. However, Python
  196. itself cannot completely unload binary extension modules and there are several
  197. caveats with regard to interpreter restarting. All the details can be found
  198. in the CPython documentation. In short, not all interpreter memory may be
  199. freed, either due to reference cycles or user-created global data.
  200. \endrst */
  201. inline void finalize_interpreter() {
  202. // get rid of any thread-local interpreter cache that currently exists
  203. if (detail::get_num_interpreters_seen() > 1) {
  204. detail::get_internals_pp_manager().unref();
  205. detail::get_local_internals_pp_manager().unref();
  206. // We know there can be no other interpreter alive now, so we can lower the count
  207. detail::get_num_interpreters_seen() = 1;
  208. }
  209. // Re-fetch the internals pointer-to-pointer (but not the internals itself, which might not
  210. // exist). It's possible for the internals to be created during Py_Finalize() (e.g. if a
  211. // py::capsule calls `get_internals()` during destruction), so we get the pointer-pointer here
  212. // and check it after Py_Finalize().
  213. detail::get_internals_pp_manager().get_pp();
  214. detail::get_local_internals_pp_manager().get_pp();
  215. Py_Finalize();
  216. detail::get_internals_pp_manager().destroy();
  217. // Local internals contains data managed by the current interpreter, so we must clear them to
  218. // avoid undefined behaviors when initializing another interpreter
  219. detail::get_local_internals_pp_manager().destroy();
  220. // We know there is no interpreter alive now, so we can reset the count
  221. detail::get_num_interpreters_seen() = 0;
  222. }
  223. /** \rst
  224. Scope guard version of `initialize_interpreter` and `finalize_interpreter`.
  225. This a move-only guard and only a single instance can exist.
  226. See `initialize_interpreter` for a discussion of its constructor arguments.
  227. .. code-block:: cpp
  228. #include <pybind11/embed.h>
  229. int main() {
  230. py::scoped_interpreter guard{};
  231. py::print(Hello, World!);
  232. } // <-- interpreter shutdown
  233. \endrst */
  234. class scoped_interpreter {
  235. public:
  236. explicit scoped_interpreter(bool init_signal_handlers = true,
  237. int argc = 0,
  238. const char *const *argv = nullptr,
  239. bool add_program_dir_to_path = true) {
  240. initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path);
  241. }
  242. #if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
  243. explicit scoped_interpreter(PyConfig *config,
  244. int argc = 0,
  245. const char *const *argv = nullptr,
  246. bool add_program_dir_to_path = true) {
  247. initialize_interpreter(config, argc, argv, add_program_dir_to_path);
  248. }
  249. #endif
  250. scoped_interpreter(const scoped_interpreter &) = delete;
  251. scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; }
  252. scoped_interpreter &operator=(const scoped_interpreter &) = delete;
  253. scoped_interpreter &operator=(scoped_interpreter &&) = delete;
  254. ~scoped_interpreter() {
  255. if (is_valid) {
  256. finalize_interpreter();
  257. }
  258. }
  259. private:
  260. bool is_valid = true;
  261. };
  262. PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)