_runtime.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395
  1. # coding=utf-8
  2. # Copyright 2022-present, the HuggingFace Inc. team.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. """Check presence of installed packages at runtime."""
  16. import importlib.metadata
  17. import os
  18. import platform
  19. import sys
  20. import warnings
  21. from typing import Any, Dict
  22. from .. import __version__, constants
  23. _PY_VERSION: str = sys.version.split()[0].rstrip("+")
  24. _package_versions = {}
  25. _CANDIDATES = {
  26. "aiohttp": {"aiohttp"},
  27. "fastai": {"fastai"},
  28. "fastapi": {"fastapi"},
  29. "fastcore": {"fastcore"},
  30. "gradio": {"gradio"},
  31. "graphviz": {"graphviz"},
  32. "hf_transfer": {"hf_transfer"},
  33. "hf_xet": {"hf_xet"},
  34. "jinja": {"Jinja2"},
  35. "keras": {"keras"},
  36. "numpy": {"numpy"},
  37. "pillow": {"Pillow"},
  38. "pydantic": {"pydantic"},
  39. "pydot": {"pydot"},
  40. "safetensors": {"safetensors"},
  41. "tensorboard": {"tensorboardX"},
  42. "tensorflow": (
  43. "tensorflow",
  44. "tensorflow-cpu",
  45. "tensorflow-gpu",
  46. "tf-nightly",
  47. "tf-nightly-cpu",
  48. "tf-nightly-gpu",
  49. "intel-tensorflow",
  50. "intel-tensorflow-avx512",
  51. "tensorflow-rocm",
  52. "tensorflow-macos",
  53. ),
  54. "torch": {"torch"},
  55. }
  56. # Check once at runtime
  57. for candidate_name, package_names in _CANDIDATES.items():
  58. _package_versions[candidate_name] = "N/A"
  59. for name in package_names:
  60. try:
  61. _package_versions[candidate_name] = importlib.metadata.version(name)
  62. break
  63. except importlib.metadata.PackageNotFoundError:
  64. pass
  65. def _get_version(package_name: str) -> str:
  66. return _package_versions.get(package_name, "N/A")
  67. def is_package_available(package_name: str) -> bool:
  68. return _get_version(package_name) != "N/A"
  69. # Python
  70. def get_python_version() -> str:
  71. return _PY_VERSION
  72. # Huggingface Hub
  73. def get_hf_hub_version() -> str:
  74. return __version__
  75. # aiohttp
  76. def is_aiohttp_available() -> bool:
  77. return is_package_available("aiohttp")
  78. def get_aiohttp_version() -> str:
  79. return _get_version("aiohttp")
  80. # FastAI
  81. def is_fastai_available() -> bool:
  82. return is_package_available("fastai")
  83. def get_fastai_version() -> str:
  84. return _get_version("fastai")
  85. # FastAPI
  86. def is_fastapi_available() -> bool:
  87. return is_package_available("fastapi")
  88. def get_fastapi_version() -> str:
  89. return _get_version("fastapi")
  90. # Fastcore
  91. def is_fastcore_available() -> bool:
  92. return is_package_available("fastcore")
  93. def get_fastcore_version() -> str:
  94. return _get_version("fastcore")
  95. # FastAI
  96. def is_gradio_available() -> bool:
  97. return is_package_available("gradio")
  98. def get_gradio_version() -> str:
  99. return _get_version("gradio")
  100. # Graphviz
  101. def is_graphviz_available() -> bool:
  102. return is_package_available("graphviz")
  103. def get_graphviz_version() -> str:
  104. return _get_version("graphviz")
  105. # hf_transfer
  106. def is_hf_transfer_available() -> bool:
  107. return is_package_available("hf_transfer")
  108. def get_hf_transfer_version() -> str:
  109. return _get_version("hf_transfer")
  110. # xet
  111. def is_xet_available() -> bool:
  112. # since hf_xet is automatically used if available, allow explicit disabling via environment variable
  113. if constants.HF_HUB_DISABLE_XET:
  114. return False
  115. return is_package_available("hf_xet")
  116. def get_xet_version() -> str:
  117. return _get_version("hf_xet")
  118. # keras
  119. def is_keras_available() -> bool:
  120. return is_package_available("keras")
  121. def get_keras_version() -> str:
  122. return _get_version("keras")
  123. # Numpy
  124. def is_numpy_available() -> bool:
  125. return is_package_available("numpy")
  126. def get_numpy_version() -> str:
  127. return _get_version("numpy")
  128. # Jinja
  129. def is_jinja_available() -> bool:
  130. return is_package_available("jinja")
  131. def get_jinja_version() -> str:
  132. return _get_version("jinja")
  133. # Pillow
  134. def is_pillow_available() -> bool:
  135. return is_package_available("pillow")
  136. def get_pillow_version() -> str:
  137. return _get_version("pillow")
  138. # Pydantic
  139. def is_pydantic_available() -> bool:
  140. if not is_package_available("pydantic"):
  141. return False
  142. # For Pydantic, we add an extra check to test whether it is correctly installed or not. If both pydantic 2.x and
  143. # typing_extensions<=4.5.0 are installed, then pydantic will fail at import time. This should not happen when
  144. # it is installed with `pip install huggingface_hub[inference]` but it can happen when it is installed manually
  145. # by the user in an environment that we don't control.
  146. #
  147. # Usually we won't need to do this kind of check on optional dependencies. However, pydantic is a special case
  148. # as it is automatically imported when doing `from huggingface_hub import ...` even if the user doesn't use it.
  149. #
  150. # See https://github.com/huggingface/huggingface_hub/pull/1829 for more details.
  151. try:
  152. from pydantic import validator # noqa: F401
  153. except ImportError:
  154. # Example: "ImportError: cannot import name 'TypeAliasType' from 'typing_extensions'"
  155. warnings.warn(
  156. "Pydantic is installed but cannot be imported. Please check your installation. `huggingface_hub` will "
  157. "default to not using Pydantic. Error message: '{e}'"
  158. )
  159. return False
  160. return True
  161. def get_pydantic_version() -> str:
  162. return _get_version("pydantic")
  163. # Pydot
  164. def is_pydot_available() -> bool:
  165. return is_package_available("pydot")
  166. def get_pydot_version() -> str:
  167. return _get_version("pydot")
  168. # Tensorboard
  169. def is_tensorboard_available() -> bool:
  170. return is_package_available("tensorboard")
  171. def get_tensorboard_version() -> str:
  172. return _get_version("tensorboard")
  173. # Tensorflow
  174. def is_tf_available() -> bool:
  175. return is_package_available("tensorflow")
  176. def get_tf_version() -> str:
  177. return _get_version("tensorflow")
  178. # Torch
  179. def is_torch_available() -> bool:
  180. return is_package_available("torch")
  181. def get_torch_version() -> str:
  182. return _get_version("torch")
  183. # Safetensors
  184. def is_safetensors_available() -> bool:
  185. return is_package_available("safetensors")
  186. # Shell-related helpers
  187. try:
  188. # Set to `True` if script is running in a Google Colab notebook.
  189. # If running in Google Colab, git credential store is set globally which makes the
  190. # warning disappear. See https://github.com/huggingface/huggingface_hub/issues/1043
  191. #
  192. # Taken from https://stackoverflow.com/a/63519730.
  193. _is_google_colab = "google.colab" in str(get_ipython()) # type: ignore # noqa: F821
  194. except NameError:
  195. _is_google_colab = False
  196. def is_notebook() -> bool:
  197. """Return `True` if code is executed in a notebook (Jupyter, Colab, QTconsole).
  198. Taken from https://stackoverflow.com/a/39662359.
  199. Adapted to make it work with Google colab as well.
  200. """
  201. try:
  202. shell_class = get_ipython().__class__ # type: ignore # noqa: F821
  203. for parent_class in shell_class.__mro__: # e.g. "is subclass of"
  204. if parent_class.__name__ == "ZMQInteractiveShell":
  205. return True # Jupyter notebook, Google colab or qtconsole
  206. return False
  207. except NameError:
  208. return False # Probably standard Python interpreter
  209. def is_google_colab() -> bool:
  210. """Return `True` if code is executed in a Google colab.
  211. Taken from https://stackoverflow.com/a/63519730.
  212. """
  213. return _is_google_colab
  214. def is_colab_enterprise() -> bool:
  215. """Return `True` if code is executed in a Google Colab Enterprise environment."""
  216. return os.environ.get("VERTEX_PRODUCT") == "COLAB_ENTERPRISE"
  217. def dump_environment_info() -> Dict[str, Any]:
  218. """Dump information about the machine to help debugging issues.
  219. Similar helper exist in:
  220. - `datasets` (https://github.com/huggingface/datasets/blob/main/src/datasets/commands/env.py)
  221. - `diffusers` (https://github.com/huggingface/diffusers/blob/main/src/diffusers/commands/env.py)
  222. - `transformers` (https://github.com/huggingface/transformers/blob/main/src/transformers/commands/env.py)
  223. """
  224. from huggingface_hub import get_token, whoami
  225. from huggingface_hub.utils import list_credential_helpers
  226. token = get_token()
  227. # Generic machine info
  228. info: Dict[str, Any] = {
  229. "huggingface_hub version": get_hf_hub_version(),
  230. "Platform": platform.platform(),
  231. "Python version": get_python_version(),
  232. }
  233. # Interpreter info
  234. try:
  235. shell_class = get_ipython().__class__ # type: ignore # noqa: F821
  236. info["Running in iPython ?"] = "Yes"
  237. info["iPython shell"] = shell_class.__name__
  238. except NameError:
  239. info["Running in iPython ?"] = "No"
  240. info["Running in notebook ?"] = "Yes" if is_notebook() else "No"
  241. info["Running in Google Colab ?"] = "Yes" if is_google_colab() else "No"
  242. info["Running in Google Colab Enterprise ?"] = "Yes" if is_colab_enterprise() else "No"
  243. # Login info
  244. info["Token path ?"] = constants.HF_TOKEN_PATH
  245. info["Has saved token ?"] = token is not None
  246. if token is not None:
  247. try:
  248. info["Who am I ?"] = whoami()["name"]
  249. except Exception:
  250. pass
  251. try:
  252. info["Configured git credential helpers"] = ", ".join(list_credential_helpers())
  253. except Exception:
  254. pass
  255. # Installed dependencies
  256. info["FastAI"] = get_fastai_version()
  257. info["Tensorflow"] = get_tf_version()
  258. info["Torch"] = get_torch_version()
  259. info["Jinja2"] = get_jinja_version()
  260. info["Graphviz"] = get_graphviz_version()
  261. info["keras"] = get_keras_version()
  262. info["Pydot"] = get_pydot_version()
  263. info["Pillow"] = get_pillow_version()
  264. info["hf_transfer"] = get_hf_transfer_version()
  265. info["gradio"] = get_gradio_version()
  266. info["tensorboard"] = get_tensorboard_version()
  267. info["numpy"] = get_numpy_version()
  268. info["pydantic"] = get_pydantic_version()
  269. info["aiohttp"] = get_aiohttp_version()
  270. info["hf_xet"] = get_xet_version()
  271. # Environment variables
  272. info["ENDPOINT"] = constants.ENDPOINT
  273. info["HF_HUB_CACHE"] = constants.HF_HUB_CACHE
  274. info["HF_ASSETS_CACHE"] = constants.HF_ASSETS_CACHE
  275. info["HF_TOKEN_PATH"] = constants.HF_TOKEN_PATH
  276. info["HF_STORED_TOKENS_PATH"] = constants.HF_STORED_TOKENS_PATH
  277. info["HF_HUB_OFFLINE"] = constants.HF_HUB_OFFLINE
  278. info["HF_HUB_DISABLE_TELEMETRY"] = constants.HF_HUB_DISABLE_TELEMETRY
  279. info["HF_HUB_DISABLE_PROGRESS_BARS"] = constants.HF_HUB_DISABLE_PROGRESS_BARS
  280. info["HF_HUB_DISABLE_SYMLINKS_WARNING"] = constants.HF_HUB_DISABLE_SYMLINKS_WARNING
  281. info["HF_HUB_DISABLE_EXPERIMENTAL_WARNING"] = constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING
  282. info["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = constants.HF_HUB_DISABLE_IMPLICIT_TOKEN
  283. info["HF_HUB_DISABLE_XET"] = constants.HF_HUB_DISABLE_XET
  284. info["HF_HUB_ENABLE_HF_TRANSFER"] = constants.HF_HUB_ENABLE_HF_TRANSFER
  285. info["HF_HUB_ETAG_TIMEOUT"] = constants.HF_HUB_ETAG_TIMEOUT
  286. info["HF_HUB_DOWNLOAD_TIMEOUT"] = constants.HF_HUB_DOWNLOAD_TIMEOUT
  287. print("\nCopy-and-paste the text below in your GitHub issue.\n")
  288. print("\n".join([f"- {prop}: {val}" for prop, val in info.items()]) + "\n")
  289. return info