jobs.py 43 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100
  1. # Copyright 2025 The HuggingFace Team. All rights reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. """Contains commands to interact with jobs on the Hugging Face Hub.
  15. Usage:
  16. # run a job
  17. hf jobs run <image> <command>
  18. # List running or completed jobs
  19. hf jobs ps [-a] [-f key=value] [--format TEMPLATE]
  20. # Stream logs from a job
  21. hf jobs logs <job-id>
  22. # Inspect detailed information about a job
  23. hf jobs inspect <job-id>
  24. # Cancel a running job
  25. hf jobs cancel <job-id>
  26. """
  27. import json
  28. import os
  29. import re
  30. from argparse import Namespace, _SubParsersAction
  31. from dataclasses import asdict
  32. from pathlib import Path
  33. from typing import Dict, List, Optional, Union
  34. import requests
  35. from huggingface_hub import HfApi, SpaceHardware, get_token
  36. from huggingface_hub.utils import logging
  37. from huggingface_hub.utils._dotenv import load_dotenv
  38. from . import BaseHuggingfaceCLICommand
  39. logger = logging.get_logger(__name__)
  40. SUGGESTED_FLAVORS = [item.value for item in SpaceHardware if item.value != "zero-a10g"]
  41. class JobsCommands(BaseHuggingfaceCLICommand):
  42. @staticmethod
  43. def register_subcommand(parser: _SubParsersAction):
  44. jobs_parser = parser.add_parser("jobs", help="Run and manage Jobs on the Hub.")
  45. jobs_subparsers = jobs_parser.add_subparsers(help="huggingface.co jobs related commands")
  46. # Show help if no subcommand is provided
  47. jobs_parser.set_defaults(func=lambda args: jobs_parser.print_help())
  48. # Register commands
  49. InspectCommand.register_subcommand(jobs_subparsers)
  50. LogsCommand.register_subcommand(jobs_subparsers)
  51. PsCommand.register_subcommand(jobs_subparsers)
  52. RunCommand.register_subcommand(jobs_subparsers)
  53. CancelCommand.register_subcommand(jobs_subparsers)
  54. UvCommand.register_subcommand(jobs_subparsers)
  55. ScheduledJobsCommands.register_subcommand(jobs_subparsers)
  56. class RunCommand(BaseHuggingfaceCLICommand):
  57. @staticmethod
  58. def register_subcommand(parser: _SubParsersAction) -> None:
  59. run_parser = parser.add_parser("run", help="Run a Job")
  60. run_parser.add_argument("image", type=str, help="The Docker image to use.")
  61. run_parser.add_argument("-e", "--env", action="append", help="Set environment variables. E.g. --env ENV=value")
  62. run_parser.add_argument(
  63. "-s",
  64. "--secrets",
  65. action="append",
  66. help=(
  67. "Set secret environment variables. E.g. --secrets SECRET=value "
  68. "or `--secrets HF_TOKEN` to pass your Hugging Face token."
  69. ),
  70. )
  71. run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.")
  72. run_parser.add_argument("--secrets-file", type=str, help="Read in a file of secret environment variables.")
  73. run_parser.add_argument(
  74. "--flavor",
  75. type=str,
  76. help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.",
  77. )
  78. run_parser.add_argument(
  79. "--timeout",
  80. type=str,
  81. help="Max duration: int/float with s (seconds, default), m (minutes), h (hours) or d (days).",
  82. )
  83. run_parser.add_argument(
  84. "-d",
  85. "--detach",
  86. action="store_true",
  87. help="Run the Job in the background and print the Job ID.",
  88. )
  89. run_parser.add_argument(
  90. "--namespace",
  91. type=str,
  92. help="The namespace where the Job will be created. Defaults to the current user's namespace.",
  93. )
  94. run_parser.add_argument(
  95. "--token",
  96. type=str,
  97. help="A User Access Token generated from https://huggingface.co/settings/tokens",
  98. )
  99. run_parser.add_argument("command", nargs="...", help="The command to run.")
  100. run_parser.set_defaults(func=RunCommand)
  101. def __init__(self, args: Namespace) -> None:
  102. self.image: str = args.image
  103. self.command: List[str] = args.command
  104. self.env: dict[str, Optional[str]] = {}
  105. if args.env_file:
  106. self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy()))
  107. for env_value in args.env or []:
  108. self.env.update(load_dotenv(env_value, environ=os.environ.copy()))
  109. self.secrets: dict[str, Optional[str]] = {}
  110. extended_environ = _get_extended_environ()
  111. if args.secrets_file:
  112. self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ))
  113. for secret in args.secrets or []:
  114. self.secrets.update(load_dotenv(secret, environ=extended_environ))
  115. self.flavor: Optional[SpaceHardware] = args.flavor
  116. self.timeout: Optional[str] = args.timeout
  117. self.detach: bool = args.detach
  118. self.namespace: Optional[str] = args.namespace
  119. self.token: Optional[str] = args.token
  120. def run(self) -> None:
  121. api = HfApi(token=self.token)
  122. job = api.run_job(
  123. image=self.image,
  124. command=self.command,
  125. env=self.env,
  126. secrets=self.secrets,
  127. flavor=self.flavor,
  128. timeout=self.timeout,
  129. namespace=self.namespace,
  130. )
  131. # Always print the job ID to the user
  132. print(f"Job started with ID: {job.id}")
  133. print(f"View at: {job.url}")
  134. if self.detach:
  135. return
  136. # Now let's stream the logs
  137. for log in api.fetch_job_logs(job_id=job.id):
  138. print(log)
  139. class LogsCommand(BaseHuggingfaceCLICommand):
  140. @staticmethod
  141. def register_subcommand(parser: _SubParsersAction) -> None:
  142. run_parser = parser.add_parser("logs", help="Fetch the logs of a Job")
  143. run_parser.add_argument("job_id", type=str, help="Job ID")
  144. run_parser.add_argument(
  145. "--namespace",
  146. type=str,
  147. help="The namespace where the job is running. Defaults to the current user's namespace.",
  148. )
  149. run_parser.add_argument(
  150. "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
  151. )
  152. run_parser.set_defaults(func=LogsCommand)
  153. def __init__(self, args: Namespace) -> None:
  154. self.job_id: str = args.job_id
  155. self.namespace: Optional[str] = args.namespace
  156. self.token: Optional[str] = args.token
  157. def run(self) -> None:
  158. api = HfApi(token=self.token)
  159. for log in api.fetch_job_logs(job_id=self.job_id, namespace=self.namespace):
  160. print(log)
  161. def _tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str:
  162. """
  163. Inspired by:
  164. - stackoverflow.com/a/8356620/593036
  165. - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data
  166. """
  167. col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)]
  168. terminal_width = max(os.get_terminal_size().columns, len(headers) * 12)
  169. while len(headers) + sum(col_widths) > terminal_width:
  170. col_to_minimize = col_widths.index(max(col_widths))
  171. col_widths[col_to_minimize] //= 2
  172. if len(headers) + sum(col_widths) <= terminal_width:
  173. col_widths[col_to_minimize] = terminal_width - sum(col_widths) - len(headers) + col_widths[col_to_minimize]
  174. row_format = ("{{:{}}} " * len(headers)).format(*col_widths)
  175. lines = []
  176. lines.append(row_format.format(*headers))
  177. lines.append(row_format.format(*["-" * w for w in col_widths]))
  178. for row in rows:
  179. row_format_args = [
  180. str(x)[: col_width - 3] + "..." if len(str(x)) > col_width else str(x)
  181. for x, col_width in zip(row, col_widths)
  182. ]
  183. lines.append(row_format.format(*row_format_args))
  184. return "\n".join(lines)
  185. class PsCommand(BaseHuggingfaceCLICommand):
  186. @staticmethod
  187. def register_subcommand(parser: _SubParsersAction) -> None:
  188. run_parser = parser.add_parser("ps", help="List Jobs")
  189. run_parser.add_argument(
  190. "-a",
  191. "--all",
  192. action="store_true",
  193. help="Show all Jobs (default shows just running)",
  194. )
  195. run_parser.add_argument(
  196. "--namespace",
  197. type=str,
  198. help="The namespace from where it lists the jobs. Defaults to the current user's namespace.",
  199. )
  200. run_parser.add_argument(
  201. "--token",
  202. type=str,
  203. help="A User Access Token generated from https://huggingface.co/settings/tokens",
  204. )
  205. # Add Docker-style filtering argument
  206. run_parser.add_argument(
  207. "-f",
  208. "--filter",
  209. action="append",
  210. default=[],
  211. help="Filter output based on conditions provided (format: key=value)",
  212. )
  213. # Add option to format output
  214. run_parser.add_argument(
  215. "--format",
  216. type=str,
  217. help="Format output using a custom template",
  218. )
  219. run_parser.set_defaults(func=PsCommand)
  220. def __init__(self, args: Namespace) -> None:
  221. self.all: bool = args.all
  222. self.namespace: Optional[str] = args.namespace
  223. self.token: Optional[str] = args.token
  224. self.format: Optional[str] = args.format
  225. self.filters: Dict[str, str] = {}
  226. # Parse filter arguments (key=value pairs)
  227. for f in args.filter:
  228. if "=" in f:
  229. key, value = f.split("=", 1)
  230. self.filters[key.lower()] = value
  231. else:
  232. print(f"Warning: Ignoring invalid filter format '{f}'. Use key=value format.")
  233. def run(self) -> None:
  234. """
  235. Fetch and display job information for the current user.
  236. Uses Docker-style filtering with -f/--filter flag and key=value pairs.
  237. """
  238. try:
  239. api = HfApi(token=self.token)
  240. # Fetch jobs data
  241. jobs = api.list_jobs(namespace=self.namespace)
  242. # Define table headers
  243. table_headers = ["JOB ID", "IMAGE/SPACE", "COMMAND", "CREATED", "STATUS"]
  244. # Process jobs data
  245. rows = []
  246. for job in jobs:
  247. # Extract job data for filtering
  248. status = job.status.stage if job.status else "UNKNOWN"
  249. # Skip job if not all jobs should be shown and status doesn't match criteria
  250. if not self.all and status not in ("RUNNING", "UPDATING"):
  251. continue
  252. # Extract job ID
  253. job_id = job.id
  254. # Extract image or space information
  255. image_or_space = job.docker_image or "N/A"
  256. # Extract and format command
  257. command = job.command or []
  258. command_str = " ".join(command) if command else "N/A"
  259. # Extract creation time
  260. created_at = job.created_at.strftime("%Y-%m-%d %H:%M:%S") if job.created_at else "N/A"
  261. # Create a dict with all job properties for filtering
  262. job_properties = {
  263. "id": job_id,
  264. "image": image_or_space,
  265. "status": status.lower(),
  266. "command": command_str,
  267. }
  268. # Check if job matches all filters
  269. if not self._matches_filters(job_properties):
  270. continue
  271. # Create row
  272. rows.append([job_id, image_or_space, command_str, created_at, status])
  273. # Handle empty results
  274. if not rows:
  275. filters_msg = ""
  276. if self.filters:
  277. filters_msg = f" matching filters: {', '.join([f'{k}={v}' for k, v in self.filters.items()])}"
  278. print(f"No jobs found{filters_msg}")
  279. return
  280. # Apply custom format if provided or use default tabular format
  281. self._print_output(rows, table_headers)
  282. except requests.RequestException as e:
  283. print(f"Error fetching jobs data: {e}")
  284. except (KeyError, ValueError, TypeError) as e:
  285. print(f"Error processing jobs data: {e}")
  286. except Exception as e:
  287. print(f"Unexpected error - {type(e).__name__}: {e}")
  288. def _matches_filters(self, job_properties: Dict[str, str]) -> bool:
  289. """Check if job matches all specified filters."""
  290. for key, pattern in self.filters.items():
  291. # Check if property exists
  292. if key not in job_properties:
  293. return False
  294. # Support pattern matching with wildcards
  295. if "*" in pattern or "?" in pattern:
  296. # Convert glob pattern to regex
  297. regex_pattern = pattern.replace("*", ".*").replace("?", ".")
  298. if not re.search(f"^{regex_pattern}$", job_properties[key], re.IGNORECASE):
  299. return False
  300. # Simple substring matching
  301. elif pattern.lower() not in job_properties[key].lower():
  302. return False
  303. return True
  304. def _print_output(self, rows, headers):
  305. """Print output according to the chosen format."""
  306. if self.format:
  307. # Custom template formatting (simplified)
  308. template = self.format
  309. for row in rows:
  310. line = template
  311. for i, field in enumerate(["id", "image", "command", "created", "status"]):
  312. placeholder = f"{{{{.{field}}}}}"
  313. if placeholder in line:
  314. line = line.replace(placeholder, str(row[i]))
  315. print(line)
  316. else:
  317. # Default tabular format
  318. print(
  319. _tabulate(
  320. rows,
  321. headers=headers,
  322. )
  323. )
  324. class InspectCommand(BaseHuggingfaceCLICommand):
  325. @staticmethod
  326. def register_subcommand(parser: _SubParsersAction) -> None:
  327. run_parser = parser.add_parser("inspect", help="Display detailed information on one or more Jobs")
  328. run_parser.add_argument(
  329. "--namespace",
  330. type=str,
  331. help="The namespace where the job is running. Defaults to the current user's namespace.",
  332. )
  333. run_parser.add_argument(
  334. "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
  335. )
  336. run_parser.add_argument("job_ids", nargs="...", help="The jobs to inspect")
  337. run_parser.set_defaults(func=InspectCommand)
  338. def __init__(self, args: Namespace) -> None:
  339. self.namespace: Optional[str] = args.namespace
  340. self.token: Optional[str] = args.token
  341. self.job_ids: List[str] = args.job_ids
  342. def run(self) -> None:
  343. api = HfApi(token=self.token)
  344. jobs = [api.inspect_job(job_id=job_id, namespace=self.namespace) for job_id in self.job_ids]
  345. print(json.dumps([asdict(job) for job in jobs], indent=4, default=str))
  346. class CancelCommand(BaseHuggingfaceCLICommand):
  347. @staticmethod
  348. def register_subcommand(parser: _SubParsersAction) -> None:
  349. run_parser = parser.add_parser("cancel", help="Cancel a Job")
  350. run_parser.add_argument("job_id", type=str, help="Job ID")
  351. run_parser.add_argument(
  352. "--namespace",
  353. type=str,
  354. help="The namespace where the job is running. Defaults to the current user's namespace.",
  355. )
  356. run_parser.add_argument(
  357. "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
  358. )
  359. run_parser.set_defaults(func=CancelCommand)
  360. def __init__(self, args: Namespace) -> None:
  361. self.job_id: str = args.job_id
  362. self.namespace = args.namespace
  363. self.token: Optional[str] = args.token
  364. def run(self) -> None:
  365. api = HfApi(token=self.token)
  366. api.cancel_job(job_id=self.job_id, namespace=self.namespace)
  367. class UvCommand(BaseHuggingfaceCLICommand):
  368. """Run UV scripts on Hugging Face infrastructure."""
  369. @staticmethod
  370. def register_subcommand(parser):
  371. """Register UV run subcommand."""
  372. uv_parser = parser.add_parser(
  373. "uv",
  374. help="Run UV scripts (Python with inline dependencies) on HF infrastructure",
  375. )
  376. subparsers = uv_parser.add_subparsers(dest="uv_command", help="UV commands", required=True)
  377. # Run command only
  378. run_parser = subparsers.add_parser(
  379. "run",
  380. help="Run a UV script (local file or URL) on HF infrastructure",
  381. )
  382. run_parser.add_argument("script", help="UV script to run (local file or URL)")
  383. run_parser.add_argument("script_args", nargs="...", help="Arguments for the script", default=[])
  384. run_parser.add_argument("--image", type=str, help="Use a custom Docker image with `uv` installed.")
  385. run_parser.add_argument(
  386. "--repo",
  387. help="Repository name for the script (creates ephemeral if not specified)",
  388. )
  389. run_parser.add_argument(
  390. "--flavor",
  391. type=str,
  392. help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.",
  393. )
  394. run_parser.add_argument("-e", "--env", action="append", help="Environment variables")
  395. run_parser.add_argument(
  396. "-s",
  397. "--secrets",
  398. action="append",
  399. help=(
  400. "Set secret environment variables. E.g. --secrets SECRET=value "
  401. "or `--secrets HF_TOKEN` to pass your Hugging Face token."
  402. ),
  403. )
  404. run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.")
  405. run_parser.add_argument(
  406. "--secrets-file",
  407. type=str,
  408. help="Read in a file of secret environment variables.",
  409. )
  410. run_parser.add_argument("--timeout", type=str, help="Max duration (e.g., 30s, 5m, 1h)")
  411. run_parser.add_argument("-d", "--detach", action="store_true", help="Run in background")
  412. run_parser.add_argument(
  413. "--namespace",
  414. type=str,
  415. help="The namespace where the Job will be created. Defaults to the current user's namespace.",
  416. )
  417. run_parser.add_argument("--token", type=str, help="HF token")
  418. # UV options
  419. run_parser.add_argument("--with", action="append", help="Run with the given packages installed", dest="with_")
  420. run_parser.add_argument(
  421. "-p", "--python", type=str, help="The Python interpreter to use for the run environment"
  422. )
  423. run_parser.set_defaults(func=UvCommand)
  424. def __init__(self, args: Namespace) -> None:
  425. """Initialize the command with parsed arguments."""
  426. self.script = args.script
  427. self.script_args = args.script_args
  428. self.dependencies = args.with_
  429. self.python = args.python
  430. self.image = args.image
  431. self.env: dict[str, Optional[str]] = {}
  432. if args.env_file:
  433. self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy()))
  434. for env_value in args.env or []:
  435. self.env.update(load_dotenv(env_value, environ=os.environ.copy()))
  436. self.secrets: dict[str, Optional[str]] = {}
  437. extended_environ = _get_extended_environ()
  438. if args.secrets_file:
  439. self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ))
  440. for secret in args.secrets or []:
  441. self.secrets.update(load_dotenv(secret, environ=extended_environ))
  442. self.flavor: Optional[SpaceHardware] = args.flavor
  443. self.timeout: Optional[str] = args.timeout
  444. self.detach: bool = args.detach
  445. self.namespace: Optional[str] = args.namespace
  446. self.token: Optional[str] = args.token
  447. self._repo = args.repo
  448. def run(self) -> None:
  449. """Execute UV command."""
  450. logging.set_verbosity(logging.INFO)
  451. api = HfApi(token=self.token)
  452. job = api.run_uv_job(
  453. script=self.script,
  454. script_args=self.script_args,
  455. dependencies=self.dependencies,
  456. python=self.python,
  457. image=self.image,
  458. env=self.env,
  459. secrets=self.secrets,
  460. flavor=self.flavor,
  461. timeout=self.timeout,
  462. namespace=self.namespace,
  463. _repo=self._repo,
  464. )
  465. # Always print the job ID to the user
  466. print(f"Job started with ID: {job.id}")
  467. print(f"View at: {job.url}")
  468. if self.detach:
  469. return
  470. # Now let's stream the logs
  471. for log in api.fetch_job_logs(job_id=job.id):
  472. print(log)
  473. def _get_extended_environ() -> Dict[str, str]:
  474. extended_environ = os.environ.copy()
  475. if (token := get_token()) is not None:
  476. extended_environ["HF_TOKEN"] = token
  477. return extended_environ
  478. class ScheduledJobsCommands(BaseHuggingfaceCLICommand):
  479. @staticmethod
  480. def register_subcommand(parser: _SubParsersAction):
  481. scheduled_jobs_parser = parser.add_parser("scheduled", help="Create and manage scheduled Jobs on the Hub.")
  482. scheduled_jobs_subparsers = scheduled_jobs_parser.add_subparsers(
  483. help="huggingface.co scheduled jobs related commands"
  484. )
  485. # Show help if no subcommand is provided
  486. scheduled_jobs_parser.set_defaults(func=lambda args: scheduled_jobs_subparsers.print_help())
  487. # Register commands
  488. ScheduledRunCommand.register_subcommand(scheduled_jobs_subparsers)
  489. ScheduledPsCommand.register_subcommand(scheduled_jobs_subparsers)
  490. ScheduledInspectCommand.register_subcommand(scheduled_jobs_subparsers)
  491. ScheduledDeleteCommand.register_subcommand(scheduled_jobs_subparsers)
  492. ScheduledSuspendCommand.register_subcommand(scheduled_jobs_subparsers)
  493. ScheduledResumeCommand.register_subcommand(scheduled_jobs_subparsers)
  494. ScheduledUvCommand.register_subcommand(scheduled_jobs_subparsers)
  495. class ScheduledRunCommand(BaseHuggingfaceCLICommand):
  496. @staticmethod
  497. def register_subcommand(parser: _SubParsersAction) -> None:
  498. run_parser = parser.add_parser("run", help="Schedule a Job")
  499. run_parser.add_argument(
  500. "schedule",
  501. type=str,
  502. help="One of annually, yearly, monthly, weekly, daily, hourly, or a CRON schedule expression.",
  503. )
  504. run_parser.add_argument("image", type=str, help="The Docker image to use.")
  505. run_parser.add_argument(
  506. "--suspend",
  507. action="store_true",
  508. help="Suspend (pause) the scheduled Job",
  509. default=None,
  510. )
  511. run_parser.add_argument(
  512. "--concurrency",
  513. action="store_true",
  514. help="Allow multiple instances of this Job to run concurrently",
  515. default=None,
  516. )
  517. run_parser.add_argument("-e", "--env", action="append", help="Set environment variables. E.g. --env ENV=value")
  518. run_parser.add_argument(
  519. "-s",
  520. "--secrets",
  521. action="append",
  522. help=(
  523. "Set secret environment variables. E.g. --secrets SECRET=value "
  524. "or `--secrets HF_TOKEN` to pass your Hugging Face token."
  525. ),
  526. )
  527. run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.")
  528. run_parser.add_argument("--secrets-file", type=str, help="Read in a file of secret environment variables.")
  529. run_parser.add_argument(
  530. "--flavor",
  531. type=str,
  532. help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.",
  533. )
  534. run_parser.add_argument(
  535. "--timeout",
  536. type=str,
  537. help="Max duration: int/float with s (seconds, default), m (minutes), h (hours) or d (days).",
  538. )
  539. run_parser.add_argument(
  540. "--namespace",
  541. type=str,
  542. help="The namespace where the scheduled Job will be created. Defaults to the current user's namespace.",
  543. )
  544. run_parser.add_argument(
  545. "--token",
  546. type=str,
  547. help="A User Access Token generated from https://huggingface.co/settings/tokens",
  548. )
  549. run_parser.add_argument("command", nargs="...", help="The command to run.")
  550. run_parser.set_defaults(func=ScheduledRunCommand)
  551. def __init__(self, args: Namespace) -> None:
  552. self.schedule: str = args.schedule
  553. self.image: str = args.image
  554. self.command: List[str] = args.command
  555. self.suspend: Optional[bool] = args.suspend
  556. self.concurrency: Optional[bool] = args.concurrency
  557. self.env: dict[str, Optional[str]] = {}
  558. if args.env_file:
  559. self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy()))
  560. for env_value in args.env or []:
  561. self.env.update(load_dotenv(env_value, environ=os.environ.copy()))
  562. self.secrets: dict[str, Optional[str]] = {}
  563. extended_environ = _get_extended_environ()
  564. if args.secrets_file:
  565. self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ))
  566. for secret in args.secrets or []:
  567. self.secrets.update(load_dotenv(secret, environ=extended_environ))
  568. self.flavor: Optional[SpaceHardware] = args.flavor
  569. self.timeout: Optional[str] = args.timeout
  570. self.namespace: Optional[str] = args.namespace
  571. self.token: Optional[str] = args.token
  572. def run(self) -> None:
  573. api = HfApi(token=self.token)
  574. scheduled_job = api.create_scheduled_job(
  575. image=self.image,
  576. command=self.command,
  577. schedule=self.schedule,
  578. suspend=self.suspend,
  579. concurrency=self.concurrency,
  580. env=self.env,
  581. secrets=self.secrets,
  582. flavor=self.flavor,
  583. timeout=self.timeout,
  584. namespace=self.namespace,
  585. )
  586. # Always print the scheduled job ID to the user
  587. print(f"Scheduled Job created with ID: {scheduled_job.id}")
  588. class ScheduledPsCommand(BaseHuggingfaceCLICommand):
  589. @staticmethod
  590. def register_subcommand(parser: _SubParsersAction) -> None:
  591. run_parser = parser.add_parser("ps", help="List scheduled Jobs")
  592. run_parser.add_argument(
  593. "-a",
  594. "--all",
  595. action="store_true",
  596. help="Show all scheduled Jobs (default hides suspended)",
  597. )
  598. run_parser.add_argument(
  599. "--namespace",
  600. type=str,
  601. help="The namespace from where it lists the jobs. Defaults to the current user's namespace.",
  602. )
  603. run_parser.add_argument(
  604. "--token",
  605. type=str,
  606. help="A User Access Token generated from https://huggingface.co/settings/tokens",
  607. )
  608. # Add Docker-style filtering argument
  609. run_parser.add_argument(
  610. "-f",
  611. "--filter",
  612. action="append",
  613. default=[],
  614. help="Filter output based on conditions provided (format: key=value)",
  615. )
  616. # Add option to format output
  617. run_parser.add_argument(
  618. "--format",
  619. type=str,
  620. help="Format output using a custom template",
  621. )
  622. run_parser.set_defaults(func=ScheduledPsCommand)
  623. def __init__(self, args: Namespace) -> None:
  624. self.all: bool = args.all
  625. self.namespace: Optional[str] = args.namespace
  626. self.token: Optional[str] = args.token
  627. self.format: Optional[str] = args.format
  628. self.filters: Dict[str, str] = {}
  629. # Parse filter arguments (key=value pairs)
  630. for f in args.filter:
  631. if "=" in f:
  632. key, value = f.split("=", 1)
  633. self.filters[key.lower()] = value
  634. else:
  635. print(f"Warning: Ignoring invalid filter format '{f}'. Use key=value format.")
  636. def run(self) -> None:
  637. """
  638. Fetch and display scheduked job information for the current user.
  639. Uses Docker-style filtering with -f/--filter flag and key=value pairs.
  640. """
  641. try:
  642. api = HfApi(token=self.token)
  643. # Fetch jobs data
  644. scheduled_jobs = api.list_scheduled_jobs(namespace=self.namespace)
  645. # Define table headers
  646. table_headers = [
  647. "ID",
  648. "SCHEDULE",
  649. "IMAGE/SPACE",
  650. "COMMAND",
  651. "LAST RUN",
  652. "NEXT RUN",
  653. "SUSPEND",
  654. ]
  655. # Process jobs data
  656. rows = []
  657. for scheduled_job in scheduled_jobs:
  658. # Extract job data for filtering
  659. suspend = scheduled_job.suspend
  660. # Skip job if not all jobs should be shown and status doesn't match criteria
  661. if not self.all and suspend:
  662. continue
  663. # Extract job ID
  664. scheduled_job_id = scheduled_job.id
  665. # Extract schedule
  666. schedule = scheduled_job.schedule
  667. # Extract image or space information
  668. image_or_space = scheduled_job.job_spec.docker_image or "N/A"
  669. # Extract and format command
  670. command = scheduled_job.job_spec.command or []
  671. command_str = " ".join(command) if command else "N/A"
  672. # Extract status
  673. last_job_at = (
  674. scheduled_job.status.last_job.at.strftime("%Y-%m-%d %H:%M:%S")
  675. if scheduled_job.status.last_job
  676. else "N/A"
  677. )
  678. next_job_run_at = (
  679. scheduled_job.status.next_job_run_at.strftime("%Y-%m-%d %H:%M:%S")
  680. if scheduled_job.status.next_job_run_at
  681. else "N/A"
  682. )
  683. # Create a dict with all job properties for filtering
  684. job_properties = {
  685. "id": scheduled_job_id,
  686. "image": image_or_space,
  687. "suspend": str(suspend),
  688. "command": command_str,
  689. }
  690. # Check if job matches all filters
  691. if not self._matches_filters(job_properties):
  692. continue
  693. # Create row
  694. rows.append(
  695. [
  696. scheduled_job_id,
  697. schedule,
  698. image_or_space,
  699. command_str,
  700. last_job_at,
  701. next_job_run_at,
  702. suspend,
  703. ]
  704. )
  705. # Handle empty results
  706. if not rows:
  707. filters_msg = ""
  708. if self.filters:
  709. filters_msg = f" matching filters: {', '.join([f'{k}={v}' for k, v in self.filters.items()])}"
  710. print(f"No scheduled jobs found{filters_msg}")
  711. return
  712. # Apply custom format if provided or use default tabular format
  713. self._print_output(rows, table_headers)
  714. except requests.RequestException as e:
  715. print(f"Error fetching scheduled jobs data: {e}")
  716. except (KeyError, ValueError, TypeError) as e:
  717. print(f"Error processing scheduled jobs data: {e}")
  718. except Exception as e:
  719. print(f"Unexpected error - {type(e).__name__}: {e}")
  720. def _matches_filters(self, job_properties: Dict[str, str]) -> bool:
  721. """Check if scheduled job matches all specified filters."""
  722. for key, pattern in self.filters.items():
  723. # Check if property exists
  724. if key not in job_properties:
  725. return False
  726. # Support pattern matching with wildcards
  727. if "*" in pattern or "?" in pattern:
  728. # Convert glob pattern to regex
  729. regex_pattern = pattern.replace("*", ".*").replace("?", ".")
  730. if not re.search(f"^{regex_pattern}$", job_properties[key], re.IGNORECASE):
  731. return False
  732. # Simple substring matching
  733. elif pattern.lower() not in job_properties[key].lower():
  734. return False
  735. return True
  736. def _print_output(self, rows, headers):
  737. """Print output according to the chosen format."""
  738. if self.format:
  739. # Custom template formatting (simplified)
  740. template = self.format
  741. for row in rows:
  742. line = template
  743. for i, field in enumerate(
  744. ["id", "schedule", "image", "command", "last_job_at", "next_job_run_at", "suspend"]
  745. ):
  746. placeholder = f"{{{{.{field}}}}}"
  747. if placeholder in line:
  748. line = line.replace(placeholder, str(row[i]))
  749. print(line)
  750. else:
  751. # Default tabular format
  752. print(
  753. _tabulate(
  754. rows,
  755. headers=headers,
  756. )
  757. )
  758. class ScheduledInspectCommand(BaseHuggingfaceCLICommand):
  759. @staticmethod
  760. def register_subcommand(parser: _SubParsersAction) -> None:
  761. run_parser = parser.add_parser("inspect", help="Display detailed information on one or more scheduled Jobs")
  762. run_parser.add_argument(
  763. "--namespace",
  764. type=str,
  765. help="The namespace where the scheduled job is. Defaults to the current user's namespace.",
  766. )
  767. run_parser.add_argument(
  768. "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
  769. )
  770. run_parser.add_argument("scheduled_job_ids", nargs="...", help="The scheduled jobs to inspect")
  771. run_parser.set_defaults(func=ScheduledInspectCommand)
  772. def __init__(self, args: Namespace) -> None:
  773. self.namespace: Optional[str] = args.namespace
  774. self.token: Optional[str] = args.token
  775. self.scheduled_job_ids: List[str] = args.scheduled_job_ids
  776. def run(self) -> None:
  777. api = HfApi(token=self.token)
  778. scheduled_jobs = [
  779. api.inspect_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=self.namespace)
  780. for scheduled_job_id in self.scheduled_job_ids
  781. ]
  782. print(json.dumps([asdict(scheduled_job) for scheduled_job in scheduled_jobs], indent=4, default=str))
  783. class ScheduledDeleteCommand(BaseHuggingfaceCLICommand):
  784. @staticmethod
  785. def register_subcommand(parser: _SubParsersAction) -> None:
  786. run_parser = parser.add_parser("delete", help="Delete a scheduled Job")
  787. run_parser.add_argument("scheduled_job_id", type=str, help="Scheduled Job ID")
  788. run_parser.add_argument(
  789. "--namespace",
  790. type=str,
  791. help="The namespace where the scheduled job is. Defaults to the current user's namespace.",
  792. )
  793. run_parser.add_argument(
  794. "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
  795. )
  796. run_parser.set_defaults(func=ScheduledDeleteCommand)
  797. def __init__(self, args: Namespace) -> None:
  798. self.scheduled_job_id: str = args.scheduled_job_id
  799. self.namespace = args.namespace
  800. self.token: Optional[str] = args.token
  801. def run(self) -> None:
  802. api = HfApi(token=self.token)
  803. api.delete_scheduled_job(scheduled_job_id=self.scheduled_job_id, namespace=self.namespace)
  804. class ScheduledSuspendCommand(BaseHuggingfaceCLICommand):
  805. @staticmethod
  806. def register_subcommand(parser: _SubParsersAction) -> None:
  807. run_parser = parser.add_parser("suspend", help="Suspend (pause) a scheduled Job")
  808. run_parser.add_argument("scheduled_job_id", type=str, help="Scheduled Job ID")
  809. run_parser.add_argument(
  810. "--namespace",
  811. type=str,
  812. help="The namespace where the scheduled job is. Defaults to the current user's namespace.",
  813. )
  814. run_parser.add_argument(
  815. "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
  816. )
  817. run_parser.set_defaults(func=ScheduledSuspendCommand)
  818. def __init__(self, args: Namespace) -> None:
  819. self.scheduled_job_id: str = args.scheduled_job_id
  820. self.namespace = args.namespace
  821. self.token: Optional[str] = args.token
  822. def run(self) -> None:
  823. api = HfApi(token=self.token)
  824. api.suspend_scheduled_job(scheduled_job_id=self.scheduled_job_id, namespace=self.namespace)
  825. class ScheduledResumeCommand(BaseHuggingfaceCLICommand):
  826. @staticmethod
  827. def register_subcommand(parser: _SubParsersAction) -> None:
  828. run_parser = parser.add_parser("resume", help="Resume (unpause) a scheduled Job")
  829. run_parser.add_argument("scheduled_job_id", type=str, help="Scheduled Job ID")
  830. run_parser.add_argument(
  831. "--namespace",
  832. type=str,
  833. help="The namespace where the scheduled job is. Defaults to the current user's namespace.",
  834. )
  835. run_parser.add_argument(
  836. "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
  837. )
  838. run_parser.set_defaults(func=ScheduledResumeCommand)
  839. def __init__(self, args: Namespace) -> None:
  840. self.scheduled_job_id: str = args.scheduled_job_id
  841. self.namespace = args.namespace
  842. self.token: Optional[str] = args.token
  843. def run(self) -> None:
  844. api = HfApi(token=self.token)
  845. api.resume_scheduled_job(scheduled_job_id=self.scheduled_job_id, namespace=self.namespace)
  846. class ScheduledUvCommand(BaseHuggingfaceCLICommand):
  847. """Schedule UV scripts on Hugging Face infrastructure."""
  848. @staticmethod
  849. def register_subcommand(parser):
  850. """Register UV run subcommand."""
  851. uv_parser = parser.add_parser(
  852. "uv",
  853. help="Schedule UV scripts (Python with inline dependencies) on HF infrastructure",
  854. )
  855. subparsers = uv_parser.add_subparsers(dest="uv_command", help="UV commands", required=True)
  856. # Run command only
  857. run_parser = subparsers.add_parser(
  858. "run",
  859. help="Run a UV script (local file or URL) on HF infrastructure",
  860. )
  861. run_parser.add_argument(
  862. "schedule",
  863. type=str,
  864. help="One of annually, yearly, monthly, weekly, daily, hourly, or a CRON schedule expression.",
  865. )
  866. run_parser.add_argument("script", help="UV script to run (local file or URL)")
  867. run_parser.add_argument("script_args", nargs="...", help="Arguments for the script", default=[])
  868. run_parser.add_argument(
  869. "--suspend",
  870. action="store_true",
  871. help="Suspend (pause) the scheduled Job",
  872. default=None,
  873. )
  874. run_parser.add_argument(
  875. "--concurrency",
  876. action="store_true",
  877. help="Allow multiple instances of this Job to run concurrently",
  878. default=None,
  879. )
  880. run_parser.add_argument("--image", type=str, help="Use a custom Docker image with `uv` installed.")
  881. run_parser.add_argument(
  882. "--repo",
  883. help="Repository name for the script (creates ephemeral if not specified)",
  884. )
  885. run_parser.add_argument(
  886. "--flavor",
  887. type=str,
  888. help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.",
  889. )
  890. run_parser.add_argument("-e", "--env", action="append", help="Environment variables")
  891. run_parser.add_argument(
  892. "-s",
  893. "--secrets",
  894. action="append",
  895. help=(
  896. "Set secret environment variables. E.g. --secrets SECRET=value "
  897. "or `--secrets HF_TOKEN` to pass your Hugging Face token."
  898. ),
  899. )
  900. run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.")
  901. run_parser.add_argument(
  902. "--secrets-file",
  903. type=str,
  904. help="Read in a file of secret environment variables.",
  905. )
  906. run_parser.add_argument("--timeout", type=str, help="Max duration (e.g., 30s, 5m, 1h)")
  907. run_parser.add_argument("-d", "--detach", action="store_true", help="Run in background")
  908. run_parser.add_argument(
  909. "--namespace",
  910. type=str,
  911. help="The namespace where the Job will be created. Defaults to the current user's namespace.",
  912. )
  913. run_parser.add_argument("--token", type=str, help="HF token")
  914. # UV options
  915. run_parser.add_argument("--with", action="append", help="Run with the given packages installed", dest="with_")
  916. run_parser.add_argument(
  917. "-p", "--python", type=str, help="The Python interpreter to use for the run environment"
  918. )
  919. run_parser.set_defaults(func=ScheduledUvCommand)
  920. def __init__(self, args: Namespace) -> None:
  921. """Initialize the command with parsed arguments."""
  922. self.schedule: str = args.schedule
  923. self.script = args.script
  924. self.script_args = args.script_args
  925. self.suspend: Optional[bool] = args.suspend
  926. self.concurrency: Optional[bool] = args.concurrency
  927. self.dependencies = args.with_
  928. self.python = args.python
  929. self.image = args.image
  930. self.env: dict[str, Optional[str]] = {}
  931. if args.env_file:
  932. self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy()))
  933. for env_value in args.env or []:
  934. self.env.update(load_dotenv(env_value, environ=os.environ.copy()))
  935. self.secrets: dict[str, Optional[str]] = {}
  936. extended_environ = _get_extended_environ()
  937. if args.secrets_file:
  938. self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ))
  939. for secret in args.secrets or []:
  940. self.secrets.update(load_dotenv(secret, environ=extended_environ))
  941. self.flavor: Optional[SpaceHardware] = args.flavor
  942. self.timeout: Optional[str] = args.timeout
  943. self.detach: bool = args.detach
  944. self.namespace: Optional[str] = args.namespace
  945. self.token: Optional[str] = args.token
  946. self._repo = args.repo
  947. def run(self) -> None:
  948. """Schedule UV command."""
  949. logging.set_verbosity(logging.INFO)
  950. api = HfApi(token=self.token)
  951. job = api.create_scheduled_uv_job(
  952. script=self.script,
  953. script_args=self.script_args,
  954. schedule=self.schedule,
  955. suspend=self.suspend,
  956. concurrency=self.concurrency,
  957. dependencies=self.dependencies,
  958. python=self.python,
  959. image=self.image,
  960. env=self.env,
  961. secrets=self.secrets,
  962. flavor=self.flavor,
  963. timeout=self.timeout,
  964. namespace=self.namespace,
  965. _repo=self._repo,
  966. )
  967. # Always print the job ID to the user
  968. print(f"Scheduled Job created with ID: {job.id}")