constructor.py 68 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724
  1. from __future__ import annotations
  2. import datetime
  3. from datetime import timedelta as TimeDelta
  4. import binascii
  5. import sys
  6. import types
  7. import warnings
  8. from collections.abc import Hashable, MutableSequence, MutableMapping
  9. # fmt: off
  10. from ruamel.yaml.error import (MarkedYAMLError, MarkedYAMLFutureWarning,
  11. MantissaNoDotYAML1_1Warning)
  12. from ruamel.yaml.nodes import * # NOQA
  13. from ruamel.yaml.nodes import (SequenceNode, MappingNode, ScalarNode)
  14. from ruamel.yaml.compat import (builtins_module, # NOQA
  15. nprint, nprintf, version_tnf)
  16. from ruamel.yaml.compat import ordereddict
  17. from ruamel.yaml.tag import Tag
  18. from ruamel.yaml.comments import * # NOQA
  19. from ruamel.yaml.comments import (CommentedMap, CommentedOrderedMap, CommentedSet,
  20. CommentedKeySeq, CommentedSeq, TaggedScalar,
  21. CommentedKeyMap,
  22. C_KEY_PRE, C_KEY_EOL, C_KEY_POST,
  23. C_VALUE_PRE, C_VALUE_EOL, C_VALUE_POST,
  24. )
  25. from ruamel.yaml.scalarstring import (SingleQuotedScalarString, DoubleQuotedScalarString,
  26. LiteralScalarString, FoldedScalarString,
  27. PlainScalarString, ScalarString)
  28. from ruamel.yaml.scalarint import ScalarInt, BinaryInt, OctalInt, HexInt, HexCapsInt
  29. from ruamel.yaml.scalarfloat import ScalarFloat
  30. from ruamel.yaml.scalarbool import ScalarBoolean
  31. from ruamel.yaml.timestamp import TimeStamp
  32. from ruamel.yaml.util import timestamp_regexp, create_timestamp
  33. if False: # MYPY
  34. from typing import Any, Dict, List, Set, Iterator, Union, Optional # NOQA
  35. __all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor',
  36. 'ConstructorError', 'RoundTripConstructor']
  37. # fmt: on
  38. class ConstructorError(MarkedYAMLError):
  39. pass
  40. class DuplicateKeyFutureWarning(MarkedYAMLFutureWarning):
  41. pass
  42. DUPKEY_URL = 'https://yaml.dev/doc/ruamel.yaml/api/#Duplicate_keys'
  43. class DuplicateKeyError(MarkedYAMLError):
  44. pass
  45. class BaseConstructor:
  46. yaml_constructors = {} # type: Dict[Any, Any]
  47. yaml_multi_constructors = {} # type: Dict[Any, Any]
  48. def __init__(self, preserve_quotes: Optional[bool] = None, loader: Any = None) -> None:
  49. self.loader = loader
  50. if self.loader is not None and getattr(self.loader, '_constructor', None) is None:
  51. self.loader._constructor = self
  52. self.loader = loader
  53. self.yaml_base_dict_type = dict
  54. self.yaml_base_list_type = list
  55. self.constructed_objects: Dict[Any, Any] = {}
  56. self.recursive_objects: Dict[Any, Any] = {}
  57. self.state_generators: List[Any] = []
  58. self.deep_construct = False
  59. self._preserve_quotes = preserve_quotes
  60. self.allow_duplicate_keys = version_tnf((0, 15, 1), (0, 16))
  61. @property
  62. def composer(self) -> Any:
  63. if hasattr(self.loader, 'typ'):
  64. return self.loader.composer
  65. try:
  66. return self.loader._composer
  67. except AttributeError:
  68. sys.stdout.write(f'slt {type(self)}\n')
  69. sys.stdout.write(f'slc {self.loader._composer}\n')
  70. sys.stdout.write(f'{dir(self)}\n')
  71. raise
  72. @property
  73. def resolver(self) -> Any:
  74. if hasattr(self.loader, 'typ'):
  75. return self.loader.resolver
  76. return self.loader._resolver
  77. @property
  78. def scanner(self) -> Any:
  79. # needed to get to the expanded comments
  80. if hasattr(self.loader, 'typ'):
  81. return self.loader.scanner
  82. return self.loader._scanner
  83. def check_data(self) -> Any:
  84. # If there are more documents available?
  85. return self.composer.check_node()
  86. def get_data(self) -> Any:
  87. # Construct and return the next document.
  88. if self.composer.check_node():
  89. return self.construct_document(self.composer.get_node())
  90. def get_single_data(self) -> Any:
  91. # Ensure that the stream contains a single document and construct it.
  92. node = self.composer.get_single_node()
  93. if node is not None:
  94. return self.construct_document(node)
  95. return None
  96. def construct_document(self, node: Any) -> Any:
  97. data = self.construct_object(node)
  98. while bool(self.state_generators):
  99. state_generators = self.state_generators
  100. self.state_generators = []
  101. for generator in state_generators:
  102. for _dummy in generator:
  103. pass
  104. self.constructed_objects = {}
  105. self.recursive_objects = {}
  106. self.deep_construct = False
  107. return data
  108. def construct_object(self, node: Any, deep: bool = False) -> Any:
  109. """deep is True when creating an object/mapping recursively,
  110. in that case want the underlying elements available during construction
  111. """
  112. if node in self.constructed_objects:
  113. return self.constructed_objects[node]
  114. if deep:
  115. old_deep = self.deep_construct
  116. self.deep_construct = True
  117. if node in self.recursive_objects:
  118. return self.recursive_objects[node]
  119. # raise ConstructorError(
  120. # None, None, 'found unconstructable recursive node', node.start_mark
  121. # )
  122. self.recursive_objects[node] = None
  123. data = self.construct_non_recursive_object(node)
  124. self.constructed_objects[node] = data
  125. del self.recursive_objects[node]
  126. if deep:
  127. self.deep_construct = old_deep
  128. return data
  129. def construct_non_recursive_object(self, node: Any, tag: Optional[str] = None) -> Any:
  130. constructor: Any = None
  131. tag_suffix = None
  132. if tag is None:
  133. tag = node.tag
  134. if tag in self.yaml_constructors:
  135. constructor = self.yaml_constructors[tag]
  136. else:
  137. for tag_prefix in self.yaml_multi_constructors:
  138. if tag.startswith(tag_prefix):
  139. tag_suffix = tag[len(tag_prefix) :]
  140. constructor = self.yaml_multi_constructors[tag_prefix]
  141. break
  142. else:
  143. if None in self.yaml_multi_constructors:
  144. tag_suffix = tag
  145. constructor = self.yaml_multi_constructors[None]
  146. elif None in self.yaml_constructors:
  147. constructor = self.yaml_constructors[None]
  148. elif isinstance(node, ScalarNode):
  149. constructor = self.__class__.construct_scalar
  150. elif isinstance(node, SequenceNode):
  151. constructor = self.__class__.construct_sequence
  152. elif isinstance(node, MappingNode):
  153. constructor = self.__class__.construct_mapping
  154. if tag_suffix is None:
  155. data = constructor(self, node)
  156. else:
  157. data = constructor(self, tag_suffix, node)
  158. if isinstance(data, types.GeneratorType):
  159. generator = data
  160. data = next(generator)
  161. if self.deep_construct:
  162. for _dummy in generator:
  163. pass
  164. else:
  165. self.state_generators.append(generator)
  166. return data
  167. def construct_scalar(self, node: Any) -> Any:
  168. if not isinstance(node, ScalarNode):
  169. raise ConstructorError(
  170. None, None, f'expected a scalar node, but found {node.id!s}', node.start_mark,
  171. )
  172. return node.value
  173. def construct_sequence(self, node: Any, deep: bool = False) -> Any:
  174. """deep is True when creating an object/mapping recursively,
  175. in that case want the underlying elements available during construction
  176. """
  177. if not isinstance(node, SequenceNode):
  178. raise ConstructorError(
  179. None,
  180. None,
  181. f'expected a sequence node, but found {node.id!s}',
  182. node.start_mark,
  183. )
  184. return [self.construct_object(child, deep=deep) for child in node.value]
  185. def construct_mapping(self, node: Any, deep: bool = False) -> Any:
  186. """deep is True when creating an object/mapping recursively,
  187. in that case want the underlying elements available during construction
  188. """
  189. if not isinstance(node, MappingNode):
  190. raise ConstructorError(
  191. None, None, f'expected a mapping node, but found {node.id!s}', node.start_mark,
  192. )
  193. total_mapping = self.yaml_base_dict_type()
  194. if getattr(node, 'merge', None) is not None:
  195. todo = [(node.merge, False), (node.value, False)]
  196. else:
  197. todo = [(node.value, True)]
  198. for values, check in todo:
  199. mapping: Dict[Any, Any] = self.yaml_base_dict_type()
  200. for key_node, value_node in values:
  201. # keys can be list -> deep
  202. key = self.construct_object(key_node, deep=True)
  203. # lists are not hashable, but tuples are
  204. if not isinstance(key, Hashable):
  205. if isinstance(key, list):
  206. key = tuple(key)
  207. if not isinstance(key, Hashable):
  208. raise ConstructorError(
  209. 'while constructing a mapping',
  210. node.start_mark,
  211. 'found unhashable key',
  212. key_node.start_mark,
  213. )
  214. value = self.construct_object(value_node, deep=deep)
  215. if check:
  216. if self.check_mapping_key(node, key_node, mapping, key, value):
  217. mapping[key] = value
  218. else:
  219. mapping[key] = value
  220. total_mapping.update(mapping)
  221. return total_mapping
  222. def check_mapping_key(
  223. self, node: Any, key_node: Any, mapping: Any, key: Any, value: Any,
  224. ) -> bool:
  225. """return True if key is unique"""
  226. if key in mapping:
  227. if not self.allow_duplicate_keys:
  228. mk = mapping.get(key)
  229. args = [
  230. 'while constructing a mapping',
  231. node.start_mark,
  232. f'found duplicate key "{key}" with value "{value}" '
  233. f'(original value: "{mk}")',
  234. key_node.start_mark,
  235. f"""
  236. To suppress this check see:
  237. {DUPKEY_URL}
  238. """,
  239. """\
  240. Duplicate keys will become an error in future releases, and are errors
  241. by default when using the new API.
  242. """,
  243. ]
  244. if self.allow_duplicate_keys is None:
  245. warnings.warn(DuplicateKeyFutureWarning(*args), stacklevel=1)
  246. else:
  247. raise DuplicateKeyError(*args)
  248. return False
  249. return True
  250. def check_set_key(self: Any, node: Any, key_node: Any, setting: Any, key: Any) -> None:
  251. if key in setting:
  252. if not self.allow_duplicate_keys:
  253. args = [
  254. 'while constructing a set',
  255. node.start_mark,
  256. f'found duplicate key "{key}"',
  257. key_node.start_mark,
  258. f"""
  259. To suppress this check see:
  260. {DUPKEY_URL}
  261. """,
  262. """\
  263. Duplicate keys will become an error in future releases, and are errors
  264. by default when using the new API.
  265. """,
  266. ]
  267. if self.allow_duplicate_keys is None:
  268. warnings.warn(DuplicateKeyFutureWarning(*args), stacklevel=1)
  269. else:
  270. raise DuplicateKeyError(*args)
  271. def construct_pairs(self, node: Any, deep: bool = False) -> Any:
  272. if not isinstance(node, MappingNode):
  273. raise ConstructorError(
  274. None, None, f'expected a mapping node, but found {node.id!s}', node.start_mark,
  275. )
  276. pairs = []
  277. for key_node, value_node in node.value:
  278. key = self.construct_object(key_node, deep=deep)
  279. value = self.construct_object(value_node, deep=deep)
  280. pairs.append((key, value))
  281. return pairs
  282. # ToDo: putting stuff on the class makes it global, consider making this to work on an
  283. # instance variable once function load is dropped.
  284. @classmethod
  285. def add_constructor(cls, tag: Any, constructor: Any) -> Any:
  286. if isinstance(tag, Tag):
  287. tag = str(tag)
  288. if 'yaml_constructors' not in cls.__dict__:
  289. cls.yaml_constructors = cls.yaml_constructors.copy()
  290. ret_val = cls.yaml_constructors.get(tag, None)
  291. cls.yaml_constructors[tag] = constructor
  292. return ret_val
  293. @classmethod
  294. def add_multi_constructor(cls, tag_prefix: Any, multi_constructor: Any) -> None:
  295. if 'yaml_multi_constructors' not in cls.__dict__:
  296. cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
  297. cls.yaml_multi_constructors[tag_prefix] = multi_constructor
  298. @classmethod
  299. def add_default_constructor(
  300. cls, tag: str, method: Any = None, tag_base: str = 'tag:yaml.org,2002:',
  301. ) -> None:
  302. if not tag.startswith('tag:'):
  303. if method is None:
  304. method = 'construct_yaml_' + tag
  305. tag = tag_base + tag
  306. cls.add_constructor(tag, getattr(cls, method))
  307. class SafeConstructor(BaseConstructor):
  308. def construct_scalar(self, node: Any) -> Any:
  309. if isinstance(node, MappingNode):
  310. for key_node, value_node in node.value:
  311. if key_node.tag == 'tag:yaml.org,2002:value':
  312. return self.construct_scalar(value_node)
  313. return BaseConstructor.construct_scalar(self, node)
  314. def flatten_mapping(self, node: Any) -> Any: # SafeConstructor
  315. """
  316. This implements the merge key feature http://yaml.org/type/merge.html
  317. by inserting keys from the merge dict/list of dicts if not yet
  318. available in this node
  319. """
  320. merge: List[Any] = []
  321. index = 0
  322. while index < len(node.value):
  323. key_node, value_node = node.value[index]
  324. if key_node.tag == 'tag:yaml.org,2002:merge':
  325. if merge: # double << key
  326. if self.allow_duplicate_keys:
  327. del node.value[index]
  328. index += 1
  329. continue
  330. args = [
  331. 'while constructing a mapping',
  332. node.start_mark,
  333. 'found duplicate merge key "<<"',
  334. key_node.start_mark,
  335. """\
  336. Duplicate merge keys are never allowed, not even when
  337. `.allow_duplicate_keys` is set to True
  338. """,
  339. ]
  340. raise DuplicateKeyError(*args)
  341. del node.value[index]
  342. if isinstance(value_node, MappingNode):
  343. self.flatten_mapping(value_node)
  344. merge.extend(value_node.value)
  345. elif isinstance(value_node, SequenceNode):
  346. submerge = []
  347. for subnode in value_node.value:
  348. if not isinstance(subnode, MappingNode):
  349. raise ConstructorError(
  350. 'while constructing a mapping',
  351. node.start_mark,
  352. f'expected a mapping for merging, but found {subnode.id!s}',
  353. subnode.start_mark,
  354. )
  355. self.flatten_mapping(subnode)
  356. submerge.append(subnode.value)
  357. submerge.reverse()
  358. for value in submerge:
  359. merge.extend(value)
  360. else:
  361. raise ConstructorError(
  362. 'while constructing a mapping',
  363. node.start_mark,
  364. 'expected a mapping or list of mappings for merging, '
  365. f'but found {value_node.id!s}',
  366. value_node.start_mark,
  367. )
  368. elif key_node.tag == 'tag:yaml.org,2002:value':
  369. key_node.tag = 'tag:yaml.org,2002:str'
  370. index += 1
  371. else:
  372. index += 1
  373. if bool(merge):
  374. node.merge = merge # separate merge keys to be able to update without duplicate
  375. node.value = merge + node.value
  376. def construct_mapping(self, node: Any, deep: bool = False) -> Any:
  377. """deep is True when creating an object/mapping recursively,
  378. in that case want the underlying elements available during construction
  379. """
  380. if isinstance(node, MappingNode):
  381. self.flatten_mapping(node)
  382. return BaseConstructor.construct_mapping(self, node, deep=deep)
  383. def construct_yaml_null(self, node: Any) -> Any:
  384. self.construct_scalar(node)
  385. return None
  386. # YAML 1.2 spec doesn't mention yes/no etc any more, 1.1 does
  387. bool_values = {
  388. 'yes': True,
  389. 'no': False,
  390. 'y': True,
  391. 'n': False,
  392. 'true': True,
  393. 'false': False,
  394. 'on': True,
  395. 'off': False,
  396. }
  397. def construct_yaml_bool(self, node: Any) -> bool:
  398. value = self.construct_scalar(node)
  399. return self.bool_values[value.lower()]
  400. def construct_yaml_int(self, node: Any) -> int:
  401. value_s = self.construct_scalar(node)
  402. value_s = value_s.replace('_', "")
  403. sign = +1
  404. if value_s[0] == '-':
  405. sign = -1
  406. if value_s[0] in '+-':
  407. value_s = value_s[1:]
  408. if value_s == '0':
  409. return 0
  410. elif value_s.startswith('0b'):
  411. return sign * int(value_s[2:], 2)
  412. elif value_s.startswith('0x'):
  413. return sign * int(value_s[2:], 16)
  414. elif value_s.startswith('0o'):
  415. return sign * int(value_s[2:], 8)
  416. elif self.resolver.processing_version == (1, 1) and value_s[0] == '0':
  417. return sign * int(value_s, 8)
  418. elif self.resolver.processing_version == (1, 1) and ':' in value_s:
  419. digits = [int(part) for part in value_s.split(':')]
  420. digits.reverse()
  421. base = 1
  422. value = 0
  423. for digit in digits:
  424. value += digit * base
  425. base *= 60
  426. return sign * value
  427. else:
  428. return sign * int(value_s)
  429. inf_value = 1e300
  430. while inf_value != inf_value * inf_value:
  431. inf_value *= inf_value
  432. nan_value = -inf_value / inf_value # Trying to make a quiet NaN (like C99).
  433. def construct_yaml_float(self, node: Any) -> float:
  434. value_so = self.construct_scalar(node)
  435. value_s = value_so.replace('_', "").lower()
  436. sign = +1
  437. if value_s[0] == '-':
  438. sign = -1
  439. if value_s[0] in '+-':
  440. value_s = value_s[1:]
  441. if value_s == '.inf':
  442. return sign * self.inf_value
  443. elif value_s == '.nan':
  444. return self.nan_value
  445. elif self.resolver.processing_version != (1, 2) and ':' in value_s:
  446. digits = [float(part) for part in value_s.split(':')]
  447. digits.reverse()
  448. base = 1
  449. value = 0.0
  450. for digit in digits:
  451. value += digit * base
  452. base *= 60
  453. return sign * value
  454. else:
  455. if self.resolver.processing_version != (1, 2) and 'e' in value_s:
  456. # value_s is lower case independent of input
  457. mantissa, exponent = value_s.split('e')
  458. if '.' not in mantissa:
  459. warnings.warn(MantissaNoDotYAML1_1Warning(node, value_so), stacklevel=1)
  460. return sign * float(value_s)
  461. def construct_yaml_binary(self, node: Any) -> Any:
  462. import base64
  463. try:
  464. value = self.construct_scalar(node).encode('ascii')
  465. except UnicodeEncodeError as exc:
  466. raise ConstructorError(
  467. None,
  468. None,
  469. f'failed to convert base64 data into ascii: {exc!s}',
  470. node.start_mark,
  471. )
  472. try:
  473. return base64.decodebytes(value)
  474. except binascii.Error as exc:
  475. raise ConstructorError(
  476. None, None, f'failed to decode base64 data: {exc!s}', node.start_mark,
  477. )
  478. timestamp_regexp = timestamp_regexp # moved to util 0.17.17
  479. def construct_yaml_timestamp(self, node: Any, values: Any = None) -> Any:
  480. if values is None:
  481. try:
  482. match = self.timestamp_regexp.match(node.value)
  483. except TypeError:
  484. match = None
  485. if match is None:
  486. raise ConstructorError(
  487. None,
  488. None,
  489. f'failed to construct timestamp from "{node.value}"',
  490. node.start_mark,
  491. )
  492. values = match.groupdict()
  493. return create_timestamp(**values)
  494. def construct_yaml_omap(self, node: Any) -> Any:
  495. # Note: we do now check for duplicate keys
  496. omap = ordereddict()
  497. yield omap
  498. if not isinstance(node, SequenceNode):
  499. raise ConstructorError(
  500. 'while constructing an ordered map',
  501. node.start_mark,
  502. f'expected a sequence, but found {node.id!s}',
  503. node.start_mark,
  504. )
  505. for subnode in node.value:
  506. if not isinstance(subnode, MappingNode):
  507. raise ConstructorError(
  508. 'while constructing an ordered map',
  509. node.start_mark,
  510. f'expected a mapping of length 1, but found {subnode.id!s}',
  511. subnode.start_mark,
  512. )
  513. if len(subnode.value) != 1:
  514. raise ConstructorError(
  515. 'while constructing an ordered map',
  516. node.start_mark,
  517. f'expected a single mapping item, but found {len(subnode.value):d} items',
  518. subnode.start_mark,
  519. )
  520. key_node, value_node = subnode.value[0]
  521. key = self.construct_object(key_node)
  522. assert key not in omap
  523. value = self.construct_object(value_node)
  524. omap[key] = value
  525. def construct_yaml_pairs(self, node: Any) -> Any:
  526. # Note: the same code as `construct_yaml_omap`.
  527. pairs: List[Any] = []
  528. yield pairs
  529. if not isinstance(node, SequenceNode):
  530. raise ConstructorError(
  531. 'while constructing pairs',
  532. node.start_mark,
  533. f'expected a sequence, but found {node.id!s}',
  534. node.start_mark,
  535. )
  536. for subnode in node.value:
  537. if not isinstance(subnode, MappingNode):
  538. raise ConstructorError(
  539. 'while constructing pairs',
  540. node.start_mark,
  541. f'expected a mapping of length 1, but found {subnode.id!s}',
  542. subnode.start_mark,
  543. )
  544. if len(subnode.value) != 1:
  545. raise ConstructorError(
  546. 'while constructing pairs',
  547. node.start_mark,
  548. f'expected a single mapping item, but found {len(subnode.value):d} items',
  549. subnode.start_mark,
  550. )
  551. key_node, value_node = subnode.value[0]
  552. key = self.construct_object(key_node)
  553. value = self.construct_object(value_node)
  554. pairs.append((key, value))
  555. def construct_yaml_set(self, node: Any) -> Any:
  556. data: Set[Any] = set()
  557. yield data
  558. value = self.construct_mapping(node)
  559. data.update(value)
  560. def construct_yaml_str(self, node: Any) -> Any:
  561. value = self.construct_scalar(node)
  562. return value
  563. def construct_yaml_seq(self, node: Any) -> Any:
  564. data: List[Any] = self.yaml_base_list_type()
  565. yield data
  566. data.extend(self.construct_sequence(node))
  567. def construct_yaml_map(self, node: Any) -> Any:
  568. data: Dict[Any, Any] = self.yaml_base_dict_type()
  569. yield data
  570. value = self.construct_mapping(node)
  571. data.update(value)
  572. def construct_yaml_object(self, node: Any, cls: Any) -> Any:
  573. data = cls.__new__(cls)
  574. yield data
  575. if hasattr(data, '__setstate__'):
  576. state = self.construct_mapping(node, deep=True)
  577. data.__setstate__(state)
  578. else:
  579. state = self.construct_mapping(node)
  580. data.__dict__.update(state)
  581. def construct_undefined(self, node: Any) -> None:
  582. raise ConstructorError(
  583. None,
  584. None,
  585. f'could not determine a constructor for the tag {node.tag!r}',
  586. node.start_mark,
  587. )
  588. for tag in 'null bool int float binary timestamp omap pairs set str seq map'.split():
  589. SafeConstructor.add_default_constructor(tag)
  590. SafeConstructor.add_constructor(None, SafeConstructor.construct_undefined)
  591. class Constructor(SafeConstructor):
  592. def construct_python_str(self, node: Any) -> Any:
  593. return self.construct_scalar(node)
  594. def construct_python_unicode(self, node: Any) -> Any:
  595. return self.construct_scalar(node)
  596. def construct_python_bytes(self, node: Any) -> Any:
  597. import base64
  598. try:
  599. value = self.construct_scalar(node).encode('ascii')
  600. except UnicodeEncodeError as exc:
  601. raise ConstructorError(
  602. None,
  603. None,
  604. f'failed to convert base64 data into ascii: {exc!s}',
  605. node.start_mark,
  606. )
  607. try:
  608. return base64.decodebytes(value)
  609. except binascii.Error as exc:
  610. raise ConstructorError(
  611. None, None, f'failed to decode base64 data: {exc!s}', node.start_mark,
  612. )
  613. def construct_python_long(self, node: Any) -> int:
  614. val = self.construct_yaml_int(node)
  615. return val
  616. def construct_python_complex(self, node: Any) -> Any:
  617. return complex(self.construct_scalar(node))
  618. def construct_python_tuple(self, node: Any) -> Any:
  619. return tuple(self.construct_sequence(node))
  620. def find_python_module(self, name: Any, mark: Any) -> Any:
  621. if not name:
  622. raise ConstructorError(
  623. 'while constructing a Python module',
  624. mark,
  625. 'expected non-empty name appended to the tag',
  626. mark,
  627. )
  628. try:
  629. __import__(name)
  630. except ImportError as exc:
  631. raise ConstructorError(
  632. 'while constructing a Python module',
  633. mark,
  634. f'cannot find module {name!r} ({exc!s})',
  635. mark,
  636. )
  637. return sys.modules[name]
  638. def find_python_name(self, name: Any, mark: Any) -> Any:
  639. if not name:
  640. raise ConstructorError(
  641. 'while constructing a Python object',
  642. mark,
  643. 'expected non-empty name appended to the tag',
  644. mark,
  645. )
  646. if '.' in name:
  647. lname = name.split('.')
  648. lmodule_name = lname
  649. lobject_name: List[Any] = []
  650. while len(lmodule_name) > 1:
  651. lobject_name.insert(0, lmodule_name.pop())
  652. module_name = '.'.join(lmodule_name)
  653. try:
  654. __import__(module_name)
  655. # object_name = '.'.join(object_name)
  656. break
  657. except ImportError:
  658. continue
  659. else:
  660. module_name = builtins_module
  661. lobject_name = [name]
  662. try:
  663. __import__(module_name)
  664. except ImportError as exc:
  665. raise ConstructorError(
  666. 'while constructing a Python object',
  667. mark,
  668. f'cannot find module {module_name!r} ({exc!s})',
  669. mark,
  670. )
  671. module = sys.modules[module_name]
  672. object_name = '.'.join(lobject_name)
  673. obj = module
  674. while lobject_name:
  675. if not hasattr(obj, lobject_name[0]):
  676. raise ConstructorError(
  677. 'while constructing a Python object',
  678. mark,
  679. f'cannot find {object_name!r} in the module {module.__name__!r}',
  680. mark,
  681. )
  682. obj = getattr(obj, lobject_name.pop(0))
  683. return obj
  684. def construct_python_name(self, suffix: Any, node: Any) -> Any:
  685. value = self.construct_scalar(node)
  686. if value:
  687. raise ConstructorError(
  688. 'while constructing a Python name',
  689. node.start_mark,
  690. f'expected the empty value, but found {value!r}',
  691. node.start_mark,
  692. )
  693. return self.find_python_name(suffix, node.start_mark)
  694. def construct_python_module(self, suffix: Any, node: Any) -> Any:
  695. value = self.construct_scalar(node)
  696. if value:
  697. raise ConstructorError(
  698. 'while constructing a Python module',
  699. node.start_mark,
  700. f'expected the empty value, but found {value!r}',
  701. node.start_mark,
  702. )
  703. return self.find_python_module(suffix, node.start_mark)
  704. def make_python_instance(
  705. self, suffix: Any, node: Any, args: Any = None, kwds: Any = None, newobj: bool = False,
  706. ) -> Any:
  707. if not args:
  708. args = []
  709. if not kwds:
  710. kwds = {}
  711. cls = self.find_python_name(suffix, node.start_mark)
  712. if newobj and isinstance(cls, type):
  713. return cls.__new__(cls, *args, **kwds)
  714. else:
  715. return cls(*args, **kwds)
  716. def set_python_instance_state(self, instance: Any, state: Any) -> None:
  717. if hasattr(instance, '__setstate__'):
  718. instance.__setstate__(state)
  719. else:
  720. slotstate: Dict[Any, Any] = {}
  721. if isinstance(state, tuple) and len(state) == 2:
  722. state, slotstate = state
  723. if hasattr(instance, '__dict__'):
  724. instance.__dict__.update(state)
  725. elif state:
  726. slotstate.update(state)
  727. for key, value in slotstate.items():
  728. setattr(instance, key, value)
  729. def construct_python_object(self, suffix: Any, node: Any) -> Any:
  730. # Format:
  731. # !!python/object:module.name { ... state ... }
  732. instance = self.make_python_instance(suffix, node, newobj=True)
  733. self.recursive_objects[node] = instance
  734. yield instance
  735. deep = hasattr(instance, '__setstate__')
  736. state = self.construct_mapping(node, deep=deep)
  737. self.set_python_instance_state(instance, state)
  738. def construct_python_object_apply(
  739. self, suffix: Any, node: Any, newobj: bool = False,
  740. ) -> Any:
  741. # Format:
  742. # !!python/object/apply # (or !!python/object/new)
  743. # args: [ ... arguments ... ]
  744. # kwds: { ... keywords ... }
  745. # state: ... state ...
  746. # listitems: [ ... listitems ... ]
  747. # dictitems: { ... dictitems ... }
  748. # or short format:
  749. # !!python/object/apply [ ... arguments ... ]
  750. # The difference between !!python/object/apply and !!python/object/new
  751. # is how an object is created, check make_python_instance for details.
  752. if isinstance(node, SequenceNode):
  753. args = self.construct_sequence(node, deep=True)
  754. kwds: Dict[Any, Any] = {}
  755. state: Dict[Any, Any] = {}
  756. listitems: List[Any] = []
  757. dictitems: Dict[Any, Any] = {}
  758. else:
  759. value = self.construct_mapping(node, deep=True)
  760. args = value.get('args', [])
  761. kwds = value.get('kwds', {})
  762. state = value.get('state', {})
  763. listitems = value.get('listitems', [])
  764. dictitems = value.get('dictitems', {})
  765. instance = self.make_python_instance(suffix, node, args, kwds, newobj)
  766. if bool(state):
  767. self.set_python_instance_state(instance, state)
  768. if bool(listitems):
  769. instance.extend(listitems)
  770. if bool(dictitems):
  771. for key in dictitems:
  772. instance[key] = dictitems[key]
  773. return instance
  774. def construct_python_object_new(self, suffix: Any, node: Any) -> Any:
  775. return self.construct_python_object_apply(suffix, node, newobj=True)
  776. @classmethod
  777. def add_default_constructor(
  778. cls, tag: str, method: Any = None, tag_base: str = 'tag:yaml.org,2002:python/',
  779. ) -> None:
  780. if not tag.startswith('tag:'):
  781. if method is None:
  782. method = 'construct_yaml_' + tag
  783. tag = tag_base + tag
  784. cls.add_constructor(tag, getattr(cls, method))
  785. Constructor.add_constructor('tag:yaml.org,2002:python/none', Constructor.construct_yaml_null)
  786. Constructor.add_constructor('tag:yaml.org,2002:python/bool', Constructor.construct_yaml_bool)
  787. Constructor.add_constructor('tag:yaml.org,2002:python/str', Constructor.construct_python_str)
  788. Constructor.add_constructor(
  789. 'tag:yaml.org,2002:python/unicode', Constructor.construct_python_unicode,
  790. )
  791. Constructor.add_constructor(
  792. 'tag:yaml.org,2002:python/bytes', Constructor.construct_python_bytes,
  793. )
  794. Constructor.add_constructor('tag:yaml.org,2002:python/int', Constructor.construct_yaml_int)
  795. Constructor.add_constructor('tag:yaml.org,2002:python/long', Constructor.construct_python_long)
  796. Constructor.add_constructor('tag:yaml.org,2002:python/float', Constructor.construct_yaml_float)
  797. Constructor.add_constructor(
  798. 'tag:yaml.org,2002:python/complex', Constructor.construct_python_complex,
  799. )
  800. Constructor.add_constructor('tag:yaml.org,2002:python/list', Constructor.construct_yaml_seq)
  801. Constructor.add_constructor(
  802. 'tag:yaml.org,2002:python/tuple', Constructor.construct_python_tuple,
  803. )
  804. # for tag in 'bool str unicode bytes int long float complex tuple'.split():
  805. # Constructor.add_default_constructor(tag)
  806. Constructor.add_constructor('tag:yaml.org,2002:python/dict', Constructor.construct_yaml_map)
  807. Constructor.add_multi_constructor(
  808. 'tag:yaml.org,2002:python/name:', Constructor.construct_python_name,
  809. )
  810. Constructor.add_multi_constructor(
  811. 'tag:yaml.org,2002:python/module:', Constructor.construct_python_module,
  812. )
  813. Constructor.add_multi_constructor(
  814. 'tag:yaml.org,2002:python/object:', Constructor.construct_python_object,
  815. )
  816. Constructor.add_multi_constructor(
  817. 'tag:yaml.org,2002:python/object/apply:', Constructor.construct_python_object_apply,
  818. )
  819. Constructor.add_multi_constructor(
  820. 'tag:yaml.org,2002:python/object/new:', Constructor.construct_python_object_new,
  821. )
  822. class RoundTripConstructor(SafeConstructor):
  823. """need to store the comments on the node itself,
  824. as well as on the items
  825. """
  826. def comment(self, idx: Any) -> Any:
  827. assert self.loader.comment_handling is not None
  828. x = self.scanner.comments[idx]
  829. x.set_assigned()
  830. return x
  831. def comments(self, list_of_comments: Any, idx: Optional[Any] = None) -> Any:
  832. # hand in the comment and optional pre, eol, post segment
  833. if list_of_comments is None:
  834. return []
  835. if idx is not None:
  836. if list_of_comments[idx] is None:
  837. return []
  838. list_of_comments = list_of_comments[idx]
  839. for x in list_of_comments:
  840. yield self.comment(x)
  841. def construct_scalar(self, node: Any) -> Any:
  842. if not isinstance(node, ScalarNode):
  843. raise ConstructorError(
  844. None, None, f'expected a scalar node, but found {node.id!s}', node.start_mark,
  845. )
  846. if node.style == '|' and isinstance(node.value, str):
  847. lss = LiteralScalarString(node.value, anchor=node.anchor)
  848. if self.loader and self.loader.comment_handling is None:
  849. if node.comment and node.comment[1]:
  850. lss.comment = node.comment[1][0] # type: ignore
  851. else:
  852. # NEWCMNT
  853. if node.comment is not None and node.comment[1]:
  854. # nprintf('>>>>nc1', node.comment)
  855. # EOL comment after |
  856. lss.comment = self.comment(node.comment[1][0]) # type: ignore
  857. return lss
  858. if node.style == '>' and isinstance(node.value, str):
  859. fold_positions: List[int] = []
  860. idx = -1
  861. while True:
  862. idx = node.value.find('\a', idx + 1)
  863. if idx < 0:
  864. break
  865. fold_positions.append(idx - len(fold_positions))
  866. fss = FoldedScalarString(node.value.replace('\a', ''), anchor=node.anchor)
  867. if self.loader and self.loader.comment_handling is None:
  868. if node.comment and node.comment[1]:
  869. fss.comment = node.comment[1][0] # type: ignore
  870. else:
  871. # NEWCMNT
  872. if node.comment is not None and node.comment[1]:
  873. # nprintf('>>>>nc2', node.comment)
  874. # EOL comment after >
  875. fss.comment = self.comment(node.comment[1][0]) # type: ignore
  876. if fold_positions:
  877. fss.fold_pos = fold_positions # type: ignore
  878. return fss
  879. elif bool(self._preserve_quotes) and isinstance(node.value, str):
  880. if node.style == "'":
  881. return SingleQuotedScalarString(node.value, anchor=node.anchor)
  882. if node.style == '"':
  883. return DoubleQuotedScalarString(node.value, anchor=node.anchor)
  884. # if node.ctag:
  885. # data2 = TaggedScalar()
  886. # data2.value = node.value
  887. # data2.style = node.style
  888. # data2.yaml_set_ctag(node.ctag)
  889. # if node.anchor:
  890. # from ruamel.yaml.serializer import templated_id
  891. # if not templated_id(node.anchor):
  892. # data2.yaml_set_anchor(node.anchor, always_dump=True)
  893. # return data2
  894. if node.anchor:
  895. return PlainScalarString(node.value, anchor=node.anchor)
  896. return node.value
  897. def construct_yaml_int(self, node: Any) -> Any:
  898. width: Any = None
  899. value_su = self.construct_scalar(node)
  900. try:
  901. sx = value_su.rstrip('_')
  902. underscore: Any = [len(sx) - sx.rindex('_') - 1, False, False]
  903. except ValueError:
  904. underscore = None
  905. except IndexError:
  906. underscore = None
  907. value_s = value_su.replace('_', "")
  908. sign = +1
  909. if value_s[0] in '+-':
  910. if value_s[0] == '-':
  911. sign = -1
  912. value_s = value_s[1:]
  913. if value_s.startswith('0b'):
  914. if self.resolver.processing_version > (1, 1) and value_s[2] == '0':
  915. width = len(value_s[2:])
  916. if underscore is not None:
  917. underscore[1] = value_su[2] == '_'
  918. underscore[2] = len(value_su[2:]) > 1 and value_su[-1] == '_'
  919. return BinaryInt(
  920. sign * int(value_s[2:], 2),
  921. width=width,
  922. underscore=underscore,
  923. anchor=node.anchor,
  924. )
  925. elif value_s.startswith('0x'):
  926. # default to lower-case if no a-fA-F in string
  927. if self.resolver.processing_version > (1, 1) and value_s[2] == '0':
  928. width = len(value_s[2:])
  929. hex_fun: Any = HexInt
  930. for ch in value_s[2:]:
  931. if ch in 'ABCDEF': # first non-digit is capital
  932. hex_fun = HexCapsInt
  933. break
  934. if ch in 'abcdef':
  935. break
  936. if underscore is not None:
  937. underscore[1] = value_su[2] == '_'
  938. underscore[2] = len(value_su[2:]) > 1 and value_su[-1] == '_'
  939. return hex_fun(
  940. sign * int(value_s[2:], 16),
  941. width=width,
  942. underscore=underscore,
  943. anchor=node.anchor,
  944. )
  945. elif value_s.startswith('0o'):
  946. if self.resolver.processing_version > (1, 1) and value_s[2] == '0':
  947. width = len(value_s[2:])
  948. if underscore is not None:
  949. underscore[1] = value_su[2] == '_'
  950. underscore[2] = len(value_su[2:]) > 1 and value_su[-1] == '_'
  951. return OctalInt(
  952. sign * int(value_s[2:], 8),
  953. width=width,
  954. underscore=underscore,
  955. anchor=node.anchor,
  956. )
  957. elif (
  958. self.resolver.processing_version != (1, 2)
  959. and len(value_s) > 1
  960. and value_s[0] == '0'
  961. ):
  962. return OctalInt(
  963. sign * int(value_s, 8), width=width, underscore=underscore, anchor=node.anchor,
  964. )
  965. elif self.resolver.processing_version != (1, 2) and ':' in value_s:
  966. digits = [int(part) for part in value_s.split(':')]
  967. digits.reverse()
  968. base = 1
  969. value = 0
  970. for digit in digits:
  971. value += digit * base
  972. base *= 60
  973. return sign * value
  974. elif self.resolver.processing_version > (1, 1) and value_s[0] == '0':
  975. # not an octal, an integer with leading zero(s)
  976. if underscore is not None:
  977. # cannot have a leading underscore
  978. underscore[2] = len(value_su) > 1 and value_su[-1] == '_'
  979. return ScalarInt(
  980. sign * int(value_s),
  981. width=len(value_s),
  982. underscore=underscore,
  983. anchor=node.anchor,
  984. )
  985. elif underscore:
  986. # cannot have a leading underscore
  987. underscore[2] = len(value_su) > 1 and value_su[-1] == '_'
  988. return ScalarInt(
  989. sign * int(value_s), width=None, underscore=underscore, anchor=node.anchor,
  990. )
  991. elif node.anchor:
  992. return ScalarInt(sign * int(value_s), width=None, anchor=node.anchor)
  993. else:
  994. return sign * int(value_s)
  995. def construct_yaml_float(self, node: Any) -> Any:
  996. def leading_zeros(v: Any) -> int:
  997. lead0 = 0
  998. idx = 0
  999. while idx < len(v) and v[idx] in '0.':
  1000. if v[idx] == '0':
  1001. lead0 += 1
  1002. idx += 1
  1003. return lead0
  1004. # underscore = None
  1005. m_sign: Any = False
  1006. value_so = self.construct_scalar(node)
  1007. value_s = value_so.replace('_', "").lower()
  1008. sign = +1
  1009. if value_s[0] == '-':
  1010. sign = -1
  1011. if value_s[0] in '+-':
  1012. m_sign = value_s[0]
  1013. value_s = value_s[1:]
  1014. if value_s == '.inf':
  1015. return sign * self.inf_value
  1016. if value_s == '.nan':
  1017. return self.nan_value
  1018. if self.resolver.processing_version != (1, 2) and ':' in value_s:
  1019. digits = [float(part) for part in value_s.split(':')]
  1020. digits.reverse()
  1021. base = 1
  1022. value = 0.0
  1023. for digit in digits:
  1024. value += digit * base
  1025. base *= 60
  1026. return sign * value
  1027. if 'e' in value_s:
  1028. try:
  1029. mantissa, exponent = value_so.split('e')
  1030. exp = 'e'
  1031. except ValueError:
  1032. mantissa, exponent = value_so.split('E')
  1033. exp = 'E'
  1034. if self.resolver.processing_version != (1, 2):
  1035. # value_s is lower case independent of input
  1036. if '.' not in mantissa:
  1037. warnings.warn(MantissaNoDotYAML1_1Warning(node, value_so), stacklevel=1)
  1038. lead0 = leading_zeros(mantissa)
  1039. width = len(mantissa)
  1040. prec = mantissa.find('.')
  1041. if m_sign:
  1042. width -= 1
  1043. e_width = len(exponent)
  1044. e_sign = exponent[0] in '+-'
  1045. # nprint('sf', width, prec, m_sign, exp, e_width, e_sign)
  1046. return ScalarFloat(
  1047. sign * float(value_s),
  1048. width=width,
  1049. prec=prec,
  1050. m_sign=m_sign,
  1051. m_lead0=lead0,
  1052. exp=exp,
  1053. e_width=e_width,
  1054. e_sign=e_sign,
  1055. anchor=node.anchor,
  1056. )
  1057. width = len(value_so)
  1058. # you can't use index, !!float 42 would be a float without a dot
  1059. prec = value_so.find('.')
  1060. lead0 = leading_zeros(value_so)
  1061. return ScalarFloat(
  1062. sign * float(value_s),
  1063. width=width,
  1064. prec=prec,
  1065. m_sign=m_sign,
  1066. m_lead0=lead0,
  1067. anchor=node.anchor,
  1068. )
  1069. def construct_yaml_str(self, node: Any) -> Any:
  1070. if node.ctag.handle:
  1071. value = self.construct_unknown(node)
  1072. else:
  1073. value = self.construct_scalar(node)
  1074. if isinstance(value, ScalarString):
  1075. return value
  1076. return value
  1077. def construct_rt_sequence(self, node: Any, seqtyp: Any, deep: bool = False) -> Any:
  1078. if not isinstance(node, SequenceNode):
  1079. raise ConstructorError(
  1080. None,
  1081. None,
  1082. f'expected a sequence node, but found {node.id!s}',
  1083. node.start_mark,
  1084. )
  1085. ret_val = []
  1086. if self.loader and self.loader.comment_handling is None:
  1087. if node.comment:
  1088. seqtyp._yaml_add_comment(node.comment[:2])
  1089. if len(node.comment) > 2:
  1090. # this happens e.g. if you have a sequence element that is a flow-style
  1091. # mapping and that has no EOL comment but a following commentline or
  1092. # empty line
  1093. seqtyp.yaml_end_comment_extend(node.comment[2], clear=True)
  1094. else:
  1095. # NEWCMNT
  1096. if node.comment:
  1097. nprintf('nc3', node.comment)
  1098. if node.anchor:
  1099. from ruamel.yaml.serializer import templated_id
  1100. if not templated_id(node.anchor):
  1101. seqtyp.yaml_set_anchor(node.anchor)
  1102. for idx, child in enumerate(node.value):
  1103. if child.comment:
  1104. seqtyp._yaml_add_comment(child.comment, key=idx)
  1105. child.comment = None # if moved to sequence remove from child
  1106. ret_val.append(self.construct_object(child, deep=deep))
  1107. seqtyp._yaml_set_idx_line_col(
  1108. idx, [child.start_mark.line, child.start_mark.column],
  1109. )
  1110. return ret_val
  1111. def flatten_mapping(self, node: Any) -> Any: # RTConstructor
  1112. """
  1113. This implements the merge key feature http://yaml.org/type/merge.html
  1114. by referencing the merge dict/list of dicts
  1115. """
  1116. from ruamel.yaml.mergevalue import MergeValue
  1117. def constructed(value_node: Any) -> Any:
  1118. # If the contents of a merge are defined within the
  1119. # merge marker, then they won't have been constructed
  1120. # yet. But if they were already constructed, we need to use
  1121. # the existing object.
  1122. if value_node in self.constructed_objects:
  1123. value = self.constructed_objects[value_node]
  1124. else:
  1125. value = self.construct_object(value_node, deep=True)
  1126. return value
  1127. # merge = []
  1128. # merge_map_list: List[Any] = []
  1129. merge_map_list = MergeValue()
  1130. index = 0
  1131. while index < len(node.value):
  1132. key_node, value_node = node.value[index]
  1133. if key_node.tag == 'tag:yaml.org,2002:merge':
  1134. if len(merge_map_list): # double << key
  1135. args = [
  1136. 'while constructing a mapping',
  1137. node.start_mark,
  1138. 'found duplicate merge key "<<"',
  1139. key_node.start_mark,
  1140. """\
  1141. Duplicate merge keys are never allowed, not even when
  1142. `.allow_duplicate_keys` is set to True
  1143. """,
  1144. ]
  1145. raise DuplicateKeyError(*args)
  1146. del node.value[index]
  1147. merge_map_list.merge_pos = index
  1148. if isinstance(value_node, MappingNode):
  1149. merge_map_list.append(constructed(value_node))
  1150. elif isinstance(value_node, SequenceNode):
  1151. # submerge = []
  1152. merge_map_list.set_sequence(constructed(value_node))
  1153. for subnode in value_node.value:
  1154. if not isinstance(subnode, MappingNode):
  1155. raise ConstructorError(
  1156. 'while constructing a mapping',
  1157. node.start_mark,
  1158. f'expected a mapping for merging, but found {subnode.id!s}',
  1159. subnode.start_mark,
  1160. )
  1161. merge_map_list.append(constructed(subnode))
  1162. else:
  1163. raise ConstructorError(
  1164. 'while constructing a mapping',
  1165. node.start_mark,
  1166. 'expected a mapping or list of mappings for merging, '
  1167. f'but found {value_node.id!s}',
  1168. value_node.start_mark,
  1169. )
  1170. elif key_node.tag == 'tag:yaml.org,2002:value':
  1171. key_node.tag = 'tag:yaml.org,2002:str'
  1172. index += 1
  1173. else:
  1174. index += 1
  1175. return merge_map_list
  1176. # if merge:
  1177. # node.value = merge + node.value
  1178. def _sentinel(self) -> None:
  1179. pass
  1180. # RoundTrip
  1181. def construct_mapping(self, node: Any, maptyp: Any, deep: bool = False) -> Any: # type: ignore # NOQA
  1182. if not isinstance(node, MappingNode):
  1183. raise ConstructorError(
  1184. None, None, f'expected a mapping node, but found {node.id!s}', node.start_mark,
  1185. )
  1186. merge_map = self.flatten_mapping(node)
  1187. # mapping = {}
  1188. if self.loader and self.loader.comment_handling is None:
  1189. if node.comment:
  1190. maptyp._yaml_add_comment(node.comment[:2])
  1191. if len(node.comment) > 2:
  1192. maptyp.yaml_end_comment_extend(node.comment[2], clear=True)
  1193. else:
  1194. # NEWCMNT
  1195. if node.comment:
  1196. # nprintf('nc4', node.comment, node.start_mark)
  1197. if maptyp.ca.pre is None:
  1198. maptyp.ca.pre = []
  1199. for cmnt in self.comments(node.comment, 0):
  1200. maptyp.ca.pre.append(cmnt)
  1201. if node.anchor:
  1202. from ruamel.yaml.serializer import templated_id
  1203. if not templated_id(node.anchor):
  1204. maptyp.yaml_set_anchor(node.anchor)
  1205. last_key, last_value = None, self._sentinel
  1206. for key_node, value_node in node.value:
  1207. # keys can be list -> deep
  1208. key = self.construct_object(key_node, deep=True)
  1209. # lists are not hashable, but tuples are
  1210. if not isinstance(key, Hashable):
  1211. if isinstance(key, MutableSequence):
  1212. key_s = CommentedKeySeq(key)
  1213. if key_node.flow_style is True:
  1214. key_s.fa.set_flow_style()
  1215. elif key_node.flow_style is False:
  1216. key_s.fa.set_block_style()
  1217. key_s._yaml_set_line_col(key.lc.line, key.lc.col) # type: ignore
  1218. key = key_s
  1219. elif isinstance(key, MutableMapping):
  1220. key_m = CommentedKeyMap(key)
  1221. if key_node.flow_style is True:
  1222. key_m.fa.set_flow_style()
  1223. elif key_node.flow_style is False:
  1224. key_m.fa.set_block_style()
  1225. key_m._yaml_set_line_col(key.lc.line, key.lc.col) # type: ignore
  1226. key = key_m
  1227. if not isinstance(key, Hashable):
  1228. raise ConstructorError(
  1229. 'while constructing a mapping',
  1230. node.start_mark,
  1231. 'found unhashable key',
  1232. key_node.start_mark,
  1233. )
  1234. value = self.construct_object(value_node, deep=deep)
  1235. if self.check_mapping_key(node, key_node, maptyp, key, value):
  1236. if self.loader and self.loader.comment_handling is None:
  1237. if key_node.comment and len(key_node.comment) > 4 and key_node.comment[4]:
  1238. if last_value is None:
  1239. key_node.comment[0] = key_node.comment.pop(4)
  1240. maptyp._yaml_add_comment(key_node.comment, value=last_key)
  1241. else:
  1242. key_node.comment[2] = key_node.comment.pop(4)
  1243. maptyp._yaml_add_comment(key_node.comment, key=key)
  1244. key_node.comment = None
  1245. if key_node.comment:
  1246. maptyp._yaml_add_comment(key_node.comment, key=key)
  1247. if value_node.comment:
  1248. maptyp._yaml_add_comment(value_node.comment, value=key)
  1249. else:
  1250. # NEWCMNT
  1251. if key_node.comment:
  1252. # nprintf('nc5a', key, key_node.comment)
  1253. if key_node.comment[0]:
  1254. maptyp.ca.set(key, C_KEY_PRE, key_node.comment[0])
  1255. if key_node.comment[1]:
  1256. maptyp.ca.set(key, C_KEY_EOL, key_node.comment[1])
  1257. if key_node.comment[2]:
  1258. maptyp.ca.set(key, C_KEY_POST, key_node.comment[2])
  1259. if value_node.comment:
  1260. nprintf('nc5b', key, value_node.comment)
  1261. if value_node.comment[0]:
  1262. maptyp.ca.set(key, C_VALUE_PRE, value_node.comment[0])
  1263. if value_node.comment[1]:
  1264. maptyp.ca.set(key, C_VALUE_EOL, value_node.comment[1])
  1265. if value_node.comment[2]:
  1266. maptyp.ca.set(key, C_VALUE_POST, value_node.comment[2])
  1267. maptyp._yaml_set_kv_line_col(
  1268. key,
  1269. [
  1270. key_node.start_mark.line,
  1271. key_node.start_mark.column,
  1272. value_node.start_mark.line,
  1273. value_node.start_mark.column,
  1274. ],
  1275. )
  1276. maptyp[key] = value
  1277. last_key, last_value = key, value # could use indexing
  1278. # do this last, or <<: before a key will prevent insertion in instances
  1279. # of collections.OrderedDict (as they have no __contains__
  1280. if merge_map:
  1281. maptyp.add_yaml_merge(merge_map)
  1282. def construct_setting(self, node: Any, typ: Any, deep: bool = False) -> Any:
  1283. if not isinstance(node, MappingNode):
  1284. raise ConstructorError(
  1285. None, None, f'expected a mapping node, but found {node.id!s}', node.start_mark,
  1286. )
  1287. if self.loader and self.loader.comment_handling is None:
  1288. if node.comment:
  1289. typ._yaml_add_comment(node.comment[:2])
  1290. if len(node.comment) > 2:
  1291. typ.yaml_end_comment_extend(node.comment[2], clear=True)
  1292. else:
  1293. # NEWCMNT
  1294. if node.comment:
  1295. nprintf('nc6', node.comment)
  1296. if node.anchor:
  1297. from ruamel.yaml.serializer import templated_id
  1298. if not templated_id(node.anchor):
  1299. typ.yaml_set_anchor(node.anchor)
  1300. for key_node, value_node in node.value:
  1301. # keys can be list -> deep
  1302. key = self.construct_object(key_node, deep=True)
  1303. # lists are not hashable, but tuples are
  1304. if not isinstance(key, Hashable):
  1305. if isinstance(key, list):
  1306. key = tuple(key)
  1307. if not isinstance(key, Hashable):
  1308. raise ConstructorError(
  1309. 'while constructing a mapping',
  1310. node.start_mark,
  1311. 'found unhashable key',
  1312. key_node.start_mark,
  1313. )
  1314. # construct but should be null
  1315. value = self.construct_object(value_node, deep=deep) # NOQA
  1316. self.check_set_key(node, key_node, typ, key)
  1317. if self.loader and self.loader.comment_handling is None:
  1318. if key_node.comment:
  1319. typ._yaml_add_comment(key_node.comment, key=key)
  1320. if value_node.comment:
  1321. typ._yaml_add_comment(value_node.comment, value=key)
  1322. else:
  1323. # NEWCMNT
  1324. if key_node.comment:
  1325. nprintf('nc7a', key_node.comment)
  1326. if value_node.comment:
  1327. nprintf('nc7b', value_node.comment)
  1328. typ.add(key)
  1329. def construct_yaml_seq(self, node: Any) -> Iterator[CommentedSeq]:
  1330. data = CommentedSeq()
  1331. data._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1332. # if node.comment:
  1333. # data._yaml_add_comment(node.comment)
  1334. yield data
  1335. data.extend(self.construct_rt_sequence(node, data))
  1336. self.set_collection_style(data, node)
  1337. def construct_yaml_map(self, node: Any) -> Iterator[CommentedMap]:
  1338. data = CommentedMap()
  1339. data._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1340. yield data
  1341. self.construct_mapping(node, data, deep=True)
  1342. self.set_collection_style(data, node)
  1343. def set_collection_style(self, data: Any, node: Any) -> None:
  1344. if len(data) == 0:
  1345. return
  1346. if node.flow_style is True:
  1347. data.fa.set_flow_style()
  1348. elif node.flow_style is False:
  1349. data.fa.set_block_style()
  1350. def construct_yaml_object(self, node: Any, cls: Any) -> Any:
  1351. from dataclasses import is_dataclass, InitVar, MISSING
  1352. data = cls.__new__(cls)
  1353. yield data
  1354. if hasattr(data, '__setstate__'):
  1355. state = SafeConstructor.construct_mapping(self, node, deep=True)
  1356. data.__setstate__(state)
  1357. elif is_dataclass(data):
  1358. mapping = SafeConstructor.construct_mapping(self, node, deep=True)
  1359. init_var_defaults = {}
  1360. for field in data.__dataclass_fields__.values():
  1361. # nprintf('field', field, field.default is MISSING, isinstance(field.type, InitVar)) # NOQA
  1362. # in 3.7, InitVar is a singleton
  1363. if (
  1364. isinstance(field.type, InitVar)
  1365. or field.type is InitVar
  1366. # this following is for handling from __future__ import allocations
  1367. or (isinstance(field.type, str) and field.type.startswith('InitVar'))
  1368. ) and field.default is not MISSING:
  1369. init_var_defaults[field.name] = field.default
  1370. for attr, value in mapping.items():
  1371. if attr not in init_var_defaults:
  1372. setattr(data, attr, value)
  1373. post_init = getattr(data, '__post_init__', None)
  1374. if post_init is not None:
  1375. kw = {}
  1376. for name, default in init_var_defaults.items():
  1377. kw[name] = mapping.get(name, default)
  1378. post_init(**kw)
  1379. for field in data.__dataclass_fields__.values():
  1380. if field.name not in mapping and field.default_factory is not MISSING:
  1381. setattr(data, field.name, field.default_factory())
  1382. else:
  1383. state = SafeConstructor.construct_mapping(self, node)
  1384. if hasattr(data, '__attrs_attrs__'): # issue 394
  1385. data.__init__(**state)
  1386. else:
  1387. data.__dict__.update(state)
  1388. if node.anchor:
  1389. from ruamel.yaml.serializer import templated_id
  1390. from ruamel.yaml.anchor import Anchor
  1391. if not templated_id(node.anchor):
  1392. if not hasattr(data, Anchor.attrib):
  1393. a = Anchor()
  1394. setattr(data, Anchor.attrib, a)
  1395. else:
  1396. a = getattr(data, Anchor.attrib)
  1397. a.value = node.anchor
  1398. def construct_yaml_omap(self, node: Any) -> Iterator[CommentedOrderedMap]:
  1399. # Note: we do now check for duplicate keys
  1400. omap = CommentedOrderedMap()
  1401. omap._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1402. if node.flow_style is True:
  1403. omap.fa.set_flow_style()
  1404. elif node.flow_style is False:
  1405. omap.fa.set_block_style()
  1406. yield omap
  1407. if self.loader and self.loader.comment_handling is None:
  1408. if node.comment:
  1409. omap._yaml_add_comment(node.comment[:2])
  1410. if len(node.comment) > 2:
  1411. omap.yaml_end_comment_extend(node.comment[2], clear=True)
  1412. else:
  1413. # NEWCMNT
  1414. if node.comment:
  1415. nprintf('nc8', node.comment)
  1416. if not isinstance(node, SequenceNode):
  1417. raise ConstructorError(
  1418. 'while constructing an ordered map',
  1419. node.start_mark,
  1420. f'expected a sequence, but found {node.id!s}',
  1421. node.start_mark,
  1422. )
  1423. for subnode in node.value:
  1424. if not isinstance(subnode, MappingNode):
  1425. raise ConstructorError(
  1426. 'while constructing an ordered map',
  1427. node.start_mark,
  1428. f'expected a mapping of length 1, but found {subnode.id!s}',
  1429. subnode.start_mark,
  1430. )
  1431. if len(subnode.value) != 1:
  1432. raise ConstructorError(
  1433. 'while constructing an ordered map',
  1434. node.start_mark,
  1435. f'expected a single mapping item, but found {len(subnode.value):d} items',
  1436. subnode.start_mark,
  1437. )
  1438. key_node, value_node = subnode.value[0]
  1439. key = self.construct_object(key_node)
  1440. assert key not in omap
  1441. value = self.construct_object(value_node)
  1442. if self.loader and self.loader.comment_handling is None:
  1443. if key_node.comment:
  1444. omap._yaml_add_comment(key_node.comment, key=key)
  1445. if subnode.comment:
  1446. omap._yaml_add_comment(subnode.comment, key=key)
  1447. if value_node.comment:
  1448. omap._yaml_add_comment(value_node.comment, value=key)
  1449. else:
  1450. # NEWCMNT
  1451. if key_node.comment:
  1452. nprintf('nc9a', key_node.comment)
  1453. if subnode.comment:
  1454. nprintf('nc9b', subnode.comment)
  1455. if value_node.comment:
  1456. nprintf('nc9c', value_node.comment)
  1457. omap[key] = value
  1458. def construct_yaml_set(self, node: Any) -> Iterator[CommentedSet]:
  1459. data = CommentedSet()
  1460. data._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1461. if node.flow_style is True:
  1462. data.fa.set_flow_style()
  1463. elif node.flow_style is False:
  1464. data.fa.set_block_style()
  1465. yield data
  1466. self.construct_setting(node, data)
  1467. def construct_unknown(
  1468. self, node: Any,
  1469. ) -> Iterator[Union[CommentedMap, TaggedScalar, CommentedSeq]]:
  1470. try:
  1471. if isinstance(node, MappingNode):
  1472. data = CommentedMap()
  1473. data._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1474. if node.flow_style is True:
  1475. data.fa.set_flow_style()
  1476. elif node.flow_style is False:
  1477. data.fa.set_block_style()
  1478. data.yaml_set_ctag(node.ctag)
  1479. yield data
  1480. if node.anchor:
  1481. from ruamel.yaml.serializer import templated_id
  1482. if not templated_id(node.anchor):
  1483. data.yaml_set_anchor(node.anchor)
  1484. self.construct_mapping(node, data)
  1485. return
  1486. elif isinstance(node, ScalarNode):
  1487. data2 = TaggedScalar()
  1488. data2.value = self.construct_scalar(node)
  1489. data2.style = node.style
  1490. data2.yaml_set_ctag(node.ctag)
  1491. yield data2
  1492. if node.anchor:
  1493. from ruamel.yaml.serializer import templated_id
  1494. if not templated_id(node.anchor):
  1495. data2.yaml_set_anchor(node.anchor, always_dump=True)
  1496. return
  1497. elif isinstance(node, SequenceNode):
  1498. data3 = CommentedSeq()
  1499. data3._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1500. if node.flow_style is True:
  1501. data3.fa.set_flow_style()
  1502. elif node.flow_style is False:
  1503. data3.fa.set_block_style()
  1504. data3.yaml_set_ctag(node.ctag)
  1505. yield data3
  1506. if node.anchor:
  1507. from ruamel.yaml.serializer import templated_id
  1508. if not templated_id(node.anchor):
  1509. data3.yaml_set_anchor(node.anchor)
  1510. data3.extend(self.construct_sequence(node))
  1511. return
  1512. except: # NOQA
  1513. pass
  1514. raise ConstructorError(
  1515. None,
  1516. None,
  1517. f'could not determine a constructor for the tag {node.tag!r}',
  1518. node.start_mark,
  1519. )
  1520. def construct_yaml_timestamp(
  1521. self, node: Any, values: Any = None,
  1522. ) -> Union[datetime.date, datetime.datetime, TimeStamp]:
  1523. try:
  1524. match = self.timestamp_regexp.match(node.value)
  1525. except TypeError:
  1526. match = None
  1527. if match is None:
  1528. raise ConstructorError(
  1529. None,
  1530. None,
  1531. f'failed to construct timestamp from "{node.value}"',
  1532. node.start_mark,
  1533. )
  1534. values = match.groupdict()
  1535. if not values['hour']:
  1536. return create_timestamp(**values)
  1537. # return SafeConstructor.construct_yaml_timestamp(self, node, values)
  1538. for part in ['t', 'tz_sign', 'tz_hour', 'tz_minute']:
  1539. if values[part]:
  1540. break
  1541. else:
  1542. return create_timestamp(**values)
  1543. # return SafeConstructor.construct_yaml_timestamp(self, node, values)
  1544. # print('>>>>>>>> here', values)
  1545. dd = create_timestamp(**values) # this has tzinfo
  1546. delta = None
  1547. if values['tz_sign']:
  1548. hours = values['tz_hour']
  1549. tz_hour = int(hours)
  1550. minutes = values['tz_minute']
  1551. tz_minute = int(minutes) if minutes else 0
  1552. # ToDo: double work, replace with extraction from dd.tzinfo
  1553. delta = TimeDelta(hours=tz_hour, minutes=tz_minute)
  1554. if values['tz_sign'] == '-':
  1555. delta = -delta
  1556. if isinstance(dd, datetime.datetime):
  1557. data = TimeStamp(
  1558. dd.year, dd.month, dd.day, dd.hour, dd.minute, dd.second, dd.microsecond, dd.tzinfo, # NOQA
  1559. )
  1560. else:
  1561. # ToDo: make this into a DateStamp?
  1562. data = TimeStamp(dd.year, dd.month, dd.day, 0, 0, 0, 0)
  1563. return data
  1564. if delta:
  1565. data._yaml['delta'] = delta
  1566. tz = values['tz_sign'] + values['tz_hour']
  1567. if values['tz_minute']:
  1568. tz += ':' + values['tz_minute']
  1569. data._yaml['tz'] = tz
  1570. else:
  1571. if values['tz']: # no delta
  1572. data._yaml['tz'] = values['tz']
  1573. if values['t']:
  1574. data._yaml['t'] = True
  1575. return data
  1576. def construct_yaml_sbool(self, node: Any) -> Union[bool, ScalarBoolean]:
  1577. b = SafeConstructor.construct_yaml_bool(self, node)
  1578. if node.anchor:
  1579. return ScalarBoolean(b, anchor=node.anchor)
  1580. return b
  1581. RoundTripConstructor.add_default_constructor('bool', method='construct_yaml_sbool')
  1582. for tag in 'null int float binary timestamp omap pairs set str seq map'.split():
  1583. RoundTripConstructor.add_default_constructor(tag)
  1584. RoundTripConstructor.add_constructor(None, RoundTripConstructor.construct_unknown)