json_format.py 37 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111
  1. # Protocol Buffers - Google's data interchange format
  2. # Copyright 2008 Google Inc. All rights reserved.
  3. #
  4. # Use of this source code is governed by a BSD-style
  5. # license that can be found in the LICENSE file or at
  6. # https://developers.google.com/open-source/licenses/bsd
  7. """Contains routines for printing protocol messages in JSON format.
  8. Simple usage example:
  9. # Create a proto object and serialize it to a json format string.
  10. message = my_proto_pb2.MyMessage(foo='bar')
  11. json_string = json_format.MessageToJson(message)
  12. # Parse a json format string to proto object.
  13. message = json_format.Parse(json_string, my_proto_pb2.MyMessage())
  14. """
  15. __author__ = 'jieluo@google.com (Jie Luo)'
  16. import base64
  17. from collections import OrderedDict
  18. import json
  19. import math
  20. from operator import methodcaller
  21. import re
  22. import warnings
  23. from google.protobuf import descriptor
  24. from google.protobuf import message_factory
  25. from google.protobuf import symbol_database
  26. from google.protobuf.internal import type_checkers
  27. _INT_TYPES = frozenset([
  28. descriptor.FieldDescriptor.CPPTYPE_INT32,
  29. descriptor.FieldDescriptor.CPPTYPE_UINT32,
  30. descriptor.FieldDescriptor.CPPTYPE_INT64,
  31. descriptor.FieldDescriptor.CPPTYPE_UINT64,
  32. ])
  33. _INT64_TYPES = frozenset([
  34. descriptor.FieldDescriptor.CPPTYPE_INT64,
  35. descriptor.FieldDescriptor.CPPTYPE_UINT64,
  36. ])
  37. _FLOAT_TYPES = frozenset([
  38. descriptor.FieldDescriptor.CPPTYPE_FLOAT,
  39. descriptor.FieldDescriptor.CPPTYPE_DOUBLE,
  40. ])
  41. _INFINITY = 'Infinity'
  42. _NEG_INFINITY = '-Infinity'
  43. _NAN = 'NaN'
  44. _UNPAIRED_SURROGATE_PATTERN = re.compile(
  45. '[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]'
  46. )
  47. _VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$')
  48. class Error(Exception):
  49. """Top-level module error for json_format."""
  50. class SerializeToJsonError(Error):
  51. """Thrown if serialization to JSON fails."""
  52. class ParseError(Error):
  53. """Thrown in case of parsing error."""
  54. class EnumStringValueParseError(ParseError):
  55. """Thrown if unknown string enum value is encountered.
  56. This exception is suppressed if ignore_unknown_fields is set.
  57. """
  58. def MessageToJson(
  59. message,
  60. preserving_proto_field_name=False,
  61. indent=2,
  62. sort_keys=False,
  63. use_integers_for_enums=False,
  64. descriptor_pool=None,
  65. float_precision=None,
  66. ensure_ascii=True,
  67. always_print_fields_with_no_presence=False,
  68. ):
  69. """Converts protobuf message to JSON format.
  70. Args:
  71. message: The protocol buffers message instance to serialize.
  72. always_print_fields_with_no_presence: If True, fields without presence
  73. (implicit presence scalars, repeated fields, and map fields) will always
  74. be serialized. Any field that supports presence is not affected by this
  75. option (including singular message fields and oneof fields).
  76. preserving_proto_field_name: If True, use the original proto field names as
  77. defined in the .proto file. If False, convert the field names to
  78. lowerCamelCase.
  79. indent: The JSON object will be pretty-printed with this indent level. An
  80. indent level of 0 or negative will only insert newlines. If the indent
  81. level is None, no newlines will be inserted.
  82. sort_keys: If True, then the output will be sorted by field names.
  83. use_integers_for_enums: If true, print integers instead of enum names.
  84. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  85. default.
  86. float_precision: Deprecated. If set, use this to specify float field valid
  87. digits.
  88. ensure_ascii: If True, strings with non-ASCII characters are escaped. If
  89. False, Unicode strings are returned unchanged.
  90. Returns:
  91. A string containing the JSON formatted protocol buffer message.
  92. """
  93. printer = _Printer(
  94. preserving_proto_field_name,
  95. use_integers_for_enums,
  96. descriptor_pool,
  97. float_precision,
  98. always_print_fields_with_no_presence,
  99. )
  100. return printer.ToJsonString(message, indent, sort_keys, ensure_ascii)
  101. def MessageToDict(
  102. message,
  103. always_print_fields_with_no_presence=False,
  104. preserving_proto_field_name=False,
  105. use_integers_for_enums=False,
  106. descriptor_pool=None,
  107. float_precision=None,
  108. ):
  109. """Converts protobuf message to a dictionary.
  110. When the dictionary is encoded to JSON, it conforms to proto3 JSON spec.
  111. Args:
  112. message: The protocol buffers message instance to serialize.
  113. always_print_fields_with_no_presence: If True, fields without presence
  114. (implicit presence scalars, repeated fields, and map fields) will always
  115. be serialized. Any field that supports presence is not affected by this
  116. option (including singular message fields and oneof fields).
  117. preserving_proto_field_name: If True, use the original proto field names as
  118. defined in the .proto file. If False, convert the field names to
  119. lowerCamelCase.
  120. use_integers_for_enums: If true, print integers instead of enum names.
  121. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  122. default.
  123. float_precision: Deprecated. If set, use this to specify float field valid
  124. digits.
  125. Returns:
  126. A dict representation of the protocol buffer message.
  127. """
  128. printer = _Printer(
  129. preserving_proto_field_name,
  130. use_integers_for_enums,
  131. descriptor_pool,
  132. float_precision,
  133. always_print_fields_with_no_presence,
  134. )
  135. # pylint: disable=protected-access
  136. return printer._MessageToJsonObject(message)
  137. def _IsMapEntry(field):
  138. return (
  139. field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
  140. and field.message_type.has_options
  141. and field.message_type.GetOptions().map_entry
  142. )
  143. class _Printer(object):
  144. """JSON format printer for protocol message."""
  145. def __init__(
  146. self,
  147. preserving_proto_field_name=False,
  148. use_integers_for_enums=False,
  149. descriptor_pool=None,
  150. float_precision=None,
  151. always_print_fields_with_no_presence=False,
  152. ):
  153. self.always_print_fields_with_no_presence = (
  154. always_print_fields_with_no_presence
  155. )
  156. self.preserving_proto_field_name = preserving_proto_field_name
  157. self.use_integers_for_enums = use_integers_for_enums
  158. self.descriptor_pool = descriptor_pool
  159. if float_precision:
  160. warnings.warn(
  161. 'float_precision option is deprecated for json_format. '
  162. 'This will turn into error in 7.34.0, please remove it '
  163. 'before that.'
  164. )
  165. self.float_format = '.{}g'.format(float_precision)
  166. else:
  167. self.float_format = None
  168. def ToJsonString(self, message, indent, sort_keys, ensure_ascii):
  169. js = self._MessageToJsonObject(message)
  170. return json.dumps(
  171. js, indent=indent, sort_keys=sort_keys, ensure_ascii=ensure_ascii
  172. )
  173. def _MessageToJsonObject(self, message):
  174. """Converts message to an object according to Proto3 JSON Specification."""
  175. message_descriptor = message.DESCRIPTOR
  176. full_name = message_descriptor.full_name
  177. if _IsWrapperMessage(message_descriptor):
  178. return self._WrapperMessageToJsonObject(message)
  179. if full_name in _WKTJSONMETHODS:
  180. return methodcaller(_WKTJSONMETHODS[full_name][0], message)(self)
  181. js = {}
  182. return self._RegularMessageToJsonObject(message, js)
  183. def _RegularMessageToJsonObject(self, message, js):
  184. """Converts normal message according to Proto3 JSON Specification."""
  185. fields = message.ListFields()
  186. try:
  187. for field, value in fields:
  188. if field.is_extension:
  189. name = '[%s]' % field.full_name
  190. elif self.preserving_proto_field_name:
  191. name = field.name
  192. else:
  193. name = field.json_name
  194. if _IsMapEntry(field):
  195. # Convert a map field.
  196. v_field = field.message_type.fields_by_name['value']
  197. js_map = {}
  198. for key in value:
  199. if isinstance(key, bool):
  200. if key:
  201. recorded_key = 'true'
  202. else:
  203. recorded_key = 'false'
  204. else:
  205. recorded_key = str(key)
  206. js_map[recorded_key] = self._FieldToJsonObject(v_field, value[key])
  207. js[name] = js_map
  208. elif field.is_repeated:
  209. # Convert a repeated field.
  210. js[name] = [self._FieldToJsonObject(field, k) for k in value]
  211. else:
  212. js[name] = self._FieldToJsonObject(field, value)
  213. # Serialize default value if including_default_value_fields is True.
  214. if (
  215. self.always_print_fields_with_no_presence
  216. ):
  217. message_descriptor = message.DESCRIPTOR
  218. for field in message_descriptor.fields:
  219. # always_print_fields_with_no_presence doesn't apply to
  220. # any field which supports presence.
  221. if self.always_print_fields_with_no_presence and field.has_presence:
  222. continue
  223. if self.preserving_proto_field_name:
  224. name = field.name
  225. else:
  226. name = field.json_name
  227. if name in js:
  228. # Skip the field which has been serialized already.
  229. continue
  230. if _IsMapEntry(field):
  231. js[name] = {}
  232. elif field.is_repeated:
  233. js[name] = []
  234. else:
  235. js[name] = self._FieldToJsonObject(field, field.default_value)
  236. except ValueError as e:
  237. raise SerializeToJsonError(
  238. 'Failed to serialize {0} field: {1}.'.format(field.name, e)
  239. ) from e
  240. return js
  241. def _FieldToJsonObject(self, field, value):
  242. """Converts field value according to Proto3 JSON Specification."""
  243. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  244. return self._MessageToJsonObject(value)
  245. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
  246. if self.use_integers_for_enums:
  247. return value
  248. if field.enum_type.full_name == 'google.protobuf.NullValue':
  249. return None
  250. enum_value = field.enum_type.values_by_number.get(value, None)
  251. if enum_value is not None:
  252. return enum_value.name
  253. else:
  254. if field.enum_type.is_closed:
  255. raise SerializeToJsonError(
  256. 'Enum field contains an integer value '
  257. 'which can not mapped to an enum value.'
  258. )
  259. else:
  260. return value
  261. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
  262. if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
  263. # Use base64 Data encoding for bytes
  264. return base64.b64encode(value).decode('utf-8')
  265. else:
  266. return str(value)
  267. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
  268. return bool(value)
  269. elif field.cpp_type in _INT64_TYPES:
  270. return str(value)
  271. elif field.cpp_type in _FLOAT_TYPES:
  272. if math.isinf(value):
  273. if value < 0.0:
  274. return _NEG_INFINITY
  275. else:
  276. return _INFINITY
  277. if math.isnan(value):
  278. return _NAN
  279. if self.float_format:
  280. return float(format(value, self.float_format))
  281. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
  282. return type_checkers.ToShortestFloat(value)
  283. return value
  284. def _AnyMessageToJsonObject(self, message):
  285. """Converts Any message according to Proto3 JSON Specification."""
  286. if not message.ListFields():
  287. return {}
  288. # Must print @type first, use OrderedDict instead of {}
  289. js = OrderedDict()
  290. type_url = message.type_url
  291. js['@type'] = type_url
  292. sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
  293. sub_message.ParseFromString(message.value)
  294. message_descriptor = sub_message.DESCRIPTOR
  295. full_name = message_descriptor.full_name
  296. if _IsWrapperMessage(message_descriptor):
  297. js['value'] = self._WrapperMessageToJsonObject(sub_message)
  298. return js
  299. if full_name in _WKTJSONMETHODS:
  300. js['value'] = methodcaller(_WKTJSONMETHODS[full_name][0], sub_message)(
  301. self
  302. )
  303. return js
  304. return self._RegularMessageToJsonObject(sub_message, js)
  305. def _GenericMessageToJsonObject(self, message):
  306. """Converts message according to Proto3 JSON Specification."""
  307. # Duration, Timestamp and FieldMask have ToJsonString method to do the
  308. # convert. Users can also call the method directly.
  309. return message.ToJsonString()
  310. def _ValueMessageToJsonObject(self, message):
  311. """Converts Value message according to Proto3 JSON Specification."""
  312. which = message.WhichOneof('kind')
  313. # If the Value message is not set treat as null_value when serialize
  314. # to JSON. The parse back result will be different from original message.
  315. if which is None or which == 'null_value':
  316. return None
  317. if which == 'list_value':
  318. return self._ListValueMessageToJsonObject(message.list_value)
  319. if which == 'number_value':
  320. value = message.number_value
  321. if math.isinf(value):
  322. raise ValueError(
  323. 'Fail to serialize Infinity for Value.number_value, '
  324. 'which would parse as string_value'
  325. )
  326. if math.isnan(value):
  327. raise ValueError(
  328. 'Fail to serialize NaN for Value.number_value, '
  329. 'which would parse as string_value'
  330. )
  331. else:
  332. value = getattr(message, which)
  333. oneof_descriptor = message.DESCRIPTOR.fields_by_name[which]
  334. return self._FieldToJsonObject(oneof_descriptor, value)
  335. def _ListValueMessageToJsonObject(self, message):
  336. """Converts ListValue message according to Proto3 JSON Specification."""
  337. return [self._ValueMessageToJsonObject(value) for value in message.values]
  338. def _StructMessageToJsonObject(self, message):
  339. """Converts Struct message according to Proto3 JSON Specification."""
  340. fields = message.fields
  341. ret = {}
  342. for key in fields:
  343. ret[key] = self._ValueMessageToJsonObject(fields[key])
  344. return ret
  345. def _WrapperMessageToJsonObject(self, message):
  346. return self._FieldToJsonObject(
  347. message.DESCRIPTOR.fields_by_name['value'], message.value
  348. )
  349. def _IsWrapperMessage(message_descriptor):
  350. return message_descriptor.file.name == 'google/protobuf/wrappers.proto'
  351. def _DuplicateChecker(js):
  352. result = {}
  353. for name, value in js:
  354. if name in result:
  355. raise ParseError('Failed to load JSON: duplicate key {0}.'.format(name))
  356. result[name] = value
  357. return result
  358. def _CreateMessageFromTypeUrl(type_url, descriptor_pool):
  359. """Creates a message from a type URL."""
  360. db = symbol_database.Default()
  361. pool = db.pool if descriptor_pool is None else descriptor_pool
  362. type_name = type_url.split('/')[-1]
  363. try:
  364. message_descriptor = pool.FindMessageTypeByName(type_name)
  365. except KeyError as e:
  366. raise TypeError(
  367. 'Can not find message descriptor by type_url: {0}'.format(type_url)
  368. ) from e
  369. message_class = message_factory.GetMessageClass(message_descriptor)
  370. return message_class()
  371. def Parse(
  372. text,
  373. message,
  374. ignore_unknown_fields=False,
  375. descriptor_pool=None,
  376. max_recursion_depth=100,
  377. ):
  378. """Parses a JSON representation of a protocol message into a message.
  379. Args:
  380. text: Message JSON representation.
  381. message: A protocol buffer message to merge into.
  382. ignore_unknown_fields: If True, do not raise errors for unknown fields.
  383. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  384. default.
  385. max_recursion_depth: max recursion depth of JSON message to be deserialized.
  386. JSON messages over this depth will fail to be deserialized. Default value
  387. is 100.
  388. Returns:
  389. The same message passed as argument.
  390. Raises::
  391. ParseError: On JSON parsing problems.
  392. """
  393. if not isinstance(text, str):
  394. text = text.decode('utf-8')
  395. try:
  396. js = json.loads(text, object_pairs_hook=_DuplicateChecker)
  397. except Exception as e:
  398. raise ParseError('Failed to load JSON: {0}.'.format(str(e))) from e
  399. try:
  400. return ParseDict(
  401. js, message, ignore_unknown_fields, descriptor_pool, max_recursion_depth
  402. )
  403. except ParseError as e:
  404. raise e
  405. except Exception as e:
  406. raise ParseError(
  407. 'Failed to parse JSON: {0}: {1}.'.format(type(e).__name__, str(e))
  408. ) from e
  409. def ParseDict(
  410. js_dict,
  411. message,
  412. ignore_unknown_fields=False,
  413. descriptor_pool=None,
  414. max_recursion_depth=100,
  415. ):
  416. """Parses a JSON dictionary representation into a message.
  417. Args:
  418. js_dict: Dict representation of a JSON message.
  419. message: A protocol buffer message to merge into.
  420. ignore_unknown_fields: If True, do not raise errors for unknown fields.
  421. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  422. default.
  423. max_recursion_depth: max recursion depth of JSON message to be deserialized.
  424. JSON messages over this depth will fail to be deserialized. Default value
  425. is 100.
  426. Returns:
  427. The same message passed as argument.
  428. """
  429. parser = _Parser(ignore_unknown_fields, descriptor_pool, max_recursion_depth)
  430. parser.ConvertMessage(js_dict, message, '')
  431. return message
  432. _INT_OR_FLOAT = (int, float)
  433. _LIST_LIKE = (list, tuple)
  434. class _Parser(object):
  435. """JSON format parser for protocol message."""
  436. def __init__(
  437. self, ignore_unknown_fields, descriptor_pool, max_recursion_depth
  438. ):
  439. self.ignore_unknown_fields = ignore_unknown_fields
  440. self.descriptor_pool = descriptor_pool
  441. self.max_recursion_depth = max_recursion_depth
  442. self.recursion_depth = 0
  443. def ConvertMessage(self, value, message, path):
  444. """Convert a JSON object into a message.
  445. Args:
  446. value: A JSON object.
  447. message: A WKT or regular protocol message to record the data.
  448. path: parent path to log parse error info.
  449. Raises:
  450. ParseError: In case of convert problems.
  451. """
  452. self.recursion_depth += 1
  453. if self.recursion_depth > self.max_recursion_depth:
  454. raise ParseError(
  455. 'Message too deep. Max recursion depth is {0}'.format(
  456. self.max_recursion_depth
  457. )
  458. )
  459. message_descriptor = message.DESCRIPTOR
  460. full_name = message_descriptor.full_name
  461. if not path:
  462. path = message_descriptor.name
  463. if _IsWrapperMessage(message_descriptor):
  464. self._ConvertWrapperMessage(value, message, path)
  465. elif full_name in _WKTJSONMETHODS:
  466. methodcaller(_WKTJSONMETHODS[full_name][1], value, message, path)(self)
  467. else:
  468. self._ConvertFieldValuePair(value, message, path)
  469. self.recursion_depth -= 1
  470. def _ConvertFieldValuePair(self, js, message, path):
  471. """Convert field value pairs into regular message.
  472. Args:
  473. js: A JSON object to convert the field value pairs.
  474. message: A regular protocol message to record the data.
  475. path: parent path to log parse error info.
  476. Raises:
  477. ParseError: In case of problems converting.
  478. """
  479. names = []
  480. message_descriptor = message.DESCRIPTOR
  481. fields_by_json_name = dict(
  482. (f.json_name, f) for f in message_descriptor.fields
  483. )
  484. def _ClearFieldOrExtension(message, field):
  485. if field.is_extension:
  486. message.ClearExtension(field)
  487. else:
  488. message.ClearField(field.name)
  489. def _GetFieldOrExtension(message, field):
  490. if field.is_extension:
  491. return message.Extensions[field]
  492. else:
  493. return getattr(message, field.name)
  494. def _SetFieldOrExtension(message, field, value):
  495. if field.is_extension:
  496. message.Extensions[field] = value
  497. else:
  498. setattr(message, field.name, value)
  499. for name in js:
  500. try:
  501. field = fields_by_json_name.get(name, None)
  502. if not field:
  503. field = message_descriptor.fields_by_name.get(name, None)
  504. if not field and _VALID_EXTENSION_NAME.match(name):
  505. if not message_descriptor.is_extendable:
  506. raise ParseError(
  507. 'Message type {0} does not have extensions at {1}'.format(
  508. message_descriptor.full_name, path
  509. )
  510. )
  511. identifier = name[1:-1] # strip [] brackets
  512. # pylint: disable=protected-access
  513. field = message.Extensions._FindExtensionByName(identifier)
  514. # pylint: enable=protected-access
  515. if not field:
  516. # Try looking for extension by the message type name, dropping the
  517. # field name following the final . separator in full_name.
  518. identifier = '.'.join(identifier.split('.')[:-1])
  519. # pylint: disable=protected-access
  520. field = message.Extensions._FindExtensionByName(identifier)
  521. # pylint: enable=protected-access
  522. if not field:
  523. if self.ignore_unknown_fields:
  524. continue
  525. raise ParseError(
  526. (
  527. 'Message type "{0}" has no field named "{1}" at "{2}".\n'
  528. ' Available Fields(except extensions): "{3}"'
  529. ).format(
  530. message_descriptor.full_name,
  531. name,
  532. path,
  533. [f.json_name for f in message_descriptor.fields],
  534. )
  535. )
  536. if name in names:
  537. raise ParseError(
  538. 'Message type "{0}" should not have multiple '
  539. '"{1}" fields at "{2}".'.format(
  540. message.DESCRIPTOR.full_name, name, path
  541. )
  542. )
  543. names.append(name)
  544. value = js[name]
  545. # Check no other oneof field is parsed.
  546. if field.containing_oneof is not None and value is not None:
  547. oneof_name = field.containing_oneof.name
  548. if oneof_name in names:
  549. raise ParseError(
  550. 'Message type "{0}" should not have multiple '
  551. '"{1}" oneof fields at "{2}".'.format(
  552. message.DESCRIPTOR.full_name, oneof_name, path
  553. )
  554. )
  555. names.append(oneof_name)
  556. if value is None:
  557. if (
  558. field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE
  559. and field.message_type.full_name == 'google.protobuf.Value'
  560. ):
  561. sub_message = _GetFieldOrExtension(message, field)
  562. sub_message.null_value = 0
  563. elif (
  564. field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM
  565. and field.enum_type.full_name == 'google.protobuf.NullValue'
  566. ):
  567. _SetFieldOrExtension(message, field, 0)
  568. else:
  569. _ClearFieldOrExtension(message, field)
  570. continue
  571. # Parse field value.
  572. if _IsMapEntry(field):
  573. _ClearFieldOrExtension(message, field)
  574. self._ConvertMapFieldValue(
  575. value, message, field, '{0}.{1}'.format(path, name)
  576. )
  577. elif field.is_repeated:
  578. _ClearFieldOrExtension(message, field)
  579. if not isinstance(value, _LIST_LIKE):
  580. raise ParseError(
  581. 'repeated field {0} must be in [] which is {1} at {2}'.format(
  582. name, value, path
  583. )
  584. )
  585. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  586. # Repeated message field.
  587. for index, item in enumerate(value):
  588. sub_message = _GetFieldOrExtension(message, field).add()
  589. # None is a null_value in Value.
  590. if (
  591. item is None
  592. and sub_message.DESCRIPTOR.full_name
  593. != 'google.protobuf.Value'
  594. ):
  595. raise ParseError(
  596. 'null is not allowed to be used as an element'
  597. ' in a repeated field at {0}.{1}[{2}]'.format(
  598. path, name, index
  599. )
  600. )
  601. self.ConvertMessage(
  602. item, sub_message, '{0}.{1}[{2}]'.format(path, name, index)
  603. )
  604. else:
  605. # Repeated scalar field.
  606. for index, item in enumerate(value):
  607. if item is None:
  608. raise ParseError(
  609. 'null is not allowed to be used as an element'
  610. ' in a repeated field at {0}.{1}[{2}]'.format(
  611. path, name, index
  612. )
  613. )
  614. self._ConvertAndAppendScalar(
  615. message, field, item, '{0}.{1}[{2}]'.format(path, name, index)
  616. )
  617. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  618. sub_message = _GetFieldOrExtension(message, field)
  619. sub_message.SetInParent()
  620. self.ConvertMessage(value, sub_message, '{0}.{1}'.format(path, name))
  621. else:
  622. self._ConvertAndSetScalar(
  623. message, field, value, '{0}.{1}'.format(path, name)
  624. )
  625. except ParseError as e:
  626. if field and field.containing_oneof is None:
  627. raise ParseError(
  628. 'Failed to parse {0} field: {1}.'.format(name, e)
  629. ) from e
  630. else:
  631. raise ParseError(str(e)) from e
  632. except ValueError as e:
  633. raise ParseError(
  634. 'Failed to parse {0} field: {1}.'.format(name, e)
  635. ) from e
  636. except TypeError as e:
  637. raise ParseError(
  638. 'Failed to parse {0} field: {1}.'.format(name, e)
  639. ) from e
  640. def _ConvertAnyMessage(self, value, message, path):
  641. """Convert a JSON representation into Any message."""
  642. if isinstance(value, dict) and not value:
  643. return
  644. try:
  645. type_url = value['@type']
  646. except KeyError as e:
  647. raise ParseError(
  648. '@type is missing when parsing any message at {0}'.format(path)
  649. ) from e
  650. try:
  651. sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
  652. except TypeError as e:
  653. raise ParseError('{0} at {1}'.format(e, path)) from e
  654. message_descriptor = sub_message.DESCRIPTOR
  655. full_name = message_descriptor.full_name
  656. if _IsWrapperMessage(message_descriptor):
  657. self._ConvertWrapperMessage(
  658. value['value'], sub_message, '{0}.value'.format(path)
  659. )
  660. elif full_name in _WKTJSONMETHODS:
  661. methodcaller(
  662. _WKTJSONMETHODS[full_name][1],
  663. value['value'],
  664. sub_message,
  665. '{0}.value'.format(path),
  666. )(self)
  667. else:
  668. del value['@type']
  669. try:
  670. self._ConvertFieldValuePair(value, sub_message, path)
  671. finally:
  672. value['@type'] = type_url
  673. # Sets Any message
  674. message.value = sub_message.SerializeToString()
  675. message.type_url = type_url
  676. def _ConvertGenericMessage(self, value, message, path):
  677. """Convert a JSON representation into message with FromJsonString."""
  678. # Duration, Timestamp, FieldMask have a FromJsonString method to do the
  679. # conversion. Users can also call the method directly.
  680. try:
  681. message.FromJsonString(value)
  682. except ValueError as e:
  683. raise ParseError('{0} at {1}'.format(e, path)) from e
  684. def _ConvertValueMessage(self, value, message, path):
  685. """Convert a JSON representation into Value message."""
  686. if isinstance(value, dict):
  687. self._ConvertStructMessage(value, message.struct_value, path)
  688. elif isinstance(value, _LIST_LIKE):
  689. self._ConvertListOrTupleValueMessage(value, message.list_value, path)
  690. elif value is None:
  691. message.null_value = 0
  692. elif isinstance(value, bool):
  693. message.bool_value = value
  694. elif isinstance(value, str):
  695. message.string_value = value
  696. elif isinstance(value, _INT_OR_FLOAT):
  697. message.number_value = value
  698. else:
  699. raise ParseError(
  700. 'Value {0} has unexpected type {1} at {2}'.format(
  701. value, type(value), path
  702. )
  703. )
  704. def _ConvertListOrTupleValueMessage(self, value, message, path):
  705. """Convert a JSON representation into ListValue message."""
  706. if not isinstance(value, _LIST_LIKE):
  707. raise ParseError(
  708. 'ListValue must be in [] which is {0} at {1}'.format(value, path)
  709. )
  710. message.ClearField('values')
  711. for index, item in enumerate(value):
  712. self._ConvertValueMessage(
  713. item, message.values.add(), '{0}[{1}]'.format(path, index)
  714. )
  715. def _ConvertStructMessage(self, value, message, path):
  716. """Convert a JSON representation into Struct message."""
  717. if not isinstance(value, dict):
  718. raise ParseError(
  719. 'Struct must be in a dict which is {0} at {1}'.format(value, path)
  720. )
  721. # Clear will mark the struct as modified so it will be created even if
  722. # there are no values.
  723. message.Clear()
  724. for key in value:
  725. self._ConvertValueMessage(
  726. value[key], message.fields[key], '{0}.{1}'.format(path, key)
  727. )
  728. return
  729. def _ConvertWrapperMessage(self, value, message, path):
  730. """Convert a JSON representation into Wrapper message."""
  731. field = message.DESCRIPTOR.fields_by_name['value']
  732. self._ConvertAndSetScalar(
  733. message, field, value, path='{0}.value'.format(path)
  734. )
  735. def _ConvertMapFieldValue(self, value, message, field, path):
  736. """Convert map field value for a message map field.
  737. Args:
  738. value: A JSON object to convert the map field value.
  739. message: A protocol message to record the converted data.
  740. field: The descriptor of the map field to be converted.
  741. path: parent path to log parse error info.
  742. Raises:
  743. ParseError: In case of convert problems.
  744. """
  745. if not isinstance(value, dict):
  746. raise ParseError(
  747. 'Map field {0} must be in a dict which is {1} at {2}'.format(
  748. field.name, value, path
  749. )
  750. )
  751. key_field = field.message_type.fields_by_name['key']
  752. value_field = field.message_type.fields_by_name['value']
  753. for key in value:
  754. key_value = _ConvertScalarFieldValue(
  755. key, key_field, '{0}.key'.format(path), True
  756. )
  757. if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  758. self.ConvertMessage(
  759. value[key],
  760. getattr(message, field.name)[key_value],
  761. '{0}[{1}]'.format(path, key_value),
  762. )
  763. else:
  764. self._ConvertAndSetScalarToMapKey(
  765. message,
  766. field,
  767. key_value,
  768. value[key],
  769. path='{0}[{1}]'.format(path, key_value),
  770. )
  771. def _ConvertAndSetScalar(self, message, field, js_value, path):
  772. """Convert scalar from js_value and assign it to message.field."""
  773. try:
  774. value = _ConvertScalarFieldValue(js_value, field, path)
  775. if field.is_extension:
  776. message.Extensions[field] = value
  777. else:
  778. setattr(message, field.name, value)
  779. except EnumStringValueParseError:
  780. if not self.ignore_unknown_fields:
  781. raise
  782. def _ConvertAndAppendScalar(self, message, repeated_field, js_value, path):
  783. """Convert scalar from js_value and append it to message.repeated_field."""
  784. try:
  785. if repeated_field.is_extension:
  786. repeated = message.Extensions[repeated_field]
  787. else:
  788. repeated = getattr(message, repeated_field.name)
  789. value = _ConvertScalarFieldValue(js_value, repeated_field, path)
  790. repeated.append(value)
  791. except EnumStringValueParseError:
  792. if not self.ignore_unknown_fields:
  793. raise
  794. def _ConvertAndSetScalarToMapKey(
  795. self, message, map_field, converted_key, js_value, path
  796. ):
  797. """Convert scalar from 'js_value' and add it to message.map_field[converted_key]."""
  798. try:
  799. getattr(message, map_field.name)[converted_key] = (
  800. _ConvertScalarFieldValue(
  801. js_value,
  802. map_field.message_type.fields_by_name['value'],
  803. path,
  804. )
  805. )
  806. except EnumStringValueParseError:
  807. if not self.ignore_unknown_fields:
  808. raise
  809. def _ConvertScalarFieldValue(value, field, path, require_str=False):
  810. """Convert a single scalar field value.
  811. Args:
  812. value: A scalar value to convert the scalar field value.
  813. field: The descriptor of the field to convert.
  814. path: parent path to log parse error info.
  815. require_str: If True, the field value must be a str.
  816. Returns:
  817. The converted scalar field value
  818. Raises:
  819. ParseError: In case of convert problems.
  820. EnumStringValueParseError: In case of unknown enum string value.
  821. """
  822. try:
  823. if field.cpp_type in _INT_TYPES:
  824. return _ConvertInteger(value)
  825. elif field.cpp_type in _FLOAT_TYPES:
  826. return _ConvertFloat(value, field)
  827. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
  828. return _ConvertBool(value, require_str)
  829. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
  830. if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
  831. if isinstance(value, str):
  832. encoded = value.encode('utf-8')
  833. else:
  834. encoded = value
  835. # Add extra padding '='
  836. padded_value = encoded + b'=' * (4 - len(encoded) % 4)
  837. return base64.urlsafe_b64decode(padded_value)
  838. else:
  839. # Checking for unpaired surrogates appears to be unreliable,
  840. # depending on the specific Python version, so we check manually.
  841. if _UNPAIRED_SURROGATE_PATTERN.search(value):
  842. raise ParseError('Unpaired surrogate')
  843. return value
  844. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
  845. # Convert an enum value.
  846. enum_value = field.enum_type.values_by_name.get(value, None)
  847. if enum_value is None:
  848. try:
  849. number = int(value)
  850. enum_value = field.enum_type.values_by_number.get(number, None)
  851. except ValueError as e:
  852. # Since parsing to integer failed and lookup in values_by_name didn't
  853. # find this name, we have an enum string value which is unknown.
  854. raise EnumStringValueParseError(
  855. 'Invalid enum value {0} for enum type {1}'.format(
  856. value, field.enum_type.full_name
  857. )
  858. ) from e
  859. if enum_value is None:
  860. if field.enum_type.is_closed:
  861. raise ParseError(
  862. 'Invalid enum value {0} for enum type {1}'.format(
  863. value, field.enum_type.full_name
  864. )
  865. )
  866. else:
  867. return number
  868. return enum_value.number
  869. except EnumStringValueParseError as e:
  870. raise EnumStringValueParseError('{0} at {1}'.format(e, path)) from e
  871. except ParseError as e:
  872. raise ParseError('{0} at {1}'.format(e, path)) from e
  873. def _ConvertInteger(value):
  874. """Convert an integer.
  875. Args:
  876. value: A scalar value to convert.
  877. Returns:
  878. The integer value.
  879. Raises:
  880. ParseError: If an integer couldn't be consumed.
  881. """
  882. if isinstance(value, float) and not value.is_integer():
  883. raise ParseError("Couldn't parse integer: {0}".format(value))
  884. if isinstance(value, str) and value.find(' ') != -1:
  885. raise ParseError('Couldn\'t parse integer: "{0}"'.format(value))
  886. if isinstance(value, bool):
  887. raise ParseError(
  888. 'Bool value {0} is not acceptable for integer field'.format(value)
  889. )
  890. try:
  891. return int(value)
  892. except ValueError as e:
  893. # Attempt to parse as an integer-valued float.
  894. try:
  895. f = float(value)
  896. except ValueError:
  897. # Raise the original exception for the int parse.
  898. raise e # pylint: disable=raise-missing-from
  899. if not f.is_integer():
  900. raise ParseError(
  901. 'Couldn\'t parse non-integer string: "{0}"'.format(value)
  902. ) from e
  903. return int(f)
  904. def _ConvertFloat(value, field):
  905. """Convert an floating point number."""
  906. if isinstance(value, float):
  907. if math.isnan(value):
  908. raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead')
  909. if math.isinf(value):
  910. if value > 0:
  911. raise ParseError(
  912. "Couldn't parse Infinity or value too large, "
  913. 'use quoted "Infinity" instead'
  914. )
  915. else:
  916. raise ParseError(
  917. "Couldn't parse -Infinity or value too small, "
  918. 'use quoted "-Infinity" instead'
  919. )
  920. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
  921. # pylint: disable=protected-access
  922. if value > type_checkers._FLOAT_MAX:
  923. raise ParseError('Float value too large')
  924. # pylint: disable=protected-access
  925. if value < type_checkers._FLOAT_MIN:
  926. raise ParseError('Float value too small')
  927. if value == 'nan':
  928. raise ParseError('Couldn\'t parse float "nan", use "NaN" instead')
  929. try:
  930. # Assume Python compatible syntax.
  931. return float(value)
  932. except ValueError as e:
  933. # Check alternative spellings.
  934. if value == _NEG_INFINITY:
  935. return float('-inf')
  936. elif value == _INFINITY:
  937. return float('inf')
  938. elif value == _NAN:
  939. return float('nan')
  940. else:
  941. raise ParseError("Couldn't parse float: {0}".format(value)) from e
  942. def _ConvertBool(value, require_str):
  943. """Convert a boolean value.
  944. Args:
  945. value: A scalar value to convert.
  946. require_str: If True, value must be a str.
  947. Returns:
  948. The bool parsed.
  949. Raises:
  950. ParseError: If a boolean value couldn't be consumed.
  951. """
  952. if require_str:
  953. if value == 'true':
  954. return True
  955. elif value == 'false':
  956. return False
  957. else:
  958. raise ParseError('Expected "true" or "false", not {0}'.format(value))
  959. if not isinstance(value, bool):
  960. raise ParseError('Expected true or false without quotes')
  961. return value
  962. _WKTJSONMETHODS = {
  963. 'google.protobuf.Any': ['_AnyMessageToJsonObject', '_ConvertAnyMessage'],
  964. 'google.protobuf.Duration': [
  965. '_GenericMessageToJsonObject',
  966. '_ConvertGenericMessage',
  967. ],
  968. 'google.protobuf.FieldMask': [
  969. '_GenericMessageToJsonObject',
  970. '_ConvertGenericMessage',
  971. ],
  972. 'google.protobuf.ListValue': [
  973. '_ListValueMessageToJsonObject',
  974. '_ConvertListOrTupleValueMessage',
  975. ],
  976. 'google.protobuf.Struct': [
  977. '_StructMessageToJsonObject',
  978. '_ConvertStructMessage',
  979. ],
  980. 'google.protobuf.Timestamp': [
  981. '_GenericMessageToJsonObject',
  982. '_ConvertGenericMessage',
  983. ],
  984. 'google.protobuf.Value': [
  985. '_ValueMessageToJsonObject',
  986. '_ConvertValueMessage',
  987. ],
  988. }