misc.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958
  1. """
  2. Miscellaneous function (re)definitions from the Py3.4+ standard library
  3. for Python 2.6/2.7.
  4. - math.ceil (for Python 2.7)
  5. - collections.OrderedDict (for Python 2.6)
  6. - collections.Counter (for Python 2.6)
  7. - collections.ChainMap (for all versions prior to Python 3.3)
  8. - itertools.count (for Python 2.6, with step parameter)
  9. - subprocess.check_output (for Python 2.6)
  10. - reprlib.recursive_repr (for Python 2.6+)
  11. - functools.cmp_to_key (for Python 2.6)
  12. """
  13. from __future__ import absolute_import
  14. import subprocess
  15. from math import ceil as oldceil
  16. from operator import itemgetter as _itemgetter, eq as _eq
  17. import sys
  18. import heapq as _heapq
  19. from _weakref import proxy as _proxy
  20. from itertools import repeat as _repeat, chain as _chain, starmap as _starmap
  21. from socket import getaddrinfo, SOCK_STREAM, error, socket
  22. from future.utils import iteritems, itervalues, PY2, PY26, PY3
  23. if PY2:
  24. from collections import Mapping, MutableMapping
  25. else:
  26. from collections.abc import Mapping, MutableMapping
  27. def ceil(x):
  28. """
  29. Return the ceiling of x as an int.
  30. This is the smallest integral value >= x.
  31. """
  32. return int(oldceil(x))
  33. ########################################################################
  34. ### reprlib.recursive_repr decorator from Py3.4
  35. ########################################################################
  36. from itertools import islice
  37. if PY26:
  38. # itertools.count in Py 2.6 doesn't accept a step parameter
  39. def count(start=0, step=1):
  40. while True:
  41. yield start
  42. start += step
  43. else:
  44. from itertools import count
  45. if PY3:
  46. try:
  47. from _thread import get_ident
  48. except ImportError:
  49. from _dummy_thread import get_ident
  50. else:
  51. try:
  52. from thread import get_ident
  53. except ImportError:
  54. from dummy_thread import get_ident
  55. def recursive_repr(fillvalue='...'):
  56. 'Decorator to make a repr function return fillvalue for a recursive call'
  57. def decorating_function(user_function):
  58. repr_running = set()
  59. def wrapper(self):
  60. key = id(self), get_ident()
  61. if key in repr_running:
  62. return fillvalue
  63. repr_running.add(key)
  64. try:
  65. result = user_function(self)
  66. finally:
  67. repr_running.discard(key)
  68. return result
  69. # Can't use functools.wraps() here because of bootstrap issues
  70. wrapper.__module__ = getattr(user_function, '__module__')
  71. wrapper.__doc__ = getattr(user_function, '__doc__')
  72. wrapper.__name__ = getattr(user_function, '__name__')
  73. wrapper.__annotations__ = getattr(user_function, '__annotations__', {})
  74. return wrapper
  75. return decorating_function
  76. # OrderedDict Shim from Raymond Hettinger, python core dev
  77. # http://code.activestate.com/recipes/576693-ordered-dictionary-for-py24/
  78. # here to support version 2.6.
  79. ################################################################################
  80. ### OrderedDict
  81. ################################################################################
  82. class _Link(object):
  83. __slots__ = 'prev', 'next', 'key', '__weakref__'
  84. class OrderedDict(dict):
  85. 'Dictionary that remembers insertion order'
  86. # An inherited dict maps keys to values.
  87. # The inherited dict provides __getitem__, __len__, __contains__, and get.
  88. # The remaining methods are order-aware.
  89. # Big-O running times for all methods are the same as regular dictionaries.
  90. # The internal self.__map dict maps keys to links in a doubly linked list.
  91. # The circular doubly linked list starts and ends with a sentinel element.
  92. # The sentinel element never gets deleted (this simplifies the algorithm).
  93. # The sentinel is in self.__hardroot with a weakref proxy in self.__root.
  94. # The prev links are weakref proxies (to prevent circular references).
  95. # Individual links are kept alive by the hard reference in self.__map.
  96. # Those hard references disappear when a key is deleted from an OrderedDict.
  97. def __init__(*args, **kwds):
  98. '''Initialize an ordered dictionary. The signature is the same as
  99. regular dictionaries, but keyword arguments are not recommended because
  100. their insertion order is arbitrary.
  101. '''
  102. if not args:
  103. raise TypeError("descriptor '__init__' of 'OrderedDict' object "
  104. "needs an argument")
  105. self = args[0]
  106. args = args[1:]
  107. if len(args) > 1:
  108. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  109. try:
  110. self.__root
  111. except AttributeError:
  112. self.__hardroot = _Link()
  113. self.__root = root = _proxy(self.__hardroot)
  114. root.prev = root.next = root
  115. self.__map = {}
  116. self.__update(*args, **kwds)
  117. def __setitem__(self, key, value,
  118. dict_setitem=dict.__setitem__, proxy=_proxy, Link=_Link):
  119. 'od.__setitem__(i, y) <==> od[i]=y'
  120. # Setting a new item creates a new link at the end of the linked list,
  121. # and the inherited dictionary is updated with the new key/value pair.
  122. if key not in self:
  123. self.__map[key] = link = Link()
  124. root = self.__root
  125. last = root.prev
  126. link.prev, link.next, link.key = last, root, key
  127. last.next = link
  128. root.prev = proxy(link)
  129. dict_setitem(self, key, value)
  130. def __delitem__(self, key, dict_delitem=dict.__delitem__):
  131. 'od.__delitem__(y) <==> del od[y]'
  132. # Deleting an existing item uses self.__map to find the link which gets
  133. # removed by updating the links in the predecessor and successor nodes.
  134. dict_delitem(self, key)
  135. link = self.__map.pop(key)
  136. link_prev = link.prev
  137. link_next = link.next
  138. link_prev.next = link_next
  139. link_next.prev = link_prev
  140. def __iter__(self):
  141. 'od.__iter__() <==> iter(od)'
  142. # Traverse the linked list in order.
  143. root = self.__root
  144. curr = root.next
  145. while curr is not root:
  146. yield curr.key
  147. curr = curr.next
  148. def __reversed__(self):
  149. 'od.__reversed__() <==> reversed(od)'
  150. # Traverse the linked list in reverse order.
  151. root = self.__root
  152. curr = root.prev
  153. while curr is not root:
  154. yield curr.key
  155. curr = curr.prev
  156. def clear(self):
  157. 'od.clear() -> None. Remove all items from od.'
  158. root = self.__root
  159. root.prev = root.next = root
  160. self.__map.clear()
  161. dict.clear(self)
  162. def popitem(self, last=True):
  163. '''od.popitem() -> (k, v), return and remove a (key, value) pair.
  164. Pairs are returned in LIFO order if last is true or FIFO order if false.
  165. '''
  166. if not self:
  167. raise KeyError('dictionary is empty')
  168. root = self.__root
  169. if last:
  170. link = root.prev
  171. link_prev = link.prev
  172. link_prev.next = root
  173. root.prev = link_prev
  174. else:
  175. link = root.next
  176. link_next = link.next
  177. root.next = link_next
  178. link_next.prev = root
  179. key = link.key
  180. del self.__map[key]
  181. value = dict.pop(self, key)
  182. return key, value
  183. def move_to_end(self, key, last=True):
  184. '''Move an existing element to the end (or beginning if last==False).
  185. Raises KeyError if the element does not exist.
  186. When last=True, acts like a fast version of self[key]=self.pop(key).
  187. '''
  188. link = self.__map[key]
  189. link_prev = link.prev
  190. link_next = link.next
  191. link_prev.next = link_next
  192. link_next.prev = link_prev
  193. root = self.__root
  194. if last:
  195. last = root.prev
  196. link.prev = last
  197. link.next = root
  198. last.next = root.prev = link
  199. else:
  200. first = root.next
  201. link.prev = root
  202. link.next = first
  203. root.next = first.prev = link
  204. def __sizeof__(self):
  205. sizeof = sys.getsizeof
  206. n = len(self) + 1 # number of links including root
  207. size = sizeof(self.__dict__) # instance dictionary
  208. size += sizeof(self.__map) * 2 # internal dict and inherited dict
  209. size += sizeof(self.__hardroot) * n # link objects
  210. size += sizeof(self.__root) * n # proxy objects
  211. return size
  212. update = __update = MutableMapping.update
  213. keys = MutableMapping.keys
  214. values = MutableMapping.values
  215. items = MutableMapping.items
  216. __ne__ = MutableMapping.__ne__
  217. __marker = object()
  218. def pop(self, key, default=__marker):
  219. '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
  220. value. If key is not found, d is returned if given, otherwise KeyError
  221. is raised.
  222. '''
  223. if key in self:
  224. result = self[key]
  225. del self[key]
  226. return result
  227. if default is self.__marker:
  228. raise KeyError(key)
  229. return default
  230. def setdefault(self, key, default=None):
  231. 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
  232. if key in self:
  233. return self[key]
  234. self[key] = default
  235. return default
  236. @recursive_repr()
  237. def __repr__(self):
  238. 'od.__repr__() <==> repr(od)'
  239. if not self:
  240. return '%s()' % (self.__class__.__name__,)
  241. return '%s(%r)' % (self.__class__.__name__, list(self.items()))
  242. def __reduce__(self):
  243. 'Return state information for pickling'
  244. inst_dict = vars(self).copy()
  245. for k in vars(OrderedDict()):
  246. inst_dict.pop(k, None)
  247. return self.__class__, (), inst_dict or None, None, iter(self.items())
  248. def copy(self):
  249. 'od.copy() -> a shallow copy of od'
  250. return self.__class__(self)
  251. @classmethod
  252. def fromkeys(cls, iterable, value=None):
  253. '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
  254. If not specified, the value defaults to None.
  255. '''
  256. self = cls()
  257. for key in iterable:
  258. self[key] = value
  259. return self
  260. def __eq__(self, other):
  261. '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
  262. while comparison to a regular mapping is order-insensitive.
  263. '''
  264. if isinstance(other, OrderedDict):
  265. return dict.__eq__(self, other) and all(map(_eq, self, other))
  266. return dict.__eq__(self, other)
  267. # {{{ http://code.activestate.com/recipes/576611/ (r11)
  268. try:
  269. from operator import itemgetter
  270. from heapq import nlargest
  271. except ImportError:
  272. pass
  273. ########################################################################
  274. ### Counter
  275. ########################################################################
  276. def _count_elements(mapping, iterable):
  277. 'Tally elements from the iterable.'
  278. mapping_get = mapping.get
  279. for elem in iterable:
  280. mapping[elem] = mapping_get(elem, 0) + 1
  281. class Counter(dict):
  282. '''Dict subclass for counting hashable items. Sometimes called a bag
  283. or multiset. Elements are stored as dictionary keys and their counts
  284. are stored as dictionary values.
  285. >>> c = Counter('abcdeabcdabcaba') # count elements from a string
  286. >>> c.most_common(3) # three most common elements
  287. [('a', 5), ('b', 4), ('c', 3)]
  288. >>> sorted(c) # list all unique elements
  289. ['a', 'b', 'c', 'd', 'e']
  290. >>> ''.join(sorted(c.elements())) # list elements with repetitions
  291. 'aaaaabbbbcccdde'
  292. >>> sum(c.values()) # total of all counts
  293. 15
  294. >>> c['a'] # count of letter 'a'
  295. 5
  296. >>> for elem in 'shazam': # update counts from an iterable
  297. ... c[elem] += 1 # by adding 1 to each element's count
  298. >>> c['a'] # now there are seven 'a'
  299. 7
  300. >>> del c['b'] # remove all 'b'
  301. >>> c['b'] # now there are zero 'b'
  302. 0
  303. >>> d = Counter('simsalabim') # make another counter
  304. >>> c.update(d) # add in the second counter
  305. >>> c['a'] # now there are nine 'a'
  306. 9
  307. >>> c.clear() # empty the counter
  308. >>> c
  309. Counter()
  310. Note: If a count is set to zero or reduced to zero, it will remain
  311. in the counter until the entry is deleted or the counter is cleared:
  312. >>> c = Counter('aaabbc')
  313. >>> c['b'] -= 2 # reduce the count of 'b' by two
  314. >>> c.most_common() # 'b' is still in, but its count is zero
  315. [('a', 3), ('c', 1), ('b', 0)]
  316. '''
  317. # References:
  318. # http://en.wikipedia.org/wiki/Multiset
  319. # http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html
  320. # http://www.demo2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm
  321. # http://code.activestate.com/recipes/259174/
  322. # Knuth, TAOCP Vol. II section 4.6.3
  323. def __init__(*args, **kwds):
  324. '''Create a new, empty Counter object. And if given, count elements
  325. from an input iterable. Or, initialize the count from another mapping
  326. of elements to their counts.
  327. >>> c = Counter() # a new, empty counter
  328. >>> c = Counter('gallahad') # a new counter from an iterable
  329. >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping
  330. >>> c = Counter(a=4, b=2) # a new counter from keyword args
  331. '''
  332. if not args:
  333. raise TypeError("descriptor '__init__' of 'Counter' object "
  334. "needs an argument")
  335. self = args[0]
  336. args = args[1:]
  337. if len(args) > 1:
  338. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  339. super(Counter, self).__init__()
  340. self.update(*args, **kwds)
  341. def __missing__(self, key):
  342. 'The count of elements not in the Counter is zero.'
  343. # Needed so that self[missing_item] does not raise KeyError
  344. return 0
  345. def most_common(self, n=None):
  346. '''List the n most common elements and their counts from the most
  347. common to the least. If n is None, then list all element counts.
  348. >>> Counter('abcdeabcdabcaba').most_common(3)
  349. [('a', 5), ('b', 4), ('c', 3)]
  350. '''
  351. # Emulate Bag.sortedByCount from Smalltalk
  352. if n is None:
  353. return sorted(self.items(), key=_itemgetter(1), reverse=True)
  354. return _heapq.nlargest(n, self.items(), key=_itemgetter(1))
  355. def elements(self):
  356. '''Iterator over elements repeating each as many times as its count.
  357. >>> c = Counter('ABCABC')
  358. >>> sorted(c.elements())
  359. ['A', 'A', 'B', 'B', 'C', 'C']
  360. # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1
  361. >>> prime_factors = Counter({2: 2, 3: 3, 17: 1})
  362. >>> product = 1
  363. >>> for factor in prime_factors.elements(): # loop over factors
  364. ... product *= factor # and multiply them
  365. >>> product
  366. 1836
  367. Note, if an element's count has been set to zero or is a negative
  368. number, elements() will ignore it.
  369. '''
  370. # Emulate Bag.do from Smalltalk and Multiset.begin from C++.
  371. return _chain.from_iterable(_starmap(_repeat, self.items()))
  372. # Override dict methods where necessary
  373. @classmethod
  374. def fromkeys(cls, iterable, v=None):
  375. # There is no equivalent method for counters because setting v=1
  376. # means that no element can have a count greater than one.
  377. raise NotImplementedError(
  378. 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.')
  379. def update(*args, **kwds):
  380. '''Like dict.update() but add counts instead of replacing them.
  381. Source can be an iterable, a dictionary, or another Counter instance.
  382. >>> c = Counter('which')
  383. >>> c.update('witch') # add elements from another iterable
  384. >>> d = Counter('watch')
  385. >>> c.update(d) # add elements from another counter
  386. >>> c['h'] # four 'h' in which, witch, and watch
  387. 4
  388. '''
  389. # The regular dict.update() operation makes no sense here because the
  390. # replace behavior results in the some of original untouched counts
  391. # being mixed-in with all of the other counts for a mismash that
  392. # doesn't have a straight-forward interpretation in most counting
  393. # contexts. Instead, we implement straight-addition. Both the inputs
  394. # and outputs are allowed to contain zero and negative counts.
  395. if not args:
  396. raise TypeError("descriptor 'update' of 'Counter' object "
  397. "needs an argument")
  398. self = args[0]
  399. args = args[1:]
  400. if len(args) > 1:
  401. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  402. iterable = args[0] if args else None
  403. if iterable is not None:
  404. if isinstance(iterable, Mapping):
  405. if self:
  406. self_get = self.get
  407. for elem, count in iterable.items():
  408. self[elem] = count + self_get(elem, 0)
  409. else:
  410. super(Counter, self).update(iterable) # fast path when counter is empty
  411. else:
  412. _count_elements(self, iterable)
  413. if kwds:
  414. self.update(kwds)
  415. def subtract(*args, **kwds):
  416. '''Like dict.update() but subtracts counts instead of replacing them.
  417. Counts can be reduced below zero. Both the inputs and outputs are
  418. allowed to contain zero and negative counts.
  419. Source can be an iterable, a dictionary, or another Counter instance.
  420. >>> c = Counter('which')
  421. >>> c.subtract('witch') # subtract elements from another iterable
  422. >>> c.subtract(Counter('watch')) # subtract elements from another counter
  423. >>> c['h'] # 2 in which, minus 1 in witch, minus 1 in watch
  424. 0
  425. >>> c['w'] # 1 in which, minus 1 in witch, minus 1 in watch
  426. -1
  427. '''
  428. if not args:
  429. raise TypeError("descriptor 'subtract' of 'Counter' object "
  430. "needs an argument")
  431. self = args[0]
  432. args = args[1:]
  433. if len(args) > 1:
  434. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  435. iterable = args[0] if args else None
  436. if iterable is not None:
  437. self_get = self.get
  438. if isinstance(iterable, Mapping):
  439. for elem, count in iterable.items():
  440. self[elem] = self_get(elem, 0) - count
  441. else:
  442. for elem in iterable:
  443. self[elem] = self_get(elem, 0) - 1
  444. if kwds:
  445. self.subtract(kwds)
  446. def copy(self):
  447. 'Return a shallow copy.'
  448. return self.__class__(self)
  449. def __reduce__(self):
  450. return self.__class__, (dict(self),)
  451. def __delitem__(self, elem):
  452. 'Like dict.__delitem__() but does not raise KeyError for missing values.'
  453. if elem in self:
  454. super(Counter, self).__delitem__(elem)
  455. def __repr__(self):
  456. if not self:
  457. return '%s()' % self.__class__.__name__
  458. try:
  459. items = ', '.join(map('%r: %r'.__mod__, self.most_common()))
  460. return '%s({%s})' % (self.__class__.__name__, items)
  461. except TypeError:
  462. # handle case where values are not orderable
  463. return '{0}({1!r})'.format(self.__class__.__name__, dict(self))
  464. # Multiset-style mathematical operations discussed in:
  465. # Knuth TAOCP Volume II section 4.6.3 exercise 19
  466. # and at http://en.wikipedia.org/wiki/Multiset
  467. #
  468. # Outputs guaranteed to only include positive counts.
  469. #
  470. # To strip negative and zero counts, add-in an empty counter:
  471. # c += Counter()
  472. def __add__(self, other):
  473. '''Add counts from two counters.
  474. >>> Counter('abbb') + Counter('bcc')
  475. Counter({'b': 4, 'c': 2, 'a': 1})
  476. '''
  477. if not isinstance(other, Counter):
  478. return NotImplemented
  479. result = Counter()
  480. for elem, count in self.items():
  481. newcount = count + other[elem]
  482. if newcount > 0:
  483. result[elem] = newcount
  484. for elem, count in other.items():
  485. if elem not in self and count > 0:
  486. result[elem] = count
  487. return result
  488. def __sub__(self, other):
  489. ''' Subtract count, but keep only results with positive counts.
  490. >>> Counter('abbbc') - Counter('bccd')
  491. Counter({'b': 2, 'a': 1})
  492. '''
  493. if not isinstance(other, Counter):
  494. return NotImplemented
  495. result = Counter()
  496. for elem, count in self.items():
  497. newcount = count - other[elem]
  498. if newcount > 0:
  499. result[elem] = newcount
  500. for elem, count in other.items():
  501. if elem not in self and count < 0:
  502. result[elem] = 0 - count
  503. return result
  504. def __or__(self, other):
  505. '''Union is the maximum of value in either of the input counters.
  506. >>> Counter('abbb') | Counter('bcc')
  507. Counter({'b': 3, 'c': 2, 'a': 1})
  508. '''
  509. if not isinstance(other, Counter):
  510. return NotImplemented
  511. result = Counter()
  512. for elem, count in self.items():
  513. other_count = other[elem]
  514. newcount = other_count if count < other_count else count
  515. if newcount > 0:
  516. result[elem] = newcount
  517. for elem, count in other.items():
  518. if elem not in self and count > 0:
  519. result[elem] = count
  520. return result
  521. def __and__(self, other):
  522. ''' Intersection is the minimum of corresponding counts.
  523. >>> Counter('abbb') & Counter('bcc')
  524. Counter({'b': 1})
  525. '''
  526. if not isinstance(other, Counter):
  527. return NotImplemented
  528. result = Counter()
  529. for elem, count in self.items():
  530. other_count = other[elem]
  531. newcount = count if count < other_count else other_count
  532. if newcount > 0:
  533. result[elem] = newcount
  534. return result
  535. def __pos__(self):
  536. 'Adds an empty counter, effectively stripping negative and zero counts'
  537. return self + Counter()
  538. def __neg__(self):
  539. '''Subtracts from an empty counter. Strips positive and zero counts,
  540. and flips the sign on negative counts.
  541. '''
  542. return Counter() - self
  543. def _keep_positive(self):
  544. '''Internal method to strip elements with a negative or zero count'''
  545. nonpositive = [elem for elem, count in self.items() if not count > 0]
  546. for elem in nonpositive:
  547. del self[elem]
  548. return self
  549. def __iadd__(self, other):
  550. '''Inplace add from another counter, keeping only positive counts.
  551. >>> c = Counter('abbb')
  552. >>> c += Counter('bcc')
  553. >>> c
  554. Counter({'b': 4, 'c': 2, 'a': 1})
  555. '''
  556. for elem, count in other.items():
  557. self[elem] += count
  558. return self._keep_positive()
  559. def __isub__(self, other):
  560. '''Inplace subtract counter, but keep only results with positive counts.
  561. >>> c = Counter('abbbc')
  562. >>> c -= Counter('bccd')
  563. >>> c
  564. Counter({'b': 2, 'a': 1})
  565. '''
  566. for elem, count in other.items():
  567. self[elem] -= count
  568. return self._keep_positive()
  569. def __ior__(self, other):
  570. '''Inplace union is the maximum of value from either counter.
  571. >>> c = Counter('abbb')
  572. >>> c |= Counter('bcc')
  573. >>> c
  574. Counter({'b': 3, 'c': 2, 'a': 1})
  575. '''
  576. for elem, other_count in other.items():
  577. count = self[elem]
  578. if other_count > count:
  579. self[elem] = other_count
  580. return self._keep_positive()
  581. def __iand__(self, other):
  582. '''Inplace intersection is the minimum of corresponding counts.
  583. >>> c = Counter('abbb')
  584. >>> c &= Counter('bcc')
  585. >>> c
  586. Counter({'b': 1})
  587. '''
  588. for elem, count in self.items():
  589. other_count = other[elem]
  590. if other_count < count:
  591. self[elem] = other_count
  592. return self._keep_positive()
  593. def check_output(*popenargs, **kwargs):
  594. """
  595. For Python 2.6 compatibility: see
  596. http://stackoverflow.com/questions/4814970/
  597. """
  598. if 'stdout' in kwargs:
  599. raise ValueError('stdout argument not allowed, it will be overridden.')
  600. process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
  601. output, unused_err = process.communicate()
  602. retcode = process.poll()
  603. if retcode:
  604. cmd = kwargs.get("args")
  605. if cmd is None:
  606. cmd = popenargs[0]
  607. raise subprocess.CalledProcessError(retcode, cmd)
  608. return output
  609. def count(start=0, step=1):
  610. """
  611. ``itertools.count`` in Py 2.6 doesn't accept a step
  612. parameter. This is an enhanced version of ``itertools.count``
  613. for Py2.6 equivalent to ``itertools.count`` in Python 2.7+.
  614. """
  615. while True:
  616. yield start
  617. start += step
  618. ########################################################################
  619. ### ChainMap (helper for configparser and string.Template)
  620. ### From the Py3.4 source code. See also:
  621. ### https://github.com/kkxue/Py2ChainMap/blob/master/py2chainmap.py
  622. ########################################################################
  623. class ChainMap(MutableMapping):
  624. ''' A ChainMap groups multiple dicts (or other mappings) together
  625. to create a single, updateable view.
  626. The underlying mappings are stored in a list. That list is public and can
  627. accessed or updated using the *maps* attribute. There is no other state.
  628. Lookups search the underlying mappings successively until a key is found.
  629. In contrast, writes, updates, and deletions only operate on the first
  630. mapping.
  631. '''
  632. def __init__(self, *maps):
  633. '''Initialize a ChainMap by setting *maps* to the given mappings.
  634. If no mappings are provided, a single empty dictionary is used.
  635. '''
  636. self.maps = list(maps) or [{}] # always at least one map
  637. def __missing__(self, key):
  638. raise KeyError(key)
  639. def __getitem__(self, key):
  640. for mapping in self.maps:
  641. try:
  642. return mapping[key] # can't use 'key in mapping' with defaultdict
  643. except KeyError:
  644. pass
  645. return self.__missing__(key) # support subclasses that define __missing__
  646. def get(self, key, default=None):
  647. return self[key] if key in self else default
  648. def __len__(self):
  649. return len(set().union(*self.maps)) # reuses stored hash values if possible
  650. def __iter__(self):
  651. return iter(set().union(*self.maps))
  652. def __contains__(self, key):
  653. return any(key in m for m in self.maps)
  654. def __bool__(self):
  655. return any(self.maps)
  656. # Py2 compatibility:
  657. __nonzero__ = __bool__
  658. @recursive_repr()
  659. def __repr__(self):
  660. return '{0.__class__.__name__}({1})'.format(
  661. self, ', '.join(map(repr, self.maps)))
  662. @classmethod
  663. def fromkeys(cls, iterable, *args):
  664. 'Create a ChainMap with a single dict created from the iterable.'
  665. return cls(dict.fromkeys(iterable, *args))
  666. def copy(self):
  667. 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]'
  668. return self.__class__(self.maps[0].copy(), *self.maps[1:])
  669. __copy__ = copy
  670. def new_child(self, m=None): # like Django's Context.push()
  671. '''
  672. New ChainMap with a new map followed by all previous maps. If no
  673. map is provided, an empty dict is used.
  674. '''
  675. if m is None:
  676. m = {}
  677. return self.__class__(m, *self.maps)
  678. @property
  679. def parents(self): # like Django's Context.pop()
  680. 'New ChainMap from maps[1:].'
  681. return self.__class__(*self.maps[1:])
  682. def __setitem__(self, key, value):
  683. self.maps[0][key] = value
  684. def __delitem__(self, key):
  685. try:
  686. del self.maps[0][key]
  687. except KeyError:
  688. raise KeyError('Key not found in the first mapping: {0!r}'.format(key))
  689. def popitem(self):
  690. 'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.'
  691. try:
  692. return self.maps[0].popitem()
  693. except KeyError:
  694. raise KeyError('No keys found in the first mapping.')
  695. def pop(self, key, *args):
  696. 'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].'
  697. try:
  698. return self.maps[0].pop(key, *args)
  699. except KeyError:
  700. raise KeyError('Key not found in the first mapping: {0!r}'.format(key))
  701. def clear(self):
  702. 'Clear maps[0], leaving maps[1:] intact.'
  703. self.maps[0].clear()
  704. # Re-use the same sentinel as in the Python stdlib socket module:
  705. from socket import _GLOBAL_DEFAULT_TIMEOUT
  706. # Was: _GLOBAL_DEFAULT_TIMEOUT = object()
  707. def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
  708. source_address=None):
  709. """Backport of 3-argument create_connection() for Py2.6.
  710. Connect to *address* and return the socket object.
  711. Convenience function. Connect to *address* (a 2-tuple ``(host,
  712. port)``) and return the socket object. Passing the optional
  713. *timeout* parameter will set the timeout on the socket instance
  714. before attempting to connect. If no *timeout* is supplied, the
  715. global default timeout setting returned by :func:`getdefaulttimeout`
  716. is used. If *source_address* is set it must be a tuple of (host, port)
  717. for the socket to bind as a source address before making the connection.
  718. An host of '' or port 0 tells the OS to use the default.
  719. """
  720. host, port = address
  721. err = None
  722. for res in getaddrinfo(host, port, 0, SOCK_STREAM):
  723. af, socktype, proto, canonname, sa = res
  724. sock = None
  725. try:
  726. sock = socket(af, socktype, proto)
  727. if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
  728. sock.settimeout(timeout)
  729. if source_address:
  730. sock.bind(source_address)
  731. sock.connect(sa)
  732. return sock
  733. except error as _:
  734. err = _
  735. if sock is not None:
  736. sock.close()
  737. if err is not None:
  738. raise err
  739. else:
  740. raise error("getaddrinfo returns an empty list")
  741. # Backport from Py2.7 for Py2.6:
  742. def cmp_to_key(mycmp):
  743. """Convert a cmp= function into a key= function"""
  744. class K(object):
  745. __slots__ = ['obj']
  746. def __init__(self, obj, *args):
  747. self.obj = obj
  748. def __lt__(self, other):
  749. return mycmp(self.obj, other.obj) < 0
  750. def __gt__(self, other):
  751. return mycmp(self.obj, other.obj) > 0
  752. def __eq__(self, other):
  753. return mycmp(self.obj, other.obj) == 0
  754. def __le__(self, other):
  755. return mycmp(self.obj, other.obj) <= 0
  756. def __ge__(self, other):
  757. return mycmp(self.obj, other.obj) >= 0
  758. def __ne__(self, other):
  759. return mycmp(self.obj, other.obj) != 0
  760. def __hash__(self):
  761. raise TypeError('hash not implemented')
  762. return K
  763. # Back up our definitions above in case they're useful
  764. _OrderedDict = OrderedDict
  765. _Counter = Counter
  766. _check_output = check_output
  767. _count = count
  768. _ceil = ceil
  769. __count_elements = _count_elements
  770. _recursive_repr = recursive_repr
  771. _ChainMap = ChainMap
  772. _create_connection = create_connection
  773. _cmp_to_key = cmp_to_key
  774. # Overwrite the definitions above with the usual ones
  775. # from the standard library:
  776. if sys.version_info >= (2, 7):
  777. from collections import OrderedDict, Counter
  778. from itertools import count
  779. from functools import cmp_to_key
  780. try:
  781. from subprocess import check_output
  782. except ImportError:
  783. # Not available. This happens with Google App Engine: see issue #231
  784. pass
  785. from socket import create_connection
  786. if sys.version_info >= (3, 0):
  787. from math import ceil
  788. from collections import _count_elements
  789. if sys.version_info >= (3, 3):
  790. from reprlib import recursive_repr
  791. from collections import ChainMap