hash.h 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379
  1. #pragma once
  2. #include <c10/util/Exception.h>
  3. #include <cstddef>
  4. #include <functional>
  5. #include <iomanip>
  6. #include <ios>
  7. #include <sstream>
  8. #include <string>
  9. #include <tuple>
  10. #include <type_traits>
  11. #include <utility>
  12. #include <vector>
  13. #include <c10/util/ArrayRef.h>
  14. #include <c10/util/complex.h>
  15. namespace c10 {
  16. // NOTE: hash_combine and SHA1 hashing is based on implementation from Boost
  17. //
  18. // Boost Software License - Version 1.0 - August 17th, 2003
  19. //
  20. // Permission is hereby granted, free of charge, to any person or organization
  21. // obtaining a copy of the software and accompanying documentation covered by
  22. // this license (the "Software") to use, reproduce, display, distribute,
  23. // execute, and transmit the Software, and to prepare derivative works of the
  24. // Software, and to permit third-parties to whom the Software is furnished to
  25. // do so, all subject to the following:
  26. //
  27. // The copyright notices in the Software and this entire statement, including
  28. // the above license grant, this restriction and the following disclaimer,
  29. // must be included in all copies of the Software, in whole or in part, and
  30. // all derivative works of the Software, unless such copies or derivative
  31. // works are solely in the form of machine-executable object code generated by
  32. // a source language processor.
  33. //
  34. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  35. // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  36. // FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
  37. // SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
  38. // FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
  39. // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  40. // DEALINGS IN THE SOFTWARE.
  41. inline size_t hash_combine(size_t seed, size_t value) {
  42. return seed ^ (value + 0x9e3779b9 + (seed << 6u) + (seed >> 2u));
  43. }
  44. // Creates the SHA1 hash of a string. A 160-bit hash.
  45. // Based on the implementation in Boost (see notice above).
  46. // Note that SHA1 hashes are no longer considered cryptographically
  47. // secure, but are the standard hash for generating unique ids.
  48. // Usage:
  49. // // Let 'code' be a std::string
  50. // c10::sha1 sha1_hash{code};
  51. // const auto hash_code = sha1_hash.str();
  52. // TODO: Compare vs OpenSSL and/or CryptoPP implementations
  53. struct sha1 {
  54. typedef unsigned int(digest_type)[5];
  55. sha1(const std::string& s = "") {
  56. if (!s.empty()) {
  57. reset();
  58. process_bytes(s.c_str(), s.size());
  59. }
  60. }
  61. void reset() {
  62. h_[0] = 0x67452301;
  63. h_[1] = 0xEFCDAB89;
  64. h_[2] = 0x98BADCFE;
  65. h_[3] = 0x10325476;
  66. h_[4] = 0xC3D2E1F0;
  67. block_byte_index_ = 0;
  68. bit_count_low = 0;
  69. bit_count_high = 0;
  70. }
  71. std::string str() {
  72. unsigned int digest[5];
  73. get_digest(digest);
  74. std::ostringstream buf;
  75. for (unsigned int i : digest) {
  76. buf << std::hex << std::setfill('0') << std::setw(8) << i;
  77. }
  78. return buf.str();
  79. }
  80. private:
  81. unsigned int left_rotate(unsigned int x, std::size_t n) {
  82. return (x << n) ^ (x >> (32 - n));
  83. }
  84. void process_block_impl() {
  85. unsigned int w[80];
  86. for (std::size_t i = 0; i < 16; ++i) {
  87. w[i] = (block_[i * 4 + 0] << 24);
  88. w[i] |= (block_[i * 4 + 1] << 16);
  89. w[i] |= (block_[i * 4 + 2] << 8);
  90. w[i] |= (block_[i * 4 + 3]);
  91. }
  92. for (std::size_t i = 16; i < 80; ++i) {
  93. w[i] = left_rotate((w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16]), 1);
  94. }
  95. unsigned int a = h_[0];
  96. unsigned int b = h_[1];
  97. unsigned int c = h_[2];
  98. unsigned int d = h_[3];
  99. unsigned int e = h_[4];
  100. for (std::size_t i = 0; i < 80; ++i) {
  101. unsigned int f = 0;
  102. unsigned int k = 0;
  103. if (i < 20) {
  104. f = (b & c) | (~b & d);
  105. k = 0x5A827999;
  106. } else if (i < 40) {
  107. f = b ^ c ^ d;
  108. k = 0x6ED9EBA1;
  109. } else if (i < 60) {
  110. f = (b & c) | (b & d) | (c & d);
  111. k = 0x8F1BBCDC;
  112. } else {
  113. f = b ^ c ^ d;
  114. k = 0xCA62C1D6;
  115. }
  116. unsigned temp = left_rotate(a, 5) + f + e + k + w[i];
  117. e = d;
  118. d = c;
  119. c = left_rotate(b, 30);
  120. b = a;
  121. a = temp;
  122. }
  123. h_[0] += a;
  124. h_[1] += b;
  125. h_[2] += c;
  126. h_[3] += d;
  127. h_[4] += e;
  128. }
  129. void process_byte_impl(unsigned char byte) {
  130. block_[block_byte_index_++] = byte;
  131. if (block_byte_index_ == 64) {
  132. block_byte_index_ = 0;
  133. process_block_impl();
  134. }
  135. }
  136. void process_byte(unsigned char byte) {
  137. process_byte_impl(byte);
  138. // size_t max value = 0xFFFFFFFF
  139. // if (bit_count_low + 8 >= 0x100000000) { // would overflow
  140. // if (bit_count_low >= 0x100000000-8) {
  141. if (bit_count_low < 0xFFFFFFF8) {
  142. bit_count_low += 8;
  143. } else {
  144. bit_count_low = 0;
  145. if (bit_count_high <= 0xFFFFFFFE) {
  146. ++bit_count_high;
  147. } else {
  148. TORCH_CHECK(false, "sha1 too many bytes");
  149. }
  150. }
  151. }
  152. void process_block(void const* bytes_begin, void const* bytes_end) {
  153. unsigned char const* begin = static_cast<unsigned char const*>(bytes_begin);
  154. unsigned char const* end = static_cast<unsigned char const*>(bytes_end);
  155. for (; begin != end; ++begin) {
  156. process_byte(*begin);
  157. }
  158. }
  159. void process_bytes(void const* buffer, std::size_t byte_count) {
  160. unsigned char const* b = static_cast<unsigned char const*>(buffer);
  161. process_block(b, b + byte_count);
  162. }
  163. void get_digest(digest_type& digest) {
  164. // append the bit '1' to the message
  165. process_byte_impl(0x80);
  166. // append k bits '0', where k is the minimum number >= 0
  167. // such that the resulting message length is congruent to 56 (mod 64)
  168. // check if there is enough space for padding and bit_count
  169. if (block_byte_index_ > 56) {
  170. // finish this block
  171. while (block_byte_index_ != 0) {
  172. process_byte_impl(0);
  173. }
  174. // one more block
  175. while (block_byte_index_ < 56) {
  176. process_byte_impl(0);
  177. }
  178. } else {
  179. while (block_byte_index_ < 56) {
  180. process_byte_impl(0);
  181. }
  182. }
  183. // append length of message (before pre-processing)
  184. // as a 64-bit big-endian integer
  185. process_byte_impl(
  186. static_cast<unsigned char>((bit_count_high >> 24) & 0xFF));
  187. process_byte_impl(
  188. static_cast<unsigned char>((bit_count_high >> 16) & 0xFF));
  189. process_byte_impl(static_cast<unsigned char>((bit_count_high >> 8) & 0xFF));
  190. process_byte_impl(static_cast<unsigned char>((bit_count_high) & 0xFF));
  191. process_byte_impl(static_cast<unsigned char>((bit_count_low >> 24) & 0xFF));
  192. process_byte_impl(static_cast<unsigned char>((bit_count_low >> 16) & 0xFF));
  193. process_byte_impl(static_cast<unsigned char>((bit_count_low >> 8) & 0xFF));
  194. process_byte_impl(static_cast<unsigned char>((bit_count_low) & 0xFF));
  195. // get final digest
  196. digest[0] = h_[0];
  197. digest[1] = h_[1];
  198. digest[2] = h_[2];
  199. digest[3] = h_[3];
  200. digest[4] = h_[4];
  201. }
  202. unsigned int h_[5]{};
  203. unsigned char block_[64]{};
  204. std::size_t block_byte_index_{};
  205. std::size_t bit_count_low{};
  206. std::size_t bit_count_high{};
  207. };
  208. constexpr uint64_t twang_mix64(uint64_t key) noexcept {
  209. key = (~key) + (key << 21); // key *= (1 << 21) - 1; key -= 1;
  210. key = key ^ (key >> 24);
  211. key = key + (key << 3) + (key << 8); // key *= 1 + (1 << 3) + (1 << 8)
  212. key = key ^ (key >> 14);
  213. key = key + (key << 2) + (key << 4); // key *= 1 + (1 << 2) + (1 << 4)
  214. key = key ^ (key >> 28);
  215. key = key + (key << 31); // key *= 1 + (1 << 31)
  216. return key;
  217. }
  218. ////////////////////////////////////////////////////////////////////////////////
  219. // c10::hash implementation
  220. ////////////////////////////////////////////////////////////////////////////////
  221. namespace _hash_detail {
  222. // Use template argument deduction to shorten calls to c10::hash
  223. template <typename T>
  224. size_t simple_get_hash(const T& o);
  225. template <typename T, typename V>
  226. using type_if_not_enum = std::enable_if_t<!std::is_enum_v<T>, V>;
  227. // Use SFINAE to dispatch to std::hash if possible, cast enum types to int
  228. // automatically, and fall back to T::hash otherwise. NOTE: C++14 added support
  229. // for hashing enum types to the standard, and some compilers implement it even
  230. // when C++14 flags aren't specified. This is why we have to disable this
  231. // overload if T is an enum type (and use the one below in this case).
  232. template <typename T>
  233. auto dispatch_hash(const T& o)
  234. -> decltype(std::hash<T>()(o), type_if_not_enum<T, size_t>()) {
  235. return std::hash<T>()(o);
  236. }
  237. template <typename T>
  238. std::enable_if_t<std::is_enum_v<T>, size_t> dispatch_hash(const T& o) {
  239. using R = std::underlying_type_t<T>;
  240. return std::hash<R>()(static_cast<R>(o));
  241. }
  242. template <typename T>
  243. auto dispatch_hash(const T& o) -> decltype(T::hash(o), size_t()) {
  244. return T::hash(o);
  245. }
  246. } // namespace _hash_detail
  247. // Hasher struct
  248. template <typename T>
  249. struct hash {
  250. size_t operator()(const T& o) const {
  251. return _hash_detail::dispatch_hash(o);
  252. }
  253. };
  254. // Specialization for std::tuple
  255. template <typename... Types>
  256. struct hash<std::tuple<Types...>> {
  257. template <size_t idx, typename... Ts>
  258. struct tuple_hash {
  259. size_t operator()(const std::tuple<Ts...>& t) const {
  260. return hash_combine(
  261. _hash_detail::simple_get_hash(std::get<idx>(t)),
  262. tuple_hash<idx - 1, Ts...>()(t));
  263. }
  264. };
  265. template <typename... Ts>
  266. struct tuple_hash<0, Ts...> {
  267. size_t operator()(const std::tuple<Ts...>& t) const {
  268. return _hash_detail::simple_get_hash(std::get<0>(t));
  269. }
  270. };
  271. size_t operator()(const std::tuple<Types...>& t) const {
  272. return tuple_hash<sizeof...(Types) - 1, Types...>()(t);
  273. }
  274. };
  275. template <typename T1, typename T2>
  276. struct hash<std::pair<T1, T2>> {
  277. size_t operator()(const std::pair<T1, T2>& pair) const {
  278. std::tuple<T1, T2> tuple = std::make_tuple(pair.first, pair.second);
  279. return _hash_detail::simple_get_hash(tuple);
  280. }
  281. };
  282. template <typename T>
  283. struct hash<c10::ArrayRef<T>> {
  284. size_t operator()(c10::ArrayRef<T> v) const {
  285. size_t seed = 0;
  286. for (const auto& elem : v) {
  287. seed = hash_combine(seed, _hash_detail::simple_get_hash(elem));
  288. }
  289. return seed;
  290. }
  291. };
  292. // Specialization for std::vector
  293. template <typename T>
  294. struct hash<std::vector<T>> {
  295. size_t operator()(const std::vector<T>& v) const {
  296. return hash<c10::ArrayRef<T>>()(v);
  297. }
  298. };
  299. namespace _hash_detail {
  300. template <typename T>
  301. size_t simple_get_hash(const T& o) {
  302. return c10::hash<T>()(o);
  303. }
  304. } // namespace _hash_detail
  305. // Use this function to actually hash multiple things in one line.
  306. // Dispatches to c10::hash, so it can hash containers.
  307. // Example:
  308. //
  309. // static size_t hash(const MyStruct& s) {
  310. // return get_hash(s.member1, s.member2, s.member3);
  311. // }
  312. template <typename... Types>
  313. size_t get_hash(const Types&... args) {
  314. return c10::hash<decltype(std::tie(args...))>()(std::tie(args...));
  315. }
  316. // Specialization for c10::complex
  317. template <typename T>
  318. struct hash<c10::complex<T>> {
  319. size_t operator()(const c10::complex<T>& c) const {
  320. return get_hash(c.real(), c.imag());
  321. }
  322. };
  323. } // namespace c10