From 712e6ee960744b8b6167fa8d8553313b06e9424a Mon Sep 17 00:00:00 2001 From: Yuri Kunde Schlesner Date: Mon, 15 Jan 2018 01:38:54 -0800 Subject: [PATCH] Common: Remove CityHash32 and CityHashCrc128 variants In 64-bit systems, CityHash64 is always strictly better than CityHash32. CityHashCrc128 requires SSE 4.2. --- src/common/cityhash.cpp | 272 ---------------------------------------- src/common/cityhash.h | 3 - 2 files changed, 275 deletions(-) diff --git a/src/common/cityhash.cpp b/src/common/cityhash.cpp index ab8db77746..596b943e76 100644 --- a/src/common/cityhash.cpp +++ b/src/common/cityhash.cpp @@ -135,141 +135,6 @@ static const uint64 k0 = 0xc3a5c85c97cb3127ULL; static const uint64 k1 = 0xb492b66fbe98f273ULL; static const uint64 k2 = 0x9ae16a3b2f90404fULL; -// Magic numbers for 32-bit hashing. Copied from Murmur3. -static const uint32 c1 = 0xcc9e2d51; -static const uint32 c2 = 0x1b873593; - -// A 32-bit to 32-bit integer hash copied from Murmur3. -static uint32 fmix(uint32 h) { - h ^= h >> 16; - h *= 0x85ebca6b; - h ^= h >> 13; - h *= 0xc2b2ae35; - h ^= h >> 16; - return h; -} - -static uint32 Rotate32(uint32 val, int shift) { - // Avoid shifting by 32: doing so yields an undefined result. - return shift == 0 ? val : ((val >> shift) | (val << (32 - shift))); -} - -#undef PERMUTE3 -#define PERMUTE3(a, b, c) \ - do { \ - std::swap(a, b); \ - std::swap(a, c); \ - } while (0) - -static uint32 Mur(uint32 a, uint32 h) { - // Helper from Murmur3 for combining two 32-bit values. - a *= c1; - a = Rotate32(a, 17); - a *= c2; - h ^= a; - h = Rotate32(h, 19); - return h * 5 + 0xe6546b64; -} - -static uint32 Hash32Len13to24(const char* s, size_t len) { - uint32 a = Fetch32(s - 4 + (len >> 1)); - uint32 b = Fetch32(s + 4); - uint32 c = Fetch32(s + len - 8); - uint32 d = Fetch32(s + (len >> 1)); - uint32 e = Fetch32(s); - uint32 f = Fetch32(s + len - 4); - uint32 h = len; - - return fmix(Mur(f, Mur(e, Mur(d, Mur(c, Mur(b, Mur(a, h))))))); -} - -static uint32 Hash32Len0to4(const char* s, size_t len) { - uint32 b = 0; - uint32 c = 9; - for (size_t i = 0; i < len; i++) { - signed char v = s[i]; - b = b * c1 + v; - c ^= b; - } - return fmix(Mur(b, Mur(len, c))); -} - -static uint32 Hash32Len5to12(const char* s, size_t len) { - uint32 a = len, b = len * 5, c = 9, d = b; - a += Fetch32(s); - b += Fetch32(s + len - 4); - c += Fetch32(s + ((len >> 1) & 4)); - return fmix(Mur(c, Mur(b, Mur(a, d)))); -} - -uint32 CityHash32(const char* s, size_t len) { - if (len <= 24) { - return len <= 12 ? (len <= 4 ? Hash32Len0to4(s, len) : Hash32Len5to12(s, len)) - : Hash32Len13to24(s, len); - } - - // len > 24 - uint32 h = len, g = c1 * len, f = g; - uint32 a0 = Rotate32(Fetch32(s + len - 4) * c1, 17) * c2; - uint32 a1 = Rotate32(Fetch32(s + len - 8) * c1, 17) * c2; - uint32 a2 = Rotate32(Fetch32(s + len - 16) * c1, 17) * c2; - uint32 a3 = Rotate32(Fetch32(s + len - 12) * c1, 17) * c2; - uint32 a4 = Rotate32(Fetch32(s + len - 20) * c1, 17) * c2; - h ^= a0; - h = Rotate32(h, 19); - h = h * 5 + 0xe6546b64; - h ^= a2; - h = Rotate32(h, 19); - h = h * 5 + 0xe6546b64; - g ^= a1; - g = Rotate32(g, 19); - g = g * 5 + 0xe6546b64; - g ^= a3; - g = Rotate32(g, 19); - g = g * 5 + 0xe6546b64; - f += a4; - f = Rotate32(f, 19); - f = f * 5 + 0xe6546b64; - size_t iters = (len - 1) / 20; - do { - uint32 a0 = Rotate32(Fetch32(s) * c1, 17) * c2; - uint32 a1 = Fetch32(s + 4); - uint32 a2 = Rotate32(Fetch32(s + 8) * c1, 17) * c2; - uint32 a3 = Rotate32(Fetch32(s + 12) * c1, 17) * c2; - uint32 a4 = Fetch32(s + 16); - h ^= a0; - h = Rotate32(h, 18); - h = h * 5 + 0xe6546b64; - f += a1; - f = Rotate32(f, 19); - f = f * c1; - g += a2; - g = Rotate32(g, 18); - g = g * 5 + 0xe6546b64; - h ^= a3 + a1; - h = Rotate32(h, 19); - h = h * 5 + 0xe6546b64; - g ^= a4; - g = bswap_32(g) * 5; - h += a4 * 5; - h = bswap_32(h); - f += a0; - PERMUTE3(f, h, g); - s += 20; - } while (--iters != 0); - g = Rotate32(g, 11) * c1; - g = Rotate32(g, 17) * c1; - f = Rotate32(f, 11) * c1; - f = Rotate32(f, 17) * c1; - h = Rotate32(h + g, 19); - h = h * 5 + 0xe6546b64; - h = Rotate32(h, 17) * c1; - h = Rotate32(h + f, 19); - h = h * 5 + 0xe6546b64; - h = Rotate32(h, 17) * c1; - return h; -} - // Bitwise right rotate. Normally this will compile to a single // instruction, especially if the shift is a manifest constant. static uint64 Rotate(uint64 val, int shift) { @@ -518,141 +383,4 @@ uint128 CityHash128(const char* s, size_t len) { : CityHash128WithSeed(s, len, uint128(k0, k1)); } -#ifdef __SSE4_2__ -#include -#include - -// Requires len >= 240. -static void CityHashCrc256Long(const char* s, size_t len, uint32 seed, uint64* result) { - uint64 a = Fetch64(s + 56) + k0; - uint64 b = Fetch64(s + 96) + k0; - uint64 c = result[0] = HashLen16(b, len); - uint64 d = result[1] = Fetch64(s + 120) * k0 + len; - uint64 e = Fetch64(s + 184) + seed; - uint64 f = 0; - uint64 g = 0; - uint64 h = c + d; - uint64 x = seed; - uint64 y = 0; - uint64 z = 0; - - // 240 bytes of input per iter. - size_t iters = len / 240; - len -= iters * 240; - do { -#undef CHUNK -#define CHUNK(r) \ - PERMUTE3(x, z, y); \ - b += Fetch64(s); \ - c += Fetch64(s + 8); \ - d += Fetch64(s + 16); \ - e += Fetch64(s + 24); \ - f += Fetch64(s + 32); \ - a += b; \ - h += f; \ - b += c; \ - f += d; \ - g += e; \ - e += z; \ - g += x; \ - z = _mm_crc32_u64(z, b + g); \ - y = _mm_crc32_u64(y, e + h); \ - x = _mm_crc32_u64(x, f + a); \ - e = Rotate(e, r); \ - c += e; \ - s += 40 - - CHUNK(0); - PERMUTE3(a, h, c); - CHUNK(33); - PERMUTE3(a, h, f); - CHUNK(0); - PERMUTE3(b, h, f); - CHUNK(42); - PERMUTE3(b, h, d); - CHUNK(0); - PERMUTE3(b, h, e); - CHUNK(33); - PERMUTE3(a, h, e); - } while (--iters > 0); - - while (len >= 40) { - CHUNK(29); - e ^= Rotate(a, 20); - h += Rotate(b, 30); - g ^= Rotate(c, 40); - f += Rotate(d, 34); - PERMUTE3(c, h, g); - len -= 40; - } - if (len > 0) { - s = s + len - 40; - CHUNK(33); - e ^= Rotate(a, 43); - h += Rotate(b, 42); - g ^= Rotate(c, 41); - f += Rotate(d, 40); - } - result[0] ^= h; - result[1] ^= g; - g += h; - a = HashLen16(a, g + z); - x += y << 32; - b += x; - c = HashLen16(c, z) + h; - d = HashLen16(d, e + result[0]); - g += e; - h += HashLen16(x, f); - e = HashLen16(a, d) + g; - z = HashLen16(b, c) + a; - y = HashLen16(g, h) + c; - result[0] = e + z + y + x; - a = ShiftMix((a + y) * k0) * k0 + b; - result[1] += a + result[0]; - a = ShiftMix(a * k0) * k0 + c; - result[2] = a + result[1]; - a = ShiftMix((a + e) * k0) * k0; - result[3] = a + result[2]; -} - -// Requires len < 240. -static void CityHashCrc256Short(const char* s, size_t len, uint64* result) { - char buf[240]; - memcpy(buf, s, len); - memset(buf + len, 0, 240 - len); - CityHashCrc256Long(buf, 240, ~static_cast(len), result); -} - -void CityHashCrc256(const char* s, size_t len, uint64* result) { - if (LIKELY(len >= 240)) { - CityHashCrc256Long(s, len, 0, result); - } else { - CityHashCrc256Short(s, len, result); - } -} - -uint128 CityHashCrc128WithSeed(const char* s, size_t len, uint128 seed) { - if (len <= 900) { - return CityHash128WithSeed(s, len, seed); - } else { - uint64 result[4]; - CityHashCrc256(s, len, result); - uint64 u = Uint128High64(seed) + result[0]; - uint64 v = Uint128Low64(seed) + result[1]; - return uint128(HashLen16(u, v + result[2]), HashLen16(Rotate(v, 32), u * k0 + result[3])); - } -} - -uint128 CityHashCrc128(const char* s, size_t len) { - if (len <= 900) { - return CityHash128(s, len); - } else { - uint64 result[4]; - CityHashCrc256(s, len, result); - return uint128(result[2], result[3]); - } -} - -#endif - } // namespace Common diff --git a/src/common/cityhash.h b/src/common/cityhash.h index e6baf6a371..bcebdb1507 100644 --- a/src/common/cityhash.h +++ b/src/common/cityhash.h @@ -94,9 +94,6 @@ uint128 CityHash128(const char* s, size_t len); // hashed into the result. uint128 CityHash128WithSeed(const char* s, size_t len, uint128 seed); -// Hash function for a byte array. Most useful in 32-bit binaries. -uint32_t CityHash32(const char* buf, size_t len); - // Hash 128 input bits down to 64 bits of output. // This is intended to be a reasonably good hash function. inline uint64_t Hash128to64(const uint128& x) {