From 56271f7205cda0e1c1320450745369aab268a625 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 20 Aug 2025 14:55:59 +0200 Subject: [PATCH 1/7] Add BufferedMurmur3Hasher to reduce allocations when hashing Strings --- .../common/hash/BufferedMurmur3Hasher.java | 107 ++++++++++++++++++ .../common/hash/MurmurHash3.java | 13 ++- .../hash/BufferedMurmur3HasherTests.java | 77 +++++++++++++ 3 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java create mode 100644 server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java diff --git a/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java b/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java new file mode 100644 index 0000000000000..c488d7db687e0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.common.hash; + +import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.common.util.ByteUtils; + +/** + * A buffered Murmur3 hasher that allows adding strings and longs efficiently. + * It uses a byte array buffer to reduce allocations for converting strings and longs to bytes before passing them to the hasher. + */ +public class BufferedMurmur3Hasher extends Murmur3Hasher { + + public static final int DEFAULT_BUFFER_SIZE = 32 * 4; // 32 characters, each character may take up to 4 bytes in UTF-8 + /** + * The buffer used for holding the UTF-8 encoded strings before passing them to the hasher. + * Should be sized so that it can hold the longest UTF-8 encoded string that is expected to be hashed, + * to avoid re-sizing the buffer. + * But should also be small enough to not waste memory in case the keys are short. + */ + private byte[] buffer; + + public BufferedMurmur3Hasher(long seed) { + this(seed, DEFAULT_BUFFER_SIZE); + } + + /** + * Constructs a BufferedMurmur3Hasher with a specified seed and buffer size. + * + * @param seed the seed for the Murmur3 hash function + * @param bufferSize the size of the buffer in bytes, must be at least 32 + */ + public BufferedMurmur3Hasher(long seed, int bufferSize) { + super(seed); + if (bufferSize < 32) { + throw new IllegalArgumentException("Buffer size must be at least 32 bytes"); + } + this.buffer = new byte[bufferSize]; + } + + /** + * Adds a string to the hasher. + * The string is converted to UTF-8 and written into the buffer. + * The buffer is resized if necessary to accommodate the UTF-8 encoded string. + * + * @param value the string value to add + */ + public void addString(String value) { + ensureCapacity(UnicodeUtil.maxUTF8Length(value.length())); + int length = UnicodeUtil.UTF16toUTF8(value, 0, value.length(), buffer); + update(buffer, 0, length); + } + + /** + * Adds a long value to the hasher. + * The long is written in little-endian format. + * + * @param value the long value to add + */ + public void addLong(long value) { + ByteUtils.writeLongLE(value, buffer, 0); + update(buffer, 0, 8); + } + + /** + * Adds two long values to the hasher. + * Each long is written in little-endian format. + * + * @param v1 the first long value to add + * @param v2 the second long value to add + */ + public void addLongs(long v1, long v2) { + ByteUtils.writeLongLE(v1, buffer, 0); + ByteUtils.writeLongLE(v2, buffer, 8); + update(buffer, 0, 16); + } + + /** + * Adds four long values to the hasher. + * Each long is written in little-endian format. + * + * @param v1 the first long value to add + * @param v2 the second long value to add + * @param v3 the third long value to add + * @param v4 the fourth long value to add + */ + public void addLongs(long v1, long v2, long v3, long v4) { + ByteUtils.writeLongLE(v1, buffer, 0); + ByteUtils.writeLongLE(v2, buffer, 8); + ByteUtils.writeLongLE(v3, buffer, 16); + ByteUtils.writeLongLE(v4, buffer, 24); + update(buffer, 0, 32); + } + + private void ensureCapacity(int requiredBufferLength) { + if (buffer.length < requiredBufferLength) { + buffer = new byte[requiredBufferLength]; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java b/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java index 0abbb5d935156..db3f022f9353e 100644 --- a/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java +++ b/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.util.ByteUtils; import java.math.BigInteger; -import java.util.Objects; /** * MurmurHash3 hashing functions. @@ -29,6 +28,14 @@ public static class Hash128 { /** higher 64 bits part **/ public long h2; + public Hash128() { + } + + public Hash128(long h1, long h2) { + this.h1 = h1; + this.h2 = h2; + } + public byte[] getBytes() { byte[] hash = new byte[16]; getBytes(hash, 0); @@ -49,12 +56,12 @@ public boolean equals(Object other) { return false; } Hash128 that = (Hash128) other; - return Objects.equals(this.h1, that.h1) && Objects.equals(this.h2, that.h2); + return this.h1 == that.h1 && this.h2 == that.h2; } @Override public int hashCode() { - return Objects.hash(h1, h2); + return (int) (h1 ^ h2); } @Override diff --git a/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java b/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java new file mode 100644 index 0000000000000..8f0592b77ac73 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.common.hash; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.test.ESTestCase; + +public class BufferedMurmur3HasherTests extends ESTestCase { + + private final BufferedMurmur3Hasher bufferedHasher = new BufferedMurmur3Hasher(0); + private final Murmur3Hasher hasher = new Murmur3Hasher(0); + + public void testAddString() { + String testString = randomUnicodeOfLengthBetween(10, 100); + bufferedHasher.addString(testString); + + BytesRef bytesRef = new BytesRef(testString); + hasher.update(bytesRef.bytes, bytesRef.offset, bytesRef.length); + assertEquals(hasher.digestHash(), bufferedHasher.digestHash()); + } + + public void testConstructorWithInvalidBufferSize() { + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new BufferedMurmur3Hasher(0, 31)); + assertEquals("Buffer size must be at least 32 bytes", exception.getMessage()); + } + + public void testAddLong() { + long value = randomLong(); + bufferedHasher.addLong(value); + + hasher.update(toBytes(value), 0, Long.BYTES); + + assertEquals(hasher.digestHash(), bufferedHasher.digestHash()); + } + + public void testAddTwoLongs() { + long value1 = randomLong(); + long value2 = randomLong(); + + bufferedHasher.addLongs(value1, value2); + + hasher.update(toBytes(value1), 0, Long.BYTES); + hasher.update(toBytes(value2), 0, Long.BYTES); + + assertEquals(hasher.digestHash(), bufferedHasher.digestHash()); + } + + public void testAddFourLongs() { + long value1 = randomLong(); + long value2 = randomLong(); + long value3 = randomLong(); + long value4 = randomLong(); + + bufferedHasher.addLongs(value1, value2, value3, value4); + + hasher.update(toBytes(value1), 0, Long.BYTES); + hasher.update(toBytes(value2), 0, Long.BYTES); + hasher.update(toBytes(value3), 0, Long.BYTES); + hasher.update(toBytes(value4), 0, Long.BYTES); + + assertEquals(hasher.digestHash(), bufferedHasher.digestHash()); + } + + private byte[] toBytes(long value) { + byte[] bytes = new byte[Long.BYTES]; + ByteUtils.writeLongLE(value, bytes, 0); + return bytes; + } +} From b5663a05f94845fa0efb7ce635f8fc98e2b54df0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 20 Aug 2025 14:14:44 +0000 Subject: [PATCH 2/7] [CI] Auto commit changes from spotless --- .../main/java/org/elasticsearch/common/hash/MurmurHash3.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java b/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java index db3f022f9353e..49755efd22912 100644 --- a/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java +++ b/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java @@ -28,8 +28,7 @@ public static class Hash128 { /** higher 64 bits part **/ public long h2; - public Hash128() { - } + public Hash128() {} public Hash128(long h1, long h2) { this.h1 = h1; From ba6125fe53d5db79b1d7277dbbf40a901085dec3 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 22 Aug 2025 09:40:09 +0200 Subject: [PATCH 3/7] Test with larger range of string values Co-authored-by: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> --- .../elasticsearch/common/hash/BufferedMurmur3HasherTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java b/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java index 8f0592b77ac73..68295b017cbd8 100644 --- a/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java +++ b/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java @@ -19,7 +19,7 @@ public class BufferedMurmur3HasherTests extends ESTestCase { private final Murmur3Hasher hasher = new Murmur3Hasher(0); public void testAddString() { - String testString = randomUnicodeOfLengthBetween(10, 100); + String testString = randomUnicodeOfLengthBetween(0, 1024); bufferedHasher.addString(testString); BytesRef bytesRef = new BytesRef(testString); From b736312ab997f9c108dc29873e8d29377229d393 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 22 Aug 2025 10:09:08 +0200 Subject: [PATCH 4/7] Reduce number of flushes --- .../common/hash/BufferedMurmur3Hasher.java | 47 +++++++++++++--- .../hash/BufferedMurmur3HasherTests.java | 53 +++++++++++++++++++ 2 files changed, 93 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java b/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java index c488d7db687e0..1e8ffcbbd8dd3 100644 --- a/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java +++ b/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java @@ -13,8 +13,10 @@ import org.elasticsearch.common.util.ByteUtils; /** - * A buffered Murmur3 hasher that allows adding strings and longs efficiently. + * A buffered Murmur3 hasher that allows hashing strings and longs efficiently. * It uses a byte array buffer to reduce allocations for converting strings and longs to bytes before passing them to the hasher. + * The buffer also allows for more efficient execution by minimizing the number of times the underlying hasher is updated, + * and by maximizing the amount of data processed in each update call. */ public class BufferedMurmur3Hasher extends Murmur3Hasher { @@ -26,6 +28,7 @@ public class BufferedMurmur3Hasher extends Murmur3Hasher { * But should also be small enough to not waste memory in case the keys are short. */ private byte[] buffer; + private int pos; public BufferedMurmur3Hasher(long seed) { this(seed, DEFAULT_BUFFER_SIZE); @@ -45,6 +48,18 @@ public BufferedMurmur3Hasher(long seed, int bufferSize) { this.buffer = new byte[bufferSize]; } + @Override + public MurmurHash3.Hash128 digestHash(MurmurHash3.Hash128 hash) { + flush(); + return super.digestHash(hash); + } + + @Override + public void reset() { + super.reset(); + pos = 0; + } + /** * Adds a string to the hasher. * The string is converted to UTF-8 and written into the buffer. @@ -53,9 +68,10 @@ public BufferedMurmur3Hasher(long seed, int bufferSize) { * @param value the string value to add */ public void addString(String value) { - ensureCapacity(UnicodeUtil.maxUTF8Length(value.length())); - int length = UnicodeUtil.UTF16toUTF8(value, 0, value.length(), buffer); - update(buffer, 0, length); + int requiredBufferLength = UnicodeUtil.maxUTF8Length(value.length()); + ensureCapacity(requiredBufferLength); + flushIfRemainingCapacityLowerThan(requiredBufferLength); + pos = UnicodeUtil.UTF16toUTF8(value, 0, value.length(), buffer, pos); } /** @@ -65,8 +81,9 @@ public void addString(String value) { * @param value the long value to add */ public void addLong(long value) { + flushIfRemainingCapacityLowerThan(Long.BYTES); ByteUtils.writeLongLE(value, buffer, 0); - update(buffer, 0, 8); + pos = Long.BYTES; } /** @@ -77,9 +94,10 @@ public void addLong(long value) { * @param v2 the second long value to add */ public void addLongs(long v1, long v2) { + flushIfRemainingCapacityLowerThan(Long.BYTES * 2); ByteUtils.writeLongLE(v1, buffer, 0); ByteUtils.writeLongLE(v2, buffer, 8); - update(buffer, 0, 16); + pos = Long.BYTES * 2; } /** @@ -92,16 +110,31 @@ public void addLongs(long v1, long v2) { * @param v4 the fourth long value to add */ public void addLongs(long v1, long v2, long v3, long v4) { + flushIfRemainingCapacityLowerThan(Long.BYTES * 4); ByteUtils.writeLongLE(v1, buffer, 0); ByteUtils.writeLongLE(v2, buffer, 8); ByteUtils.writeLongLE(v3, buffer, 16); ByteUtils.writeLongLE(v4, buffer, 24); - update(buffer, 0, 32); + pos = Long.BYTES * 4; } private void ensureCapacity(int requiredBufferLength) { if (buffer.length < requiredBufferLength) { + flush(); buffer = new byte[requiredBufferLength]; } } + + private void flush() { + if (pos > 0) { + update(buffer, 0, pos); + pos = 0; + } + } + + private void flushIfRemainingCapacityLowerThan(int requiredCapacity) { + if (buffer.length - pos < requiredCapacity) { + flush(); + } + } } diff --git a/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java b/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java index 68295b017cbd8..3aa09285eae59 100644 --- a/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java +++ b/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java @@ -69,6 +69,59 @@ public void testAddFourLongs() { assertEquals(hasher.digestHash(), bufferedHasher.digestHash()); } + public void randomAdds() { + int numAdds = randomIntBetween(128, 1024); + for (int i = 0; i < numAdds; i++) { + switch (randomIntBetween(0, 4)) { + case 0 -> { + String randomString = randomUnicodeOfLengthBetween(0, 64); + bufferedHasher.addString(randomString); + BytesRef bytesRef = new BytesRef(randomString); + hasher.update(bytesRef.bytes, bytesRef.offset, bytesRef.length); + } + case 1 -> { + String emptyString = ""; + bufferedHasher.addString(emptyString); + BytesRef bytesRef = new BytesRef(emptyString); + hasher.update(bytesRef.bytes, bytesRef.offset, bytesRef.length); + } + case 2 -> { + long randomLong = randomLong(); + bufferedHasher.addLong(randomLong); + hasher.update(toBytes(randomLong), 0, Long.BYTES); + } + case 3 -> { + long randomLong1 = randomLong(); + long randomLong2 = randomLong(); + bufferedHasher.addLongs(randomLong1, randomLong2); + hasher.update(toBytes(randomLong1), 0, Long.BYTES); + hasher.update(toBytes(randomLong2), 0, Long.BYTES); + } + case 4 -> { + long randomLong1 = randomLong(); + long randomLong2 = randomLong(); + long randomLong3 = randomLong(); + long randomLong4 = randomLong(); + bufferedHasher.addLongs(randomLong1, randomLong2, randomLong3, randomLong4); + hasher.update(toBytes(randomLong1), 0, Long.BYTES); + hasher.update(toBytes(randomLong2), 0, Long.BYTES); + hasher.update(toBytes(randomLong3), 0, Long.BYTES); + hasher.update(toBytes(randomLong4), 0, Long.BYTES); + } + } + } + assertEquals(hasher.digestHash(), bufferedHasher.digestHash()); + } + + public void testReset() { + bufferedHasher.addString(randomUnicodeOfLengthBetween(0, 1024)); + bufferedHasher.addLong(randomLong()); + bufferedHasher.addLongs(randomLong(), randomLong()); + bufferedHasher.addLongs(randomLong(), randomLong(), randomLong(), randomLong()); + bufferedHasher.reset(); + assertEquals(new MurmurHash3.Hash128(0, 0), bufferedHasher.digestHash()); + } + private byte[] toBytes(long value) { byte[] bytes = new byte[Long.BYTES]; ByteUtils.writeLongLE(value, bytes, 0); From b277cef3bb60b75533d4a7efeb741bf94d2ce4c2 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 22 Aug 2025 10:20:49 +0200 Subject: [PATCH 5/7] Fix offsets --- .../common/hash/BufferedMurmur3Hasher.java | 14 +++++++------- .../common/hash/BufferedMurmur3HasherTests.java | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java b/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java index 1e8ffcbbd8dd3..7e3d1f59c2adb 100644 --- a/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java +++ b/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java @@ -82,7 +82,7 @@ public void addString(String value) { */ public void addLong(long value) { flushIfRemainingCapacityLowerThan(Long.BYTES); - ByteUtils.writeLongLE(value, buffer, 0); + ByteUtils.writeLongLE(value, buffer, pos); pos = Long.BYTES; } @@ -95,8 +95,8 @@ public void addLong(long value) { */ public void addLongs(long v1, long v2) { flushIfRemainingCapacityLowerThan(Long.BYTES * 2); - ByteUtils.writeLongLE(v1, buffer, 0); - ByteUtils.writeLongLE(v2, buffer, 8); + ByteUtils.writeLongLE(v1, buffer, pos); + ByteUtils.writeLongLE(v2, buffer, pos + 8); pos = Long.BYTES * 2; } @@ -111,10 +111,10 @@ public void addLongs(long v1, long v2) { */ public void addLongs(long v1, long v2, long v3, long v4) { flushIfRemainingCapacityLowerThan(Long.BYTES * 4); - ByteUtils.writeLongLE(v1, buffer, 0); - ByteUtils.writeLongLE(v2, buffer, 8); - ByteUtils.writeLongLE(v3, buffer, 16); - ByteUtils.writeLongLE(v4, buffer, 24); + ByteUtils.writeLongLE(v1, buffer, pos); + ByteUtils.writeLongLE(v2, buffer, pos + 8); + ByteUtils.writeLongLE(v3, buffer, pos + 16); + ByteUtils.writeLongLE(v4, buffer, pos + 24); pos = Long.BYTES * 4; } diff --git a/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java b/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java index 3aa09285eae59..61a49d69112ee 100644 --- a/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java +++ b/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java @@ -88,14 +88,14 @@ public void randomAdds() { case 2 -> { long randomLong = randomLong(); bufferedHasher.addLong(randomLong); - hasher.update(toBytes(randomLong), 0, Long.BYTES); + hasher.update(toBytes(randomLong)); } case 3 -> { long randomLong1 = randomLong(); long randomLong2 = randomLong(); bufferedHasher.addLongs(randomLong1, randomLong2); - hasher.update(toBytes(randomLong1), 0, Long.BYTES); - hasher.update(toBytes(randomLong2), 0, Long.BYTES); + hasher.update(toBytes(randomLong1)); + hasher.update(toBytes(randomLong2)); } case 4 -> { long randomLong1 = randomLong(); @@ -103,10 +103,10 @@ public void randomAdds() { long randomLong3 = randomLong(); long randomLong4 = randomLong(); bufferedHasher.addLongs(randomLong1, randomLong2, randomLong3, randomLong4); - hasher.update(toBytes(randomLong1), 0, Long.BYTES); - hasher.update(toBytes(randomLong2), 0, Long.BYTES); - hasher.update(toBytes(randomLong3), 0, Long.BYTES); - hasher.update(toBytes(randomLong4), 0, Long.BYTES); + hasher.update(toBytes(randomLong1)); + hasher.update(toBytes(randomLong2)); + hasher.update(toBytes(randomLong3)); + hasher.update(toBytes(randomLong4)); } } } From 2d8cb1cb89c108111dab0f084dab3b60c6ac9079 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 22 Aug 2025 10:38:12 +0200 Subject: [PATCH 6/7] More randomized testing and fixes --- .../common/hash/BufferedMurmur3Hasher.java | 6 +-- .../hash/BufferedMurmur3HasherTests.java | 38 +++++++++++++++---- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java b/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java index 7e3d1f59c2adb..328dffe974229 100644 --- a/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java +++ b/server/src/main/java/org/elasticsearch/common/hash/BufferedMurmur3Hasher.java @@ -83,7 +83,7 @@ public void addString(String value) { public void addLong(long value) { flushIfRemainingCapacityLowerThan(Long.BYTES); ByteUtils.writeLongLE(value, buffer, pos); - pos = Long.BYTES; + pos += Long.BYTES; } /** @@ -97,7 +97,7 @@ public void addLongs(long v1, long v2) { flushIfRemainingCapacityLowerThan(Long.BYTES * 2); ByteUtils.writeLongLE(v1, buffer, pos); ByteUtils.writeLongLE(v2, buffer, pos + 8); - pos = Long.BYTES * 2; + pos += Long.BYTES * 2; } /** @@ -115,7 +115,7 @@ public void addLongs(long v1, long v2, long v3, long v4) { ByteUtils.writeLongLE(v2, buffer, pos + 8); ByteUtils.writeLongLE(v3, buffer, pos + 16); ByteUtils.writeLongLE(v4, buffer, pos + 24); - pos = Long.BYTES * 4; + pos += Long.BYTES * 4; } private void ensureCapacity(int requiredBufferLength) { diff --git a/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java b/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java index 61a49d69112ee..47ef7ff26c283 100644 --- a/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java +++ b/server/src/test/java/org/elasticsearch/common/hash/BufferedMurmur3HasherTests.java @@ -15,7 +15,7 @@ public class BufferedMurmur3HasherTests extends ESTestCase { - private final BufferedMurmur3Hasher bufferedHasher = new BufferedMurmur3Hasher(0); + private final BufferedMurmur3Hasher bufferedHasher = new BufferedMurmur3Hasher(0, randomIntBetween(32, 128)); private final Murmur3Hasher hasher = new Murmur3Hasher(0); public void testAddString() { @@ -41,14 +41,36 @@ public void testAddLong() { assertEquals(hasher.digestHash(), bufferedHasher.digestHash()); } + public void testAddLongs() { + long value1 = randomLong(); + long value2 = randomLong(); + long value3 = randomLong(); + long value4 = randomLong(); + bufferedHasher.addLong(value1); + bufferedHasher.addLongs(value1, value2); + bufferedHasher.addLongs(value1, value2, value3, value4); + + hasher.update(toBytes(value1)); + + hasher.update(toBytes(value1)); + hasher.update(toBytes(value2)); + + hasher.update(toBytes(value1)); + hasher.update(toBytes(value2)); + hasher.update(toBytes(value3)); + hasher.update(toBytes(value4)); + + assertEquals(hasher.digestHash(), bufferedHasher.digestHash()); + } + public void testAddTwoLongs() { long value1 = randomLong(); long value2 = randomLong(); bufferedHasher.addLongs(value1, value2); - hasher.update(toBytes(value1), 0, Long.BYTES); - hasher.update(toBytes(value2), 0, Long.BYTES); + hasher.update(toBytes(value1)); + hasher.update(toBytes(value2)); assertEquals(hasher.digestHash(), bufferedHasher.digestHash()); } @@ -61,15 +83,15 @@ public void testAddFourLongs() { bufferedHasher.addLongs(value1, value2, value3, value4); - hasher.update(toBytes(value1), 0, Long.BYTES); - hasher.update(toBytes(value2), 0, Long.BYTES); - hasher.update(toBytes(value3), 0, Long.BYTES); - hasher.update(toBytes(value4), 0, Long.BYTES); + hasher.update(toBytes(value1)); + hasher.update(toBytes(value2)); + hasher.update(toBytes(value3)); + hasher.update(toBytes(value4)); assertEquals(hasher.digestHash(), bufferedHasher.digestHash()); } - public void randomAdds() { + public void testRandomAdds() { int numAdds = randomIntBetween(128, 1024); for (int i = 0; i < numAdds; i++) { switch (randomIntBetween(0, 4)) { From 35e22afa890f0b8162197f131728b8798bbcf2fd Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 22 Aug 2025 13:34:23 +0200 Subject: [PATCH 7/7] Use BufferedMurmur3Hasher in TsidBuilder --- .../cluster/routing/TsidBuilder.java | 43 +++---------------- 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java b/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java index 173400ddfcbbd..d29dced2adb28 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java @@ -10,7 +10,7 @@ package org.elasticsearch.cluster.routing; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.hash.Murmur3Hasher; +import org.elasticsearch.common.hash.BufferedMurmur3Hasher; import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.index.mapper.RoutingPathFields; @@ -32,7 +32,7 @@ public class TsidBuilder { private static final int MAX_TSID_VALUE_FIELDS = 16; - private final Murmur3Hasher murmur3Hasher = new Murmur3Hasher(0L); + private final BufferedMurmur3Hasher murmur3Hasher = new BufferedMurmur3Hasher(0L); private final List dimensions = new ArrayList<>(); @@ -166,7 +166,7 @@ public TsidBuilder add(T value, ThrowingTsidFunnel