From e335ba163151275abe6ece536201823a0638f5ee Mon Sep 17 00:00:00 2001 From: 0xbigapple Date: Wed, 11 Jun 2025 15:53:16 +0800 Subject: [PATCH 1/7] feat(api): optimize partialMatch --- .../jsonrpc/filters/LogBlockQuery.java | 97 +++++++++++++------ 1 file changed, 70 insertions(+), 27 deletions(-) diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java b/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java index 7665c51106f..d1da3532cf6 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java @@ -2,7 +2,11 @@ import java.util.ArrayList; import java.util.BitSet; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -121,44 +125,83 @@ private BitSet subMatch(int[][] bitIndexes) throws ExecutionException, Interrupt } /** - * every section has a compound query of sectionBloomStore, works parallel - * "and" condition in second dimension of query, "or" condition in first dimension - * return a BitSet whose capacity is blockPerSection + * Match blocks using optimized bloom filter operations. This method reduces database queries + * and BitSet operations by handling duplicate bit indexes and skipping invalid groups. + * + * @param bitIndexes A 2D array where: + * - First dimension represents different groups(OR) + * - Second dimension contains bit indexes within each group(AND) + * Example: [[1,2,3], [4,5,6]] means (1 AND 2 AND 3) OR (4 AND 5 AND 6) + * @param section The section number in the bloom filter store to query + * @return A BitSet representing the matching blocks in this section + * @throws ExecutionException If there's an error in concurrent execution + * @throws InterruptedException If the concurrent execution is interrupted */ private BitSet partialMatch(final int[][] bitIndexes, int section) throws ExecutionException, InterruptedException { - List>> bitSetList = new ArrayList<>(); - + // 1. Collect all unique bitIndexes + Set uniqueBitIndexes = new HashSet<>(); for (int[] index : bitIndexes) { - List> futureList = new ArrayList<>(); - for (final int bitIndex : index) { //must be 3 - Future bitSetFuture = - sectionExecutor.submit(() -> sectionBloomStore.get(section, bitIndex)); - futureList.add(bitSetFuture); + for (int bitIndex : index) { + uniqueBitIndexes.add(bitIndex); + } + } + + // 2. Submit concurrent requests for all unique bitIndexes + Map> bitIndexResults = new HashMap<>(); + for (int bitIndex : uniqueBitIndexes) { + Future future + = sectionExecutor.submit(() -> sectionBloomStore.get(section, bitIndex)); + bitIndexResults.put(bitIndex, future); + } + + // 3. Wait for all results and cache them + Map resultCache = new HashMap<>(); + for (Map.Entry> entry : bitIndexResults.entrySet()) { + BitSet result = entry.getValue().get(); + if (result != null) { + resultCache.put(entry.getKey(), result); + } + } + + // 4. Find groups to skip (those with missing bitIndexes) + Set skipGroups = new HashSet<>(); + for (int i = 0; i < bitIndexes.length; i++) { + for (int bitIndex : bitIndexes[i]) { + if (!resultCache.containsKey(bitIndex)) { + skipGroups.add(i); + break; + } } - bitSetList.add(futureList); } - BitSet bitSet = new BitSet(SectionBloomStore.BLOCK_PER_SECTION); - - for (List> futureList : bitSetList) { - // initial a BitSet with all 1 - BitSet subBitSet = new BitSet(SectionBloomStore.BLOCK_PER_SECTION); - subBitSet.set(0, SectionBloomStore.BLOCK_PER_SECTION); - // and condition in second dimension - for (Future future : futureList) { - BitSet one = future.get(); - if (one == null) { //match nothing - subBitSet.clear(); + // 5. Process valid groups with reused BitSet objects + BitSet finalResult = new BitSet(SectionBloomStore.BLOCK_PER_SECTION); + BitSet tempBitSet = new BitSet(SectionBloomStore.BLOCK_PER_SECTION); + + for (int i = 0; i < bitIndexes.length; i++) { + if (skipGroups.contains(i)) { + continue; + } + + tempBitSet.clear(); + tempBitSet.set(0, SectionBloomStore.BLOCK_PER_SECTION); + + // Process multiple bitIndexes + for (int bitIndex : bitIndexes[i]) { + BitSet cached = resultCache.get(bitIndex); + tempBitSet.and(cached); + if (tempBitSet.isEmpty()) { break; } - // "and" condition in second dimension - subBitSet.and(one); } - // "or" condition in first dimension - bitSet.or(subBitSet); + + if (!tempBitSet.isEmpty()) { + finalResult.or(tempBitSet); + } } - return bitSet; + + return finalResult; } /** From 2f6c7944f27a9490dc1eb1d549fc4cd3bbe23d8c Mon Sep 17 00:00:00 2001 From: 0xbigapple Date: Wed, 11 Jun 2025 16:22:36 +0800 Subject: [PATCH 2/7] feat(api): remove skipGroups in partialMatch --- .../jsonrpc/filters/LogBlockQuery.java | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java b/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java index d1da3532cf6..35c655fe0e5 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java @@ -129,8 +129,8 @@ private BitSet subMatch(int[][] bitIndexes) throws ExecutionException, Interrupt * and BitSet operations by handling duplicate bit indexes and skipping invalid groups. * * @param bitIndexes A 2D array where: - * - First dimension represents different groups(OR) - * - Second dimension contains bit indexes within each group(AND) + * - First dimension represents different topic/address (OR) + * - Second dimension contains bit indexes within each topic/address (AND) * Example: [[1,2,3], [4,5,6]] means (1 AND 2 AND 3) OR (4 AND 5 AND 6) * @param section The section number in the bloom filter store to query * @return A BitSet representing the matching blocks in this section @@ -164,31 +164,17 @@ private BitSet partialMatch(final int[][] bitIndexes, int section) } } - // 4. Find groups to skip (those with missing bitIndexes) - Set skipGroups = new HashSet<>(); - for (int i = 0; i < bitIndexes.length; i++) { - for (int bitIndex : bitIndexes[i]) { - if (!resultCache.containsKey(bitIndex)) { - skipGroups.add(i); - break; - } - } - } - // 5. Process valid groups with reused BitSet objects + // 4. Process valid groups with reused BitSet objects BitSet finalResult = new BitSet(SectionBloomStore.BLOCK_PER_SECTION); BitSet tempBitSet = new BitSet(SectionBloomStore.BLOCK_PER_SECTION); - for (int i = 0; i < bitIndexes.length; i++) { - if (skipGroups.contains(i)) { - continue; - } + for (int[] index : bitIndexes) { tempBitSet.clear(); tempBitSet.set(0, SectionBloomStore.BLOCK_PER_SECTION); - // Process multiple bitIndexes - for (int bitIndex : bitIndexes[i]) { + for (int bitIndex : index) { BitSet cached = resultCache.get(bitIndex); tempBitSet.and(cached); if (tempBitSet.isEmpty()) { From fd364b1365842e0dfa958de501fbca6c8a21b619 Mon Sep 17 00:00:00 2001 From: 0xbigapple Date: Thu, 12 Jun 2025 21:30:16 +0800 Subject: [PATCH 3/7] feat(api): fix bug --- .../org/tron/core/services/jsonrpc/filters/LogBlockQuery.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java b/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java index 35c655fe0e5..bb8060ef325 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java @@ -176,6 +176,10 @@ private BitSet partialMatch(final int[][] bitIndexes, int section) for (int bitIndex : index) { BitSet cached = resultCache.get(bitIndex); + if (cached == null) { + tempBitSet.clear(); + break; + } tempBitSet.and(cached); if (tempBitSet.isEmpty()) { break; From dd73f03456d157073a37c884bae7004a30ff69c7 Mon Sep 17 00:00:00 2001 From: 0xbigapple Date: Fri, 13 Jun 2025 23:45:52 +0800 Subject: [PATCH 4/7] feat(api): add test --- .../tron/core/jsonrpc/LogBlockQueryTest.java | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 framework/src/test/java/org/tron/core/jsonrpc/LogBlockQueryTest.java diff --git a/framework/src/test/java/org/tron/core/jsonrpc/LogBlockQueryTest.java b/framework/src/test/java/org/tron/core/jsonrpc/LogBlockQueryTest.java new file mode 100644 index 00000000000..1b42bdc0b99 --- /dev/null +++ b/framework/src/test/java/org/tron/core/jsonrpc/LogBlockQueryTest.java @@ -0,0 +1,111 @@ +package org.tron.core.jsonrpc; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import javax.annotation.Resource; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.tron.common.BaseTest; +import org.tron.common.bloom.Bloom; +import org.tron.common.crypto.Hash; +import org.tron.common.runtime.vm.DataWord; +import org.tron.common.runtime.vm.LogInfo; +import org.tron.common.utils.ByteArray; +import org.tron.core.Constant; +import org.tron.core.capsule.TransactionRetCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.exception.EventBloomException; +import org.tron.core.services.jsonrpc.TronJsonRpc.FilterRequest; +import org.tron.core.services.jsonrpc.filters.LogBlockQuery; +import org.tron.core.services.jsonrpc.filters.LogFilterWrapper; +import org.tron.core.store.SectionBloomStore; +import org.tron.protos.Protocol.TransactionInfo; +import org.tron.protos.Protocol.TransactionInfo.Log; + +public class LogBlockQueryTest extends BaseTest { + + @Resource + SectionBloomStore sectionBloomStore; + private ExecutorService sectionExecutor; + private Method partialMatchMethod; + private static final long CURRENT_MAX_BLOCK_NUM = 50000L; + + static { + Args.setParam(new String[] {"--output-directory", dbPath()}, Constant.TEST_CONF); + } + + @Before + public void setup() throws Exception { + sectionExecutor = Executors.newFixedThreadPool(5); + + // Get private method through reflection + partialMatchMethod = LogBlockQuery.class.getDeclaredMethod("partialMatch", + int[][].class, int.class); + partialMatchMethod.setAccessible(true); + + BitSet bitSet = new BitSet(SectionBloomStore.BLOCK_PER_SECTION); + bitSet.set(0, SectionBloomStore.BLOCK_PER_SECTION); + sectionBloomStore.put(0, 1, bitSet); + sectionBloomStore.put(0, 2, bitSet); + sectionBloomStore.put(0, 3, bitSet); + BitSet bitSet2 = new BitSet(SectionBloomStore.BLOCK_PER_SECTION); + bitSet2.set(0); + sectionBloomStore.put(1, 1, bitSet2); + sectionBloomStore.put(1, 2, bitSet2); + sectionBloomStore.put(1, 3, bitSet2); + } + + @Test + public void testPartialMatchNormalCase() throws Exception { + // Create a basic LogFilterWrapper + LogFilterWrapper logFilterWrapper = new LogFilterWrapper( + new FilterRequest("0x0", "0x1", null, null, null), + CURRENT_MAX_BLOCK_NUM, null, false); + + LogBlockQuery logBlockQuery = new LogBlockQuery(logFilterWrapper, sectionBloomStore, + CURRENT_MAX_BLOCK_NUM, sectionExecutor); + + int section = 0; + + // Create a hit condition + int[][] bitIndexes = new int[][] { + {1, 2, 3}, // topic0 + {4, 5, 6} // topic1 + }; + + // topic0 hit section 0 + BitSet result = (BitSet) partialMatchMethod.invoke(logBlockQuery, bitIndexes, section); + Assert.assertNotNull(result); + Assert.assertEquals(SectionBloomStore.BLOCK_PER_SECTION, result.cardinality()); + + // topic0 hit section 1 + result = (BitSet) partialMatchMethod.invoke(logBlockQuery, bitIndexes, 1); + Assert.assertNotNull(result); + Assert.assertEquals(1, result.cardinality()); + + // not exist section 2 + result = (BitSet) partialMatchMethod.invoke(logBlockQuery, bitIndexes, 2); + Assert.assertNotNull(result); + Assert.assertTrue(result.isEmpty()); + + //not hit + bitIndexes = new int[][] { + {1, 2, 4}, // topic0 + {3, 5, 6} // topic1 + }; + result = (BitSet) partialMatchMethod.invoke(logBlockQuery, bitIndexes, section); + Assert.assertNotNull(result); + Assert.assertTrue(result.isEmpty()); + + // null condition + bitIndexes = new int[0][]; + result = (BitSet) partialMatchMethod.invoke(logBlockQuery, bitIndexes, section); + Assert.assertNotNull(result); + Assert.assertTrue(result.isEmpty()); + } +} \ No newline at end of file From 34edc6606294903eeec023d1c117b6a3567bccc3 Mon Sep 17 00:00:00 2001 From: 0xbigapple Date: Tue, 17 Jun 2025 15:32:25 +0800 Subject: [PATCH 5/7] feat(api): add unit test --- .../org/tron/core/jsonrpc/LogBlockQueryTest.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/framework/src/test/java/org/tron/core/jsonrpc/LogBlockQueryTest.java b/framework/src/test/java/org/tron/core/jsonrpc/LogBlockQueryTest.java index 1b42bdc0b99..deae8babb32 100644 --- a/framework/src/test/java/org/tron/core/jsonrpc/LogBlockQueryTest.java +++ b/framework/src/test/java/org/tron/core/jsonrpc/LogBlockQueryTest.java @@ -1,9 +1,7 @@ package org.tron.core.jsonrpc; import java.lang.reflect.Method; -import java.util.ArrayList; import java.util.BitSet; -import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import javax.annotation.Resource; @@ -11,21 +9,12 @@ import org.junit.Before; import org.junit.Test; import org.tron.common.BaseTest; -import org.tron.common.bloom.Bloom; -import org.tron.common.crypto.Hash; -import org.tron.common.runtime.vm.DataWord; -import org.tron.common.runtime.vm.LogInfo; -import org.tron.common.utils.ByteArray; import org.tron.core.Constant; -import org.tron.core.capsule.TransactionRetCapsule; import org.tron.core.config.args.Args; -import org.tron.core.exception.EventBloomException; import org.tron.core.services.jsonrpc.TronJsonRpc.FilterRequest; import org.tron.core.services.jsonrpc.filters.LogBlockQuery; import org.tron.core.services.jsonrpc.filters.LogFilterWrapper; import org.tron.core.store.SectionBloomStore; -import org.tron.protos.Protocol.TransactionInfo; -import org.tron.protos.Protocol.TransactionInfo.Log; public class LogBlockQueryTest extends BaseTest { @@ -61,7 +50,7 @@ public void setup() throws Exception { } @Test - public void testPartialMatchNormalCase() throws Exception { + public void testPartialMatch() throws Exception { // Create a basic LogFilterWrapper LogFilterWrapper logFilterWrapper = new LogFilterWrapper( new FilterRequest("0x0", "0x1", null, null, null), From c72ee81616b59db62ec7851b31ac6e28d316aa24 Mon Sep 17 00:00:00 2001 From: 0xbigapple Date: Wed, 18 Jun 2025 18:40:04 +0800 Subject: [PATCH 6/7] feat(api): add inline comments --- .../tron/core/services/jsonrpc/filters/LogBlockQuery.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java b/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java index bb8060ef325..b4dea4325ac 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java @@ -142,7 +142,7 @@ private BitSet partialMatch(final int[][] bitIndexes, int section) // 1. Collect all unique bitIndexes Set uniqueBitIndexes = new HashSet<>(); for (int[] index : bitIndexes) { - for (int bitIndex : index) { + for (int bitIndex : index) { //normally 3, but could be less due to hash collisions uniqueBitIndexes.add(bitIndex); } } @@ -171,21 +171,25 @@ private BitSet partialMatch(final int[][] bitIndexes, int section) for (int[] index : bitIndexes) { + // init tempBitSet with all 1 tempBitSet.clear(); tempBitSet.set(0, SectionBloomStore.BLOCK_PER_SECTION); + // and condition in second dimension for (int bitIndex : index) { BitSet cached = resultCache.get(bitIndex); - if (cached == null) { + if (cached == null) { //match nothing tempBitSet.clear(); break; } + // "and" condition in second dimension tempBitSet.and(cached); if (tempBitSet.isEmpty()) { break; } } + // "or" condition in first dimension if (!tempBitSet.isEmpty()) { finalResult.or(tempBitSet); } From 1884a103a702d601e798e3982049f4c493d6626e Mon Sep 17 00:00:00 2001 From: 0xbigapple Date: Tue, 24 Jun 2025 13:37:59 +0800 Subject: [PATCH 7/7] feat(api): remove unnecessary code --- .../org/tron/core/services/jsonrpc/filters/LogBlockQuery.java | 1 - 1 file changed, 1 deletion(-) diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java b/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java index b4dea4325ac..d627509bd40 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/filters/LogBlockQuery.java @@ -172,7 +172,6 @@ private BitSet partialMatch(final int[][] bitIndexes, int section) for (int[] index : bitIndexes) { // init tempBitSet with all 1 - tempBitSet.clear(); tempBitSet.set(0, SectionBloomStore.BLOCK_PER_SECTION); // and condition in second dimension