Skip to content

Commit 11a4317

Browse files
ahamlatahamlat
authored
Improve debug_traceBlock calls performance (#8103)
- Change debug_traceBlockByNumber implementation by using a pipeline - Change the defaults to match Geth and Nethermind defaults - Improve general performance - Add unit tests Signed-off-by: Ameziane H. <[email protected]>
1 parent daf4aae commit 11a4317

File tree

48 files changed

+4268
-7286
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+4268
-7286
lines changed

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
### Breaking Changes
66
- `--host-whitelist` has been deprecated since 2020 and this option is removed. Use the equivalent `--host-allowlist` instead.
77
- Changed tracer API to include the mining beneficiary in BlockAwareOperationTracer::traceStartBlock [#8096](https://github.com/hyperledger/besu/pull/8096)
8+
- Change the input defaults on debug_trace* calls to not trace memory by default ("disableMemory": true, "disableStack": false, "disableStorage": false)
9+
- Change the output format of debug_trace* and trace_* calls to match Geth behaviour
810

911
### Upcoming Breaking Changes
1012
- `MetricSystem::createLabelledGauge` is deprecated and will be removed in a future release, replace it with `MetricSystem::createLabelledSuppliedGauge`
@@ -20,6 +22,7 @@
2022
- Retrieve all transaction receipts for a block in one request [#6646](https://github.com/hyperledger/besu/pull/6646)
2123
- Implement EIP-7840: Add blob schedule to config files [#8042](https://github.com/hyperledger/besu/pull/8042)
2224
- Allow gasPrice (legacy) and 1559 gasPrice params to be specified simultaneously for `eth_call`, `eth_createAccessList`, and `eth_estimateGas` [#8059](https://github.com/hyperledger/besu/pull/8059)
25+
- Improve debug_traceBlock calls performance and reduce output size [#8076](https://github.com/hyperledger/besu/pull/8076)
2326
- Add support for EIP-7702 transaction in the txpool [#8018](https://github.com/hyperledger/besu/pull/8018) [#7984](https://github.com/hyperledger/besu/pull/7984)
2427

2528
### Bug fixes

‎besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright ConsenSys AG.
2+
* Copyright contributors to Besu.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
55
* the License. You may obtain a copy of the License at
@@ -74,6 +74,7 @@
7474
import org.hyperledger.besu.ethereum.core.PrivacyParameters;
7575
import org.hyperledger.besu.ethereum.core.Synchronizer;
7676
import org.hyperledger.besu.ethereum.eth.manager.EthPeers;
77+
import org.hyperledger.besu.ethereum.eth.manager.EthScheduler;
7778
import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool;
7879
import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
7980
import org.hyperledger.besu.ethereum.mainnet.precompiles.privacy.FlexiblePrivacyPrecompiledContract;
@@ -836,7 +837,8 @@ public Runner build() {
836837
besuPluginContext.getNamedPlugins(),
837838
dataDir,
838839
rpcEndpointServiceImpl,
839-
transactionSimulator);
840+
transactionSimulator,
841+
besuController.getProtocolManager().ethContext().getScheduler());
840842

841843
jsonRpcHttpService =
842844
Optional.of(
@@ -882,7 +884,8 @@ public Runner build() {
882884
besuPluginContext.getNamedPlugins(),
883885
dataDir,
884886
rpcEndpointServiceImpl,
885-
transactionSimulator);
887+
transactionSimulator,
888+
besuController.getProtocolManager().ethContext().getScheduler());
886889

887890
final Optional<AuthenticationService> authToUse =
888891
engineJsonRpcConfiguration.get().isAuthenticationEnabled()
@@ -978,7 +981,8 @@ public Runner build() {
978981
besuPluginContext.getNamedPlugins(),
979982
dataDir,
980983
rpcEndpointServiceImpl,
981-
transactionSimulator);
984+
transactionSimulator,
985+
besuController.getProtocolManager().ethContext().getScheduler());
982986

983987
createLogsSubscriptionService(
984988
context.getBlockchain(), subscriptionManager, privacyParameters, blockchainQueries);
@@ -1059,7 +1063,8 @@ public Runner build() {
10591063
besuPluginContext.getNamedPlugins(),
10601064
dataDir,
10611065
rpcEndpointServiceImpl,
1062-
transactionSimulator);
1066+
transactionSimulator,
1067+
besuController.getProtocolManager().ethContext().getScheduler());
10631068

10641069
jsonRpcIpcService =
10651070
Optional.of(
@@ -1099,7 +1104,8 @@ public Runner build() {
10991104
besuPluginContext.getNamedPlugins(),
11001105
dataDir,
11011106
rpcEndpointServiceImpl,
1102-
transactionSimulator);
1107+
transactionSimulator,
1108+
besuController.getProtocolManager().ethContext().getScheduler());
11031109
} else {
11041110
inProcessRpcMethods = Map.of();
11051111
}
@@ -1262,7 +1268,8 @@ private Map<String, JsonRpcMethod> jsonRpcMethods(
12621268
final Map<String, BesuPlugin> namedPlugins,
12631269
final Path dataDir,
12641270
final RpcEndpointServiceImpl rpcEndpointServiceImpl,
1265-
final TransactionSimulator transactionSimulator) {
1271+
final TransactionSimulator transactionSimulator,
1272+
final EthScheduler ethScheduler) {
12661273
// sync vertx for engine consensus API, to process requests in FIFO order;
12671274
final Vertx consensusEngineServer = Vertx.vertx(new VertxOptions().setWorkerPoolSize(1));
12681275

@@ -1300,7 +1307,8 @@ private Map<String, JsonRpcMethod> jsonRpcMethods(
13001307
consensusEngineServer,
13011308
apiConfiguration,
13021309
enodeDnsConfiguration,
1303-
transactionSimulator);
1310+
transactionSimulator,
1311+
ethScheduler);
13041312
methods.putAll(besuController.getAdditionalJsonRpcMethods(jsonRpcApis));
13051313

13061314
final var pluginMethods =

‎ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright ConsenSys AG.
2+
* Copyright contributors to Besu.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
55
* the License. You may obtain a copy of the License at
@@ -51,6 +51,7 @@
5151
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
5252
import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
5353
import org.hyperledger.besu.nat.NatService;
54+
import org.hyperledger.besu.testutil.DeterministicEthScheduler;
5455

5556
import java.math.BigInteger;
5657
import java.nio.file.Path;
@@ -228,6 +229,7 @@ public Map<String, JsonRpcMethod> methods() {
228229
Vertx.vertx(new VertxOptions().setWorkerPoolSize(1)),
229230
ImmutableApiConfiguration.builder().build(),
230231
Optional.empty(),
231-
transactionSimulator);
232+
transactionSimulator,
233+
new DeterministicEthScheduler());
232234
}
233235
}

‎ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractBlockParameterMethod.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright ConsenSys AG.
2+
* Copyright contributors to Besu.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
55
* the License. You may obtain a copy of the License at

‎ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractDebugTraceBlock.java‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
/*
2+
* Copyright contributors to Besu.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
5+
* the License. You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
10+
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
* specific language governing permissions and limitations under the License.
12+
*
13+
* SPDX-License-Identifier: Apache-2.0
14+
*/
15+
package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods;
16+
17+
import static org.hyperledger.besu.services.pipeline.PipelineBuilder.createPipelineFrom;
18+
19+
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
20+
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters;
21+
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter;
22+
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams;
23+
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer;
24+
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace;
25+
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType;
26+
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugTraceTransactionResult;
27+
import org.hyperledger.besu.ethereum.api.query.BlockchainQueries;
28+
import org.hyperledger.besu.ethereum.core.Block;
29+
import org.hyperledger.besu.ethereum.debug.TraceOptions;
30+
import org.hyperledger.besu.ethereum.eth.manager.EthScheduler;
31+
import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor;
32+
import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
33+
import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
34+
import org.hyperledger.besu.ethereum.vm.DebugOperationTracer;
35+
import org.hyperledger.besu.metrics.BesuMetricCategory;
36+
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
37+
import org.hyperledger.besu.plugin.services.metrics.Counter;
38+
import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
39+
import org.hyperledger.besu.services.pipeline.Pipeline;
40+
41+
import java.util.ArrayList;
42+
import java.util.Collection;
43+
import java.util.Collections;
44+
import java.util.List;
45+
import java.util.Optional;
46+
import java.util.concurrent.ExecutionException;
47+
import java.util.function.Supplier;
48+
49+
import com.google.common.base.Suppliers;
50+
51+
public abstract class AbstractDebugTraceBlock implements JsonRpcMethod {
52+
53+
private final ProtocolSchedule protocolSchedule;
54+
private final LabelledMetric<Counter> outputCounter;
55+
private final Supplier<BlockchainQueries> blockchainQueriesSupplier;
56+
private final EthScheduler ethScheduler;
57+
58+
public AbstractDebugTraceBlock(
59+
final ProtocolSchedule protocolSchedule,
60+
final BlockchainQueries blockchainQueries,
61+
final ObservableMetricsSystem metricsSystem,
62+
final EthScheduler ethScheduler) {
63+
this.blockchainQueriesSupplier = Suppliers.ofInstance(blockchainQueries);
64+
this.protocolSchedule = protocolSchedule;
65+
this.outputCounter =
66+
metricsSystem.createLabelledCounter(
67+
BesuMetricCategory.BLOCKCHAIN,
68+
"transactions_debugTraceblock_pipeline_processed_total",
69+
"Number of transactions processed for each block",
70+
"step",
71+
"action");
72+
this.ethScheduler = ethScheduler;
73+
}
74+
75+
protected BlockchainQueries getBlockchainQueries() {
76+
return blockchainQueriesSupplier.get();
77+
}
78+
79+
protected TraceOptions getTraceOptions(final JsonRpcRequestContext requestContext) {
80+
final TraceOptions traceOptions;
81+
try {
82+
traceOptions =
83+
requestContext
84+
.getOptionalParameter(1, TransactionTraceParams.class)
85+
.map(TransactionTraceParams::traceOptions)
86+
.orElse(TraceOptions.DEFAULT);
87+
} catch (JsonRpcParameter.JsonRpcParameterException e) {
88+
throw new InvalidJsonRpcParameters(
89+
"Invalid transaction trace parameter (index 1)",
90+
RpcErrorType.INVALID_TRANSACTION_TRACE_PARAMS,
91+
e);
92+
}
93+
return traceOptions;
94+
}
95+
96+
protected Collection<DebugTraceTransactionResult> getTraces(
97+
final JsonRpcRequestContext requestContext,
98+
final TraceOptions traceOptions,
99+
final Optional<Block> maybeBlock) {
100+
return maybeBlock
101+
.flatMap(
102+
block ->
103+
Tracer.processTracing(
104+
getBlockchainQueries(),
105+
block.getHash(),
106+
traceableState -> {
107+
List<DebugTraceTransactionResult> tracesList =
108+
Collections.synchronizedList(new ArrayList<>());
109+
final ProtocolSpec protocolSpec =
110+
protocolSchedule.getByBlockHeader(block.getHeader());
111+
final MainnetTransactionProcessor transactionProcessor =
112+
protocolSpec.getTransactionProcessor();
113+
final TraceBlock.ChainUpdater chainUpdater =
114+
new TraceBlock.ChainUpdater(traceableState);
115+
116+
TransactionSource transactionSource = new TransactionSource(block);
117+
DebugOperationTracer debugOperationTracer =
118+
new DebugOperationTracer(traceOptions, true);
119+
ExecuteTransactionStep executeTransactionStep =
120+
new ExecuteTransactionStep(
121+
chainUpdater,
122+
transactionProcessor,
123+
getBlockchainQueries().getBlockchain(),
124+
debugOperationTracer,
125+
protocolSpec,
126+
block);
127+
DebugTraceTransactionStep debugTraceTransactionStep =
128+
new DebugTraceTransactionStep();
129+
Pipeline<TransactionTrace> traceBlockPipeline =
130+
createPipelineFrom(
131+
"getTransactions",
132+
transactionSource,
133+
4,
134+
outputCounter,
135+
false,
136+
"debug_trace_block")
137+
.thenProcess("executeTransaction", executeTransactionStep)
138+
.thenProcessAsyncOrdered(
139+
"debugTraceTransactionStep", debugTraceTransactionStep, 4)
140+
.andFinishWith("collect_results", tracesList::add);
141+
142+
try {
143+
ethScheduler.startPipeline(traceBlockPipeline).get();
144+
} catch (InterruptedException | ExecutionException e) {
145+
throw new RuntimeException(e);
146+
}
147+
return Optional.of(tracesList);
148+
}))
149+
.orElse(null);
150+
}
151+
}

‎ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java‎

Lines changed: 19 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright ConsenSys AG.
2+
* Copyright contributors to Besu.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
55
* the License. You may obtain a copy of the License at
@@ -18,10 +18,6 @@
1818
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
1919
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters;
2020
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException;
21-
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams;
22-
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace;
23-
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer;
24-
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer;
2521
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse;
2622
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
2723
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse;
@@ -31,32 +27,32 @@
3127
import org.hyperledger.besu.ethereum.core.Block;
3228
import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions;
3329
import org.hyperledger.besu.ethereum.debug.TraceOptions;
30+
import org.hyperledger.besu.ethereum.eth.manager.EthScheduler;
31+
import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
32+
import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
3433
import org.hyperledger.besu.ethereum.rlp.RLP;
3534
import org.hyperledger.besu.ethereum.rlp.RLPException;
36-
import org.hyperledger.besu.ethereum.vm.DebugOperationTracer;
35+
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
3736

3837
import java.util.Collection;
3938
import java.util.Optional;
40-
import java.util.function.Supplier;
4139

4240
import org.apache.tuweni.bytes.Bytes;
4341
import org.slf4j.Logger;
4442
import org.slf4j.LoggerFactory;
4543

46-
public class DebugTraceBlock implements JsonRpcMethod {
44+
public class DebugTraceBlock extends AbstractDebugTraceBlock {
4745

4846
private static final Logger LOG = LoggerFactory.getLogger(DebugTraceBlock.class);
49-
private final Supplier<BlockTracer> blockTracerSupplier;
5047
private final BlockHeaderFunctions blockHeaderFunctions;
51-
private final BlockchainQueries blockchainQueries;
5248

5349
public DebugTraceBlock(
54-
final Supplier<BlockTracer> blockTracerSupplier,
55-
final BlockHeaderFunctions blockHeaderFunctions,
56-
final BlockchainQueries blockchainQueries) {
57-
this.blockTracerSupplier = blockTracerSupplier;
58-
this.blockHeaderFunctions = blockHeaderFunctions;
59-
this.blockchainQueries = blockchainQueries;
50+
final ProtocolSchedule protocolSchedule,
51+
final BlockchainQueries blockchainQueries,
52+
final ObservableMetricsSystem metricsSystem,
53+
final EthScheduler ethScheduler) {
54+
super(protocolSchedule, blockchainQueries, metricsSystem, ethScheduler);
55+
this.blockHeaderFunctions = ScheduleBasedBlockHeaderFunctions.create(protocolSchedule);
6056
}
6157

6258
@Override
@@ -70,43 +66,22 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) {
7066
try {
7167
final String input = requestContext.getRequiredParameter(0, String.class);
7268
block = Block.readFrom(RLP.input(Bytes.fromHexString(input)), this.blockHeaderFunctions);
73-
} catch (final RLPException e) {
69+
} catch (final RLPException | IllegalArgumentException e) {
7470
LOG.debug("Failed to parse block RLP (index 0)", e);
7571
return new JsonRpcErrorResponse(
7672
requestContext.getRequest().getId(), RpcErrorType.INVALID_BLOCK_PARAMS);
7773
} catch (JsonRpcParameterException e) {
7874
throw new InvalidJsonRpcParameters(
7975
"Invalid block params (index 0)", RpcErrorType.INVALID_BLOCK_PARAMS, e);
8076
}
81-
final TraceOptions traceOptions;
82-
try {
83-
traceOptions =
84-
requestContext
85-
.getOptionalParameter(1, TransactionTraceParams.class)
86-
.map(TransactionTraceParams::traceOptions)
87-
.orElse(TraceOptions.DEFAULT);
88-
} catch (JsonRpcParameterException e) {
89-
throw new InvalidJsonRpcParameters(
90-
"Invalid transaction trace parameter (index 1)",
91-
RpcErrorType.INVALID_TRANSACTION_TRACE_PARAMS,
92-
e);
93-
}
77+
final TraceOptions traceOptions = getTraceOptions(requestContext);
9478

95-
if (this.blockchainQueries.blockByHash(block.getHeader().getParentHash()).isPresent()) {
79+
if (getBlockchainQueries()
80+
.getBlockchain()
81+
.getBlockByHash(block.getHeader().getParentHash())
82+
.isPresent()) {
9683
final Collection<DebugTraceTransactionResult> results =
97-
Tracer.processTracing(
98-
blockchainQueries,
99-
Optional.of(block.getHeader()),
100-
mutableWorldState ->
101-
blockTracerSupplier
102-
.get()
103-
.trace(
104-
mutableWorldState,
105-
block,
106-
new DebugOperationTracer(traceOptions, true))
107-
.map(BlockTrace::getTransactionTraces)
108-
.map(DebugTraceTransactionResult::of))
109-
.orElse(null);
84+
getTraces(requestContext, traceOptions, Optional.ofNullable(block));
11085
return new JsonRpcSuccessResponse(requestContext.getRequest().getId(), results);
11186
} else {
11287
return new JsonRpcErrorResponse(

0 commit comments

Comments
 (0)