feat: implement call traces display for solidity tests

Author: galargh

.changeset/fluffy-days-study.md

@@ -0,0 +1,5 @@
+---
+"hardhat": patch
+---
+
+Added call traces support for solidity tests (they can be enabled via the `-vvvv` verbosity level flag)

v-next/example-project/contracts/Counter.t.sol

@@ -8,25 +8,31 @@ contract CounterTest {
   Counter counter;
 
   function setUp() public {
+    console.log("Setting up");
     counter = new Counter();
+    console.log("Counter set up");
   }
 
   function testInitialValue() public view {
+    console.log("Testing initial value");
     require(counter.x() == 0, "Initial value should be 0");
   }
 
   function testFailInitialValue() public view {
+    console.log("Testing initial value fail");
     require(counter.x() == 1, "Initial value should be 1");
   }
 
   function testFuzzInc(uint8 x) public {
+    console.log("Fuzz testing inc");
     for (uint8 i = 0; i < x; i++) {
       counter.inc();
     }
     require(counter.x() == x, "Value after calling inc x times should be x");
   }
 
   function testFailFuzzInc(uint8 x) public {
+    console.log("Fuzz testing inc fail");
     for (uint8 i = 0; i < x; i++) {
       counter.inc();
     }

v-next/hardhat/src/internal/builtin-plugins/solidity-test/formatters.ts

@@ -1,5 +1,12 @@
-import type { ArtifactId } from "@ignored/edr-optimism";
+import type {
+  LogTrace,
+  ArtifactId,
+  CallTrace,
+  DecodedTraceParameters,
+} from "@ignored/edr-optimism";
 
+import { LogKind, CallKind } from "@ignored/edr-optimism";
+import { bytesToHexString } from "@nomicfoundation/hardhat-utils/hex";
 import chalk from "chalk";
 
 export function formatArtifactId(
@@ -11,3 +18,84 @@ export function formatArtifactId(
 
   return `${chalk.bold(`${sourceName}:${artifactId.name}`)} (v${artifactId.solcVersion})`;
 }
+
+export function formatLogs(logs: string[], indent: number): string {
+  return chalk.grey(
+    logs.map((log) => `${" ".repeat(indent)}${log}`).join("\n"),
+  );
+}
+
+export function formatInputs(inputs: DecodedTraceParameters | Uint8Array): string {
+  if (inputs instanceof Uint8Array) {
+    return bytesToHexString(inputs);
+  } else {
+    return `${inputs.name}(${inputs.arguments.join(", ")})`;
+  }
+}
+
+function formatOutputs(outputs: string | Uint8Array): string {
+  if (outputs instanceof Uint8Array) {
+    return bytesToHexString(outputs);
+  } else {
+    return outputs;
+  }
+}
+
+function formatLog(log: LogTrace, indent: number): string {
+  const { parameters } = log;
+  if (Array.isArray(parameters)) {
+    const topics = parameters
+      .slice(0, parameters.length - 1)
+      .map((topic) => bytesToHexString(topic));
+    const data = bytesToHexString(parameters[parameters.length - 1]);
+    return `${" ".repeat(indent)}${chalk.grey(`(topics: [${topics.join(", ")}], data: ${data})`)}`;
+  } else {
+    return `${" ".repeat(indent)}${parameters.name}(${parameters.arguments.join(", ")})`;
+  }
+}
+
+function formatKind(kind: CallKind): string {
+  switch (kind) {
+    case CallKind.Call:
+      return "Call";
+    case CallKind.CallCode:
+      return "CallCode";
+    case CallKind.DelegateCall:
+      return "DelegateCall";
+    case CallKind.StaticCall:
+      return "StaticCall";
+    case CallKind.Create:
+      return "Create";
+  }
+}
+
+function formatTrace(trace: CallTrace, indent: number): string {
+  const {
+    success,
+    contract,
+    inputs,
+    gasUsed,
+    value,
+    kind,
+    isCheatcode,
+    outputs,
+  } = trace;
+  const color = success ? chalk.blue : chalk.yellow;
+  const sign = success ? "✔" : "✘";
+  const label = success ? "Succeeded" : "Failed";
+  const lines = [
+    `${" ".repeat(indent)}${color(`${sign} ${formatKind(kind)} ${label}`)}: ${contract}::${formatInputs(inputs)} → ${formatOutputs(outputs)} ${chalk.grey(`(gas: ${gasUsed}, tokens: ${value}, cheatcode: ${isCheatcode})`)}`,
+  ];
+  for (const child of trace.children) {
+    if (child.kind === LogKind.Log) {
+      lines.push(formatLog(child, indent + 2));
+    } else {
+      lines.push(formatTrace(child, indent + 2));
+    }
+  }
+  return lines.join("\n");
+}
+
+export function formatTraces(traces: CallTrace[], indent: number): string {
+  return traces.map((trace) => formatTrace(trace, indent)).join("\n");
+}

v-next/hardhat/src/internal/builtin-plugins/solidity-test/helpers.ts

@@ -17,6 +17,7 @@ import {
   opLatestHardfork,
   l1GenesisState,
   l1HardforkLatest,
+  IncludeTraces,
 } from "@ignored/edr-optimism";
 import { hexStringToBytes } from "@nomicfoundation/hardhat-utils/hex";
 
@@ -36,6 +37,7 @@ export function solidityTestConfigToSolidityTestRunnerConfigArgs(
   chainType: ChainType,
   projectRoot: string,
   config: SolidityTestConfig,
+  verbosity: number,
   testPattern?: string,
 ): SolidityTestRunnerConfigArgs {
   const fsPermissions: PathPermission[] | undefined = [
@@ -105,6 +107,13 @@ export function solidityTestConfigToSolidityTestRunnerConfigArgs(
       ? opGenesisState(opLatestHardfork())
       : l1GenesisState(l1HardforkLatest());
 
+  let includeTraces: IncludeTraces = IncludeTraces.None;
+  if (verbosity >= 5) {
+    includeTraces = IncludeTraces.All;
+  } else if (verbosity >= 3) {
+    includeTraces = IncludeTraces.Failing;
+  }
+
   return {
     projectRoot,
     ...config,
@@ -116,6 +125,7 @@ export function solidityTestConfigToSolidityTestRunnerConfigArgs(
     blockCoinbase,
     rpcStorageCaching,
     testPattern,
+    includeTraces,
   };
 }
 

v-next/hardhat/src/internal/builtin-plugins/solidity-test/index.ts

@@ -38,7 +38,7 @@ const hardhatPlugin: HardhatPlugin = {
         name: "verbosity",
         shortName: "v",
         description: "Log verbosity",
-        defaultValue: 2
+        defaultValue: 2,
       })
       .setAction(import.meta.resolve("./task-action.js"))
       .build(),

v-next/hardhat/src/internal/builtin-plugins/solidity-test/reporter.ts

@@ -10,7 +10,7 @@ import chalk from "chalk";
 
 import { encodeStackTraceEntry } from "../network-manager/edr/stack-traces/stack-trace-solidity-errors.js";
 
-import { formatArtifactId } from "./formatters.js";
+import { formatArtifactId, formatInputs, formatLogs, formatTraces } from "./formatters.js";
 import { getMessageFromLastStackTraceEntry } from "./stack-trace-solidity-errors.js";
 
 /**
@@ -21,6 +21,7 @@ import { getMessageFromLastStackTraceEntry } from "./stack-trace-solidity-errors
 export async function* testReporter(
   source: TestEventSource,
   sourceNameToUserSourceName: Map<string, string>,
+  verbosity: number,
 ): TestReporterResult {
   let runTestCount = 0;
   let runSuccessCount = 0;
@@ -68,16 +69,36 @@ export async function* testReporter(
             )
             .join(", ");
 
+          let printDecodedLogs = false;
+          let printSetUpTraces = false;
+          let printExecutionTraces = false;
+
           switch (status) {
             case "Success": {
               yield `${chalk.green("✔ Passed")}: ${name} ${chalk.grey(`(${details})`)}\n`;
               suiteSuccessCount++;
+              if (verbosity >= 2) {
+                printDecodedLogs = true;
+              }
+              if (verbosity >= 5) {
+                printSetUpTraces = true;
+                printExecutionTraces = true;
+              }
               break;
             }
             case "Failure": {
               failures.push(testResult);
               yield `${chalk.red(`✘ Failed(${failures.length})`)}: ${name} ${chalk.grey(`(${details})`)}\n`;
               suiteFailureCount++;
+              if (verbosity >= 1) {
+                printDecodedLogs = true;
+              }
+              if (verbosity >= 3) {
+                printExecutionTraces = true;
+              }
+              if (verbosity >= 4) {
+                printSetUpTraces = true;
+              }
               break;
             }
             case "Skipped": {
@@ -86,6 +107,41 @@ export async function* testReporter(
               break;
             }
           }
+
+          let printExtraSpace = false;
+
+          if (printDecodedLogs) {
+            const decodedLogs = testResult.decodedLogs ?? [];
+            if (decodedLogs.length > 0) {
+              yield `Decoded Logs:\n${formatLogs(decodedLogs, 2)}\n`;
+              printExtraSpace = true;
+            }
+          }
+
+          if (printSetUpTraces || printExecutionTraces) {
+            const callTraces = testResult.callTraces().filter(({inputs}) => {
+              if (printSetUpTraces && printExecutionTraces) {
+                return true;
+              }
+              const formattedInputs = formatInputs(inputs);
+              if (printSetUpTraces && formattedInputs === "setUp()") {
+                return true;
+              }
+              if (printExecutionTraces && formattedInputs !== "setUp()") {
+                return true;
+              }
+              return false;
+            });
+
+            if (callTraces.length > 0) {
+              yield `Call Traces:\n${formatTraces(callTraces, 2)}\n`;
+              printExtraSpace = true;
+            }
+          }
+
+          if (printExtraSpace) {
+            yield "\n";
+          }
         }
 
         const suiteSummary = `${suiteTestCount} tests, ${suiteSuccessCount} passed, ${suiteFailureCount} failed, ${suiteSkippedCount} skipped`;
@@ -166,14 +222,6 @@ export async function* testReporter(
           break;
       }
 
-      if (
-        failure.decodedLogs !== undefined &&
-        failure.decodedLogs !== null &&
-        failure.decodedLogs.length > 0
-      ) {
-        yield `Decoded Logs:\n${chalk.grey(failure.decodedLogs.map((log) => `  ${log}`).join("\n"))}\n`;
-      }
-
       if (
         failure.counterexample !== undefined &&
         failure.counterexample !== null

v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts

@@ -115,6 +115,7 @@ const runSolidityTests: NewTaskActionFunction<TestActionArguments> = async (
       chainType,
       hre.config.paths.root,
       solidityTestConfig,
+      hre.globalOptions.verbosity,
       grep,
     );
   const tracingConfig: TracingConfigWithBuffers = {
@@ -149,7 +150,13 @@ const runSolidityTests: NewTaskActionFunction<TestActionArguments> = async (
         }
       }
     })
-    .compose((source) => testReporter(source, sourceNameToUserSourceName));
+    .compose((source) =>
+      testReporter(
+        source,
+        sourceNameToUserSourceName,
+        hre.globalOptions.verbosity,
+      ),
+    );
 
   const outputStream = testReporterStream.pipe(
     createNonClosingWriter(process.stdout),

v-next/hardhat/src/internal/builtin-plugins/test/index.ts

@@ -33,7 +33,7 @@ const hardhatPlugin: HardhatPlugin = {
         name: "verbosity",
         shortName: "v",
         description: "Log verbosity",
-        defaultValue: 2
+        defaultValue: 2,
       })
       .setAction(import.meta.resolve("./task-action.js"))
       .build(),

v-next/hardhat/src/internal/cli/main.ts

@@ -12,7 +12,6 @@ import {
 import { isCi } from "@nomicfoundation/hardhat-utils/ci";
 import { readClosestPackageJson } from "@nomicfoundation/hardhat-utils/package";
 import { kebabToCamelCase } from "@nomicfoundation/hardhat-utils/string";
-import chalk from "chalk";
 import debug from "debug";
 import { register } from "tsx/esm/api";
 

v-next/hardhat/test/internal/cli/main.ts

@@ -11,7 +11,7 @@ import type {
 } from "../../../src/types/tasks.js";
 
 import assert from "node:assert/strict";
-import { afterEach, before, describe, it, mock } from "node:test";
+import { afterEach, before, describe, it } from "node:test";
 import { pathToFileURL } from "node:url";
 
 import { HardhatError } from "@nomicfoundation/hardhat-errors";
@@ -22,7 +22,6 @@ import {
 } from "@nomicfoundation/hardhat-test-utils";
 import { isCi } from "@nomicfoundation/hardhat-utils/ci";
 import chalk from "chalk";
-import debug from "debug";
 
 import {
   main,