fix(avm): capping rd size and fixing pipes (#19420)

Returndata size is uncapped in TS but we have an implicit cap in CPP.

This makes the cap explicit in the cpp fuzzer and truncates the
returndata from TS so that the comparisons do not trivially fail.

There's also some clean up of how reading and writing between the fuzzer
and TS process works

Author: IlyasRidhuan

barretenberg/cpp/src/barretenberg/avm_fuzzer/common/process.cpp

@@ -49,21 +49,38 @@ Process::~Process()
 void Process::write_line(const std::string& line) const
 {
     std::string command = line + "\n";
-    write(stdin_fd, command.c_str(), command.size());
-    fsync(stdin_fd);
+    const char* data = command.c_str();
+    size_t remaining = command.size();
+
+    // We use a loop to ensure all data is written but throw if we encounter an error.
+    // This enables partial writes to be handled correctly.
+    while (remaining > 0) {
+        ssize_t written = write(stdin_fd, data, remaining);
+        if (written < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+            throw std::runtime_error("write() error: " + std::string(std::strerror(errno)));
+        }
+        data += written;
+        remaining -= static_cast<size_t>(written);
+    }
 }
 
 std::string Process::read_line() const
 {
     char buffer[4096]; // NOLINT
     std::string response;
     ssize_t bytes_read = 0;
-    fsync(stdout_fd);
     while ((bytes_read = read(stdout_fd, buffer, sizeof(buffer))) > 0) {
-        response.append(buffer, static_cast<size_t>(bytes_read));
-        if (response.find('\n') != std::string::npos) {
+        // Check for newline in just the newly read data  instead of going back through the entire response
+        const char* newline_pos = static_cast<const char*>(memchr(buffer, '\n', static_cast<size_t>(bytes_read)));
+        if (newline_pos != nullptr) {
+            // Found newline - append only up to and including the newline
+            response.append(buffer, static_cast<size_t>(newline_pos - buffer + 1));
             break;
         }
+        response.append(buffer, static_cast<size_t>(bytes_read));
     }
     if (bytes_read < 0 && errno != EINTR) {
         throw std::runtime_error("read() error: " + std::string(std::strerror(errno)));

barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzz_lib/simulator.cpp

@@ -31,7 +31,9 @@ using namespace bb::avm2::simulation;
 using namespace bb::avm2::fuzzer;
 using namespace bb::world_state;
 
-// Helper function to serialize simulation request via
+const auto MAX_RETURN_DATA_SIZE_IN_FIELDS = 1024;
+
+// Helper function to serialize simulation request via msgpack
 std::string serialize_simulation_request(const Tx& tx,
                                          const GlobalVariables& globals,
                                          const FuzzerContractDB& contract_db)
@@ -79,6 +81,9 @@ SimulatorResult CppSimulator::simulate(fuzzer::FuzzerWorldStateManager& ws_mgr,
         .skip_fee_enforcement = false,
         .collect_call_metadata = true,
         .collect_public_inputs = true,
+        .collection_limits = {
+            .max_returndata_size_in_fields = MAX_RETURN_DATA_SIZE_IN_FIELDS,
+        },
     };
 
     ProtocolContracts protocol_contracts{};
@@ -164,8 +169,17 @@ SimulatorResult JsSimulator::simulate([[maybe_unused]] fuzzer::FuzzerWorldStateM
     return result;
 }
 
-bool compare_simulator_results(const SimulatorResult& result1, const SimulatorResult& result2)
+bool compare_simulator_results(SimulatorResult& result1, SimulatorResult& result2)
 {
+    // Since the simulator results are interchangeable between TS and C++, we limit the return data size for comparison
+    // todo(ilyas): we ideally specfify one param as the TS result and truncate only that one
+    if (result1.output.size() > MAX_RETURN_DATA_SIZE_IN_FIELDS) {
+        result1.output.resize(MAX_RETURN_DATA_SIZE_IN_FIELDS);
+    }
+    if (result2.output.size() > MAX_RETURN_DATA_SIZE_IN_FIELDS) {
+        result2.output.resize(MAX_RETURN_DATA_SIZE_IN_FIELDS);
+    }
+
     return result1.reverted == result2.reverted && result1.output == result2.output &&
            result1.end_tree_snapshots == result2.end_tree_snapshots;
 }

barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp

@@ -91,6 +91,6 @@ Tx create_default_tx(const AztecAddress& contract_address,
                      bool is_static_call,
                      const Gas& gas_limit);
 
-bool compare_simulator_results(const SimulatorResult& result1, const SimulatorResult& result2);
+bool compare_simulator_results(SimulatorResult& result1, SimulatorResult& result2);
 
 GlobalVariables create_default_globals();

yarn-project/simulator/src/public/fuzzing/avm_simulator_bin.ts

@@ -9,11 +9,23 @@ import {
 import { GlobalVariables, TreeSnapshots } from '@aztec/stdlib/tx';
 import { NativeWorldStateService } from '@aztec/world-state';
 
-import { writeSync } from 'fs';
 import { createInterface } from 'readline';
 
 import { AvmFuzzerSimulator, FuzzerSimulationRequest } from './avm_fuzzer_simulator.js';
 
+/** Write data to stdout, letting Node handle buffering. */
+function writeOutput(data: string): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    process.stdout.write(data, err => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
 // This cache holds opened world states to avoid reopening them for each invocation.
 // It's a map so that in the future we could support multiple world states (if we had multiple fuzzers).
 const worldStateCache = new Map<string, NativeWorldStateService>();
@@ -96,16 +108,17 @@ async function execute(base64Line: string): Promise<void> {
       revertReason: result.revertReason ?? '',
       endTreeSnapshots: result.publicInputs.endTreeSnapshots,
     });
-    writeSync(process.stdout.fd, resultBuffer.toString('base64') + '\n');
+    const base64Response = resultBuffer.toString('base64') + '\n';
+    await writeOutput(base64Response);
   } catch (error: any) {
     // If we error, treat as reverted
     const errorResult = serializeWithMessagePack({
       reverted: true,
-      output: [] as string[],
+      output: [] as Fr[],
       revertReason: `Unexpected Error ${error.message}`,
       endTreeSnapshots: TreeSnapshots.empty(),
     });
-    writeSync(process.stdout.fd, errorResult.toString('base64') + '\n');
+    await writeOutput(errorResult.toString('base64') + '\n');
   }
 }