Merge pull request #663 from ethereum/dev-windows-ci-fix

msooseth · web-flow · commit 6bbcf1f0d21a · 2025-02-17T10:45:02.000-08:00
tests: run `evm` on its own directory
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -228,8 +228,6 @@ jobs:
           echo HEVM_SOLIDITY_REPO="$PWD/ethereum-solidity" >> "$GITHUB_ENV"
           echo HEVM_ETHEREUM_TESTS_REPO="$PWD/ethereum-tests" >> "$GITHUB_ENV"
           echo HEVM_FORGE_STD_REPO="$PWD/forge-std" >> "$GITHUB_ENV"
-          # environment
-          echo HEVM_SYSTEM_SHELL="$(which bash)" >> "$GITHUB_ENV"
 
       - name: run tests
         run: |
diff --git a/test/EVM/Test/BlockchainTests.hs b/test/EVM/Test/BlockchainTests.hs
@@ -33,6 +33,8 @@ import Data.Word (Word64)
 import System.Environment (lookupEnv, getEnv)
 import System.FilePath.Find qualified as Find
 import System.FilePath.Posix (makeRelative, (</>))
+import System.IO (hPutStr, hClose)
+import System.IO.Temp (withSystemTempFile)
 import System.Process (readProcessWithExitCode)
 import GHC.IO.Exception (ExitCode(ExitSuccess))
 import Witch (into, unsafeInto)
@@ -208,8 +210,10 @@ traceVsGeth fname name x = do
     putStrLn $ "evmtool stderr output:" <> show evmtoolStderr
     putStrLn $ "evmtool stdout output:" <> show evmtoolStdout
   hevm <- traceVMTest x
-  liftIO $ writeFile "trace.jsonl" $ filterInfoLines evmtoolStderr
-  decodedContents <- liftIO $ decodeTraceOutputHelper "trace.jsonl"
+  decodedContents <- liftIO $ withSystemTempFile "trace.jsonl" $ \traceFile hdl -> do
+    hPutStr hdl $ filterInfoLines evmtoolStderr
+    hClose hdl
+    decodeTraceOutputHelper traceFile
   let EVMToolTraceOutput ts _ = fromJust decodedContents
   liftIO $ putStrLn "Comparing traces."
   _ <- liftIO $ compareTraces hevm ts
diff --git a/test/EVM/Test/Tracing.hs b/test/EVM/Test/Tracing.hs
@@ -23,18 +23,18 @@ import Data.Aeson qualified as JSON
 import Data.ByteString (ByteString)
 import Data.ByteString qualified as BS
 import Data.ByteString.Char8 qualified as Char8
-import Data.Maybe (fromJust, fromMaybe, isJust, isNothing)
+import Data.Maybe (fromJust, isJust, isNothing)
 import Data.Map.Strict qualified as Map
 import Data.Text.IO qualified as T
 import Data.Vector qualified as Vector
 import Data.Word (Word8, Word64)
 import GHC.Generics (Generic)
 import GHC.IO.Exception (ExitCode(ExitSuccess))
 import Numeric (showHex)
-import Paths_hevm qualified as Paths
-import System.Directory (removeFile)
-import System.Environment (lookupEnv)
-import System.Process (readProcessWithExitCode)
+import System.Directory (removeDirectoryRecursive)
+import System.FilePath ((</>))
+import System.IO.Temp (getCanonicalTemporaryDirectory, createTempDirectory)
+import System.Process (readCreateProcessWithExitCode, proc, CreateProcess(..))
 import Test.QuickCheck (elements)
 import Test.QuickCheck.Instances.Text()
 import Test.QuickCheck.Instances.Natural()
@@ -308,8 +308,8 @@ getHEVMRet contr txData gaslimitExec = do
   let (txn, evmEnv, contrAlloc, fromAddress, toAddress, _) = evmSetup contr txData gaslimitExec
   runCodeWithTrace Nothing evmEnv contrAlloc txn (LitAddr fromAddress) (LitAddr toAddress)
 
-getEVMToolRet :: OpContract -> ByteString -> Int -> IO (Maybe EVMToolResult)
-getEVMToolRet contr txData gaslimitExec = do
+getEVMToolRet :: FilePath -> OpContract -> ByteString -> Int -> IO (Maybe EVMToolResult)
+getEVMToolRet evmDir contr txData gaslimitExec = do
   let (txn, evmEnv, contrAlloc, fromAddress, toAddress, sk) = evmSetup contr txData gaslimitExec
       txs = [EVM.Transaction.sign sk txn]
       walletAlloc = EVMToolAlloc{ balance = 0x5ffd4878be161d74
@@ -318,24 +318,25 @@ getEVMToolRet contr txData gaslimitExec = do
                                 }
       alloc :: Map.Map Addr EVMToolAlloc
       alloc = Map.fromList ([ (fromAddress, walletAlloc), (toAddress, contrAlloc)])
-  JSON.encodeFile "txs.json" txs
-  JSON.encodeFile "alloc.json" alloc
-  JSON.encodeFile "env.json" evmEnv
-  (exitCode, evmtoolStdout, evmtoolStderr) <- readProcessWithExitCode "evm" [ "transition"
-                               , "--state.fork", "Cancun"
-                               , "--input.alloc" , "alloc.json"
-                               , "--input.env" , "env.json"
-                               , "--input.txs" , "txs.json"
-                               , "--output.alloc" , "alloc-out.json"
-                               , "--trace.returndata=true"
-                               , "--trace" , "trace.json"
-                               , "--output.result", "result.json"
-                               ] ""
+  JSON.encodeFile (evmDir </> "txs.json") txs
+  JSON.encodeFile (evmDir </> "alloc.json") alloc
+  JSON.encodeFile (evmDir </> "env.json") evmEnv
+  let cmd = (proc "evm" [ "transition"
+                        , "--state.fork", "Cancun"
+                        , "--input.alloc", "alloc.json"
+                        , "--input.env", "env.json"
+                        , "--input.txs", "txs.json"
+                        , "--output.alloc", "alloc-out.json"
+                        , "--trace.returndata=true"
+                        , "--trace", "trace.json"
+                        , "--output.result", "result.json"
+                        ]) { cwd = Just evmDir }
+  (exitCode, evmtoolStdout, evmtoolStderr) <- readCreateProcessWithExitCode cmd ""
   when (exitCode /= ExitSuccess) $ do
     putStrLn $ "evmtool exited with code " <> show exitCode
     putStrLn $ "evmtool stderr output:" <> show evmtoolStderr
     putStrLn $ "evmtool stdout output:" <> show evmtoolStdout
-  JSON.decodeFileStrict "result.json" :: IO (Maybe EVMToolResult)
+  JSON.decodeFileStrict (evmDir </> "result.json") :: IO (Maybe EVMToolResult)
 
 -- Compares traces of evmtool (from go-ethereum) and HEVM
 compareTraces :: [VMTrace] -> [EVMToolTrace] -> IO (Bool)
@@ -389,45 +390,34 @@ compareTraces hevmTrace evmTrace = go hevmTrace evmTrace
       putStrLn $ "Traces don't match. evmtool's trace is longer by:" <> (show b)
       pure False
 
-getTraceFileName :: EVMToolResult -> String
-getTraceFileName evmtoolResult = traceFileName
+getTraceFileName :: FilePath -> EVMToolResult -> String
+getTraceFileName evmDir evmtoolResult = evmDir </> traceFileName
   where
     txName = ((evmtoolResult.receipts) !! 0).transactionHash
     traceFileName = "trace-0-" ++ txName ++ ".jsonl"
 
-getTraceOutput :: Maybe EVMToolResult -> IO (Maybe EVMToolTraceOutput)
-getTraceOutput evmtoolResult =
+getTraceOutput :: FilePath -> Maybe EVMToolResult -> IO (Maybe EVMToolTraceOutput)
+getTraceOutput evmDir evmtoolResult =
   case evmtoolResult of
     Nothing -> pure Nothing
     Just res -> do
-      let traceFileName = getTraceFileName res
+      let traceFileName = getTraceFileName evmDir res
       decodeTraceOutputHelper traceFileName
 
 decodeTraceOutputHelper :: String -> IO (Maybe EVMToolTraceOutput)
 decodeTraceOutputHelper traceFileName = do
-      convertPath <- Paths.getDataFileName "test/scripts/convert_trace_to_json.sh"
-      maybeShellPath <- (fromMaybe "bash") <$> (lookupEnv "HEVM_SYSTEM_SHELL")
-      let shellPath = if maybeShellPath == "" then "bash" else maybeShellPath
-      (exitcode, stdout, stderr) <- readProcessWithExitCode shellPath [convertPath, traceFileName] ""
-      case exitcode of
-        ExitSuccess -> do
-          -- putStrLn $ "Successfully converted trace to JSON: " <> (show stdout) <> " " <>
-          --   (show stderr) <> " " <> (show exitcode) <> " " <> (show traceFileName)
-          JSON.decodeFileStrict (traceFileName ++ ".json") :: IO (Maybe EVMToolTraceOutput)
-        _ -> do
-          putStrLn $ "Error converting trace! exit code:" <> (show exitcode)
-          putStrLn $ "stdout: " <> (show stdout)
-          putStrLn $ "stderr: " <> (show stderr)
-          pure Nothing
-
-deleteTraceOutputFiles :: Maybe EVMToolResult -> IO ()
-deleteTraceOutputFiles evmtoolResult =
-  case evmtoolResult of
-    Nothing -> pure ()
-    Just res -> do
-      let traceFileName = getTraceFileName res
-      System.Directory.removeFile traceFileName
-      System.Directory.removeFile (traceFileName ++ ".json")
+  traceContents <- readFile traceFileName
+  let traceLines = lines traceContents
+  let (traces, outputLines) = splitAt (length traceLines - 1) traceLines
+  let parsedTraces = map decodeString traces :: [Maybe EVMToolTrace]
+  let parsedOutput = decodeString (outputLines !! 0) :: Maybe EVMToolOutput
+  if all isJust parsedTraces && isJust parsedOutput then
+    pure $ Just $ EVMToolTraceOutput (map fromJust parsedTraces) (fromJust parsedOutput)
+  else
+    pure Nothing
+  where
+    decodeString :: (JSON.FromJSON a) => String -> Maybe a
+    decodeString = JSON.decodeStrict . Char8.pack
 
 -- | Takes a runtime code and calls it with the provided calldata
 --   Uses evmtool's alloc and transaction to set up the VM correctly
@@ -875,9 +865,11 @@ tests = testGroup "contract-quickcheck-run"
 
 checkTraceAndOutputs :: App m => OpContract -> Int -> ByteString -> m ()
 checkTraceAndOutputs contract gasLimit txData = do
-  evmtoolResult <- liftIO $ getEVMToolRet contract txData gasLimit
+  tmpDir <- liftIO getCanonicalTemporaryDirectory
+  evmDir <- liftIO $ createTempDirectory tmpDir "evm-trace"
+  evmtoolResult <- liftIO $ getEVMToolRet evmDir contract txData gasLimit
   hevmRun <- getHEVMRet contract txData gasLimit
-  evmtoolTraceOutput <- fmap fromJust $ liftIO $ getTraceOutput evmtoolResult
+  evmtoolTraceOutput <- fmap fromJust $ liftIO $ getTraceOutput evmDir evmtoolResult
   case hevmRun of
     (Right (expr, hevmTrace, hevmTraceResult)) -> liftIO $ do
       let
@@ -915,7 +907,7 @@ checkTraceAndOutputs contract gasLimit txData = do
              putStrLn $ "ret value computed via symb+conc : " <> (bsToHex (fromJust simplConcrExprRetval))
              assertEqual "Simplified, concretized expression must match evmtool's output." True False
       else do
-        putStrLn $ "Name of trace file: " <> (getTraceFileName $ fromJust evmtoolResult)
+        putStrLn $ "Name of trace file: " <> (getTraceFileName evmDir $ fromJust evmtoolResult)
         putStrLn $ "HEVM result  :" <> (show hevmTraceResult)
         T.putStrLn $ "HEVM result: " <> (formatBinary $ bssToBs hevmTraceResult.out)
         T.putStrLn $ "evm result : " <> (formatBinary $ bssToBs evmtoolTraceOutput.output.output)
@@ -928,13 +920,7 @@ checkTraceAndOutputs contract gasLimit txData = do
       -- putStrLn $ "output by evmtool is: '" <> bsToHex evmtoolTraceOutput.toOutput.output <> "'"
       traceOK <- compareTraces hevmTrace (evmtoolTraceOutput.trace)
       assertEqual "Traces and gas must match" traceOK True
-  liftIO $ do
-    System.Directory.removeFile "txs.json"
-    System.Directory.removeFile "alloc-out.json"
-    System.Directory.removeFile "alloc.json"
-    System.Directory.removeFile "result.json"
-    System.Directory.removeFile "env.json"
-    deleteTraceOutputFiles evmtoolResult
+  liftIO $ removeDirectoryRecursive evmDir
 
 -- GasLimitInt
 newtype GasLimitInt = GasLimitInt (Int)
diff --git a/test/scripts/convert_trace_to_json.sh b/test/scripts/convert_trace_to_json.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -euxo pipefail
 
-num=$(wc -l "$1" | cut -d " " -f 1)
+num=$(awk 'END{print NR}' "$1")
 awk -v ll="$num" 'BEGIN {print "{\"trace\": [" } {if (NR!=ll) {print $0; if (NR!=ll-1) {print ","}} else {print "], \"output\":" $0}} END{print "}"}' "$1" > "$1.json"
