fix: pass structured diff and indexed diff flow as paths, not contents

otherwise it wasn't working when the files were a bit bigger:
CLI args are limited storage

for this goal, for now we don't remove the temp folder where we unzip
the multitrace on `replay`: this is imo ok for MVP; but we need to decide
on a strategy

Author: alehander92

src/ct/trace/multitrace.nim

@@ -163,7 +163,7 @@ proc makeMultitraceArchive(traceFolders: seq[string], rawDiff: string, structure
 
   zipFolder(folder, outputPath)
 
-  removeDir(folder)
+  # TODO: decide here: only when flag archive? or always? removeDir(folder)
 
 proc makeMultitrace*(traceIdList: seq[int], diffSpecification: string, outputPath: string) =
   # make a folder , copy those traces , find this diff, eventually parse it and store it

src/ct/trace/replay.nim

@@ -30,23 +30,8 @@ proc replayMultitrace*(archivePath: string, indexDiff: bool = false): bool =
     echo fmt"ERROR: couldn't import the trace with name {traceDir.extractFilename} from the multitrace"
     quit(1)
 
-  # TODO: improve this: use paths or temp folder or read from zip
-  # currently we pass them through as CLI args, but they're probably with limited memory/size
-  var structuredDiffJson = ""
-  var indexDiffJson = ""
-
   var structuredDiffPath = outputFolder / "diff.json"
-  try:
-    structuredDiffJson = readFile(structuredDiffPath)
-  except CatchableError: # assume file read error for now
-    # if no diff.json recorded: that's ok
-    # we might have just a multitrace to replay multiple traces at once without a diff
-    # in the future
-    structuredDiffPath = ""
-    structuredDiffJson = ""
-
   var diffIndexPath = ""
-  echo "index diff ", indexDiff
 
   if indexDiff:
     let backend = if trace.lang.isDbBased:
@@ -61,40 +46,38 @@ proc replayMultitrace*(archivePath: string, indexDiff: bool = false): bool =
 
     var eventualDiffIndexPath = outputFolder / "diff_index.json"
     if not fileExists(eventualDiffIndexPath):
-      let process = startProcess(backend, args = @["index-diff", structuredDiffJson, traceDir, outputFolder], options={poParentStreams})
+      let process = startProcess(backend, args = @["index-diff", structuredDiffPath, traceDir, outputFolder], options={poParentStreams})
       let exitCode = waitForExit(process)
 
       if exitCode == 0:
         # replace with an archive with the indexed data
         removeFile(archivePath)
         zipFolder(outputFolder, archivePath)
         diffIndexPath = eventualDiffIndexPath
-        indexDiffJson = readFile(diffIndexPath)
       else:
         echo "WARN: a problem with indexing diff: no diff index"
-        diffIndexPath = "" # some kind of a problem
-        indexDiffJson = ""
+        diffIndexPath = "" # some kind of a problem        # indexDiffJson = ""
 
     # if ok: trace patched, diff indexed: archive still accessible
-    # remove only the temp extracted folder
+    # remove only the temp extracted folder: ok here for index-diff; not for `replay` in the next case for now
     removeDir(outputFolder)
     return false # this means it shouldn't restart: for now restart maybe supported in dev mode only for replays
   else:
     var eventualDiffIndexPath = outputFolder / "diff_index.json"
     if fileExists(eventualDiffIndexPath):
       diffIndexPath = eventualDiffIndexPath
-      indexDiffJson = readFile(eventualDiffIndexPath)
     else:
       diffIndexPath = ""
-      indexDiffJson = ""
-
+    
     # trace imported, diff and eventually index copied: archive still accessible
-    # remove only the temp extracted folder
-    removeDir(outputFolder)
+    # remove only the temp extracted folder?
+    # TODO: stopped removing it (or remove after replay):
+    #   decide what to do for this folder: for now depend on it for index/run: 
+    # removeDir(outputFolder)
 
     let recordCore = envLoadRecordCore()
 
-    return runRecordedTrace(trace, test=false, structuredDiffJson=structuredDiffJson, indexDiffJson=indexDiffJson, recordCore=recordCore)
+    return runRecordedTrace(trace, test=false, structuredDiffPath=structuredDiffPath, indexDiffPath=diffIndexPath, recordCore=recordCore)
 
 proc indexDiff*(multitracePath: string) =
   discard replayMultitrace(multitracePath, indexDiff=true)

src/ct/trace/run.nim

@@ -10,17 +10,17 @@ import std/[osproc, strformat, sequtils],
 proc runRecordedTrace*(
   trace: Trace,
   test: bool,
-  structuredDiffJson: string = "",
-  indexDiffJson: string = "",
+  structuredDiffPath: string = "",
+  indexDiffPath: string = "",
   recordCore: bool = false
 ): bool =
   var args = if test: @[$trace.id, "--test"] else: @[$trace.id]
-  if structuredDiffJson.len > 0:
+  if structuredDiffPath.len > 0:
     args.add("--diff")
-    args.add(structuredDiffJson)
-    if indexDiffJson.len > 0:
+    args.add(structuredDiffPath)
+    if indexDiffPath.len > 0:
       args.add("--diff-index")
-      args.add(indexDiffJson)
+      args.add(indexDiffPath)
   return launchElectron(args, trace, ElectronLaunchMode.Default, recordCore, test)
 
 
@@ -57,7 +57,7 @@ proc runWithRestart(
         if not afterRestart:
           # for now assume not a multitrace/no diff
           # .. returns true if it should restart
-          runRecordedTrace(recordedTrace, test, structuredDiffJson="", indexDiffJson="", recordCore=recordCore)
+          runRecordedTrace(recordedTrace, test, structuredDiffPath="", indexDiffPath="", recordCore=recordCore)
         else:
           let process = startProcess(codetracerExe, args = @["replay", fmt"--id={recordedTrace.id}"], options = {poParentStreams})
           waitForExit(process) == RESTART_EXIT_CODE

src/db-backend/src/main.rs

@@ -65,7 +65,7 @@ enum Commands {
         stdio: bool,
     },
     IndexDiff {
-        structured_diff_raw: String, // _path: std::path::PathBuf,
+        structured_diff_path: std::path::PathBuf,
         trace_folder: std::path::PathBuf,
         multitrace_folder: std::path::PathBuf,
     },
@@ -139,13 +139,12 @@ fn main() -> Result<(), Box<dyn Error>> {
             };
         }
         Commands::IndexDiff {
-            structured_diff_raw,
+            structured_diff_path,
             trace_folder,
             multitrace_folder,
         } => {
-            // TODO: eventually structured diff path? big diff-s might not fit into CLI args
-            // let raw = std::fs::read_to_string(structured_diff_path)?;
-            let structured_diff = serde_json::from_str::<diff::Diff>(&structured_diff_raw)?;
+            let raw = std::fs::read_to_string(structured_diff_path)?;
+            let structured_diff = serde_json::from_str::<diff::Diff>(&raw)?;
             diff::index_diff(structured_diff, &trace_folder, &multitrace_folder)?;
         }
     }

src/frontend/index/args.nim

@@ -48,7 +48,7 @@ proc parseArgs* =
         data.startOptions.inTest = true
       elif arg == cstring"--diff":
         if i + 1 < args.len:
-          data.startOptions.diff = cast[Diff](JSON.parse(args[i + 1]))
+          data.startOptions.diff = cast[Diff](JSON.parse(fs.readFileSync(args[i + 1], cstring"utf8")))
           data.startOptions.withDiff = true
           i += 2
           continue
@@ -57,7 +57,7 @@ proc parseArgs* =
           break
       elif arg == cstring"--diff-index":
         if i + 1 < args.len:
-          data.startOptions.rawDiffIndex = args[i + 1]
+          data.startOptions.rawDiffIndex = fs.readFileSync(args[i + 1], cstring"utf-8")
           i += 2
           continue
         else: