feat: Implement upload/download of trace records

.gitmodules

@@ -65,3 +65,9 @@
 [submodule "libs/codetracer-python-recorder"]
 	path = libs/codetracer-python-recorder
 	url = https://github.com/metacraft-labs/codetracer-python-recorder.git
+[submodule "libs/nimcrypto"]
+	path = libs/nimcrypto
+	url = https://github.com/cheatfate/nimcrypto.git
+[submodule "libs/zip"]
+	path = libs/zip
+	url = git@github.com:nim-lang/zip.git

nim.cfg

@@ -23,5 +23,7 @@ path:"libs/chronos"
 path:"libs/parsetoml/src"
 path:"libs/nim-result"
 path:"libs/nim-confutils"
+path:"libs/nimcrypto"
+path:"libs/zip"
 
 gcc.options.debug = "-O0 -g3"

nix/shells/main.nix

@@ -84,6 +84,7 @@ in
         unixtools.killall
         # zip
         # unzip
+        libzip
         # curl
 
         # for pgrep at least
@@ -157,7 +158,7 @@ in
         # copied case for libstdc++.so (needed by better-sqlite3) from
         # https://discourse.nixos.org/t/what-package-provides-libstdc-so-6/18707/4:
         # gcc.cc.lib ..
-        export CT_LD_LIBRARY_PATH="${sqlite.out}/lib/:${pcre.out}/lib:${glib.out}/lib:${openssl.out}/lib:${gcc.cc.lib}/lib";
+        export CT_LD_LIBRARY_PATH="${sqlite.out}/lib/:${pcre.out}/lib:${glib.out}/lib:${openssl.out}/lib:${gcc.cc.lib}/lib:${libzip.out}/lib";
 
         export RUST_LOG=info
 

src/common/common_trace_index.nim

@@ -55,7 +55,8 @@ const SQL_INITIAL_INSERT_STATEMENTS = @[
 
 const SQL_ALTER_TABLE_STATEMENTS: seq[string] = @[
    # example: adding a new column
-   """ALTER TABLE traces ADD COLUMN calltraceMode text;""",
-   """ALTER TABLE traces RENAME COLUMN callgraph TO calltrace"""
+   """ALTER TABLE traces ADD COLUMN remoteShareDownloadId text;""",
+   """ALTER TABLE traces ADD COLUMN remoteShareControlId text;""",
+   """ALTER TABLE traces ADD COLUMN remoteShareExpireTime INTEGER DEFAULT -1;"""
    # """ALTER TABLE traces ADD COLUMN love integer;"""
 ]

src/common/common_types.nim

@@ -153,6 +153,9 @@ type
     rrPid*: int
     exitCode*: int
     calltraceMode*: CalltraceMode
+    downloadKey*: langstring
+    controlId*: langstring
+    onlineExpireTime*: int
 
   CalltraceMode* {.pure.} = enum NoInstrumentation, CallKeyOnly, RawRecordNoValues, FullRecord
 
@@ -1341,6 +1344,19 @@ type
   BugReportArg* = object ## BugReport arg
     title*: langstring
     description*: langstring
+
+  UploadTraceArg* = object
+    trace*: Trace
+    programName*: langstring
+
+  UploadedTraceData* = object
+    downloadKey*: langstring
+    controlId*: langstring
+    expireTime*: langstring
+
+  DeleteTraceArg* = object
+    traceId*: int
+    controlId*: langstring
 
   DbEventKind* {.pure.} = enum Record, Trace, History
 

src/common/config.nim

@@ -28,7 +28,11 @@ type
     shortcutMap* {.defaultVal: ShortcutMap().}: ShortcutMap
     defaultBuild*: string
     showMinimap*: bool
-    webApiRoot*:  string
+    baseUrl*: string
+    downloadApi*: string
+    uploadApi*: string
+    deleteApi*: string
+    traceSharingEnabled*: bool
 
   Config* = ref ConfigObject
 

src/common/trace_index.nim

@@ -64,6 +64,47 @@ proc ensureDB(test: bool): DBConn =
   globalDbMap[test.int] = db
   db
 
+proc updateField*(
+  id: int,
+  fieldName: string,
+  fieldValue: string,
+  test: bool
+) =
+  let db = ensureDB(test)
+  db.exec(
+    sql(&"UPDATE traces SET {fieldName} = ? WHERE id = ?"),
+    fieldValue, id
+  )
+  db.close()
+
+proc updateField*(
+  id: int,
+  fieldName: string,
+  fieldValue: int,
+  test: bool
+) =
+  let db = ensureDB(test)
+  db.exec(
+    sql(&"UPDATE traces SET {fieldName} = ? WHERE id = ?"),
+    fieldValue, id
+  )
+  db.close()
+
+proc getField*(
+  id: int,
+  fieldName: string,
+  test: bool
+): string =
+  let db = ensureDB(test)
+  let res = db.getAllRows(
+    sql(&"SELECT {fieldName} FROM traces WHERE id = ? LIMIT 1"),
+    id
+  )
+  db.close()
+  if res.len > 0:
+    return res[0][0]
+  return ""
+
 proc recordTrace*(
     id: int,
     program: string,
@@ -81,7 +122,8 @@ proc recordTrace*(
     exitCode: int,
     calltrace: bool,
     calltraceMode: CalltraceMode,
-    test: bool): Trace =
+    test: bool,
+    downloadKey: string = ""): Trace =
   # TODO pass here a Trace value and instead if neeeded construct it from other helpers
 
   let currentDate: DateTime = now()
@@ -107,19 +149,19 @@ proc recordTrace*(
             sourceFolders, lowLevelFolder, outputFolder,
             lang, imported, shellID,
             rrPid, exitCode,
-            calltrace, calltraceMode, date)
+            calltrace, calltraceMode, date, remoteShareDownloadId)
           VALUES (?, ?, ?,
              ?, ?, ?, ?,
              ?, ?, ?,
              ?, ?, ?,
              ?, ?,
-             ?, ?, ?)""",
+             ?, ?, ?, ?)""",
             $id, program, args.join(" "),
             compileCommand, env, workdir, "", # <- output
             sourceFolders, lowLevelFolder, outputFolder,
             $(lang.int), $(imported.int), $shellID,
             $rrPid, $exitCode,
-            ord(calltrace), $calltraceMode, $traceDate)
+            ord(calltrace), $calltraceMode, $traceDate, downloadKey)
       break
     except DbError:
       echo "error: ", getCurrentExceptionMsg()
@@ -177,6 +219,12 @@ proc loadCalltraceMode*(raw: string, lang: Lang): CalltraceMode =
 proc loadTrace(trace: Row, test: bool): Trace =
   try:
     let lang = trace[10].parseInt.Lang
+    var expireTime = -1
+    try:
+      expireTime = trace[20].parseInt
+    except:
+      discard
+
     result = Trace(
       id: trace[0].parseInt,
       program: trace[1],
@@ -196,7 +244,10 @@ proc loadTrace(trace: Row, test: bool): Trace =
       shellID: trace[14].parseInt,
 
       calltrace: trace[15].parseInt != 0,
-      calltraceMode: loadCalltraceMode(trace[16], lang))
+      calltraceMode: loadCalltraceMode(trace[16], lang),
+      downloadKey: trace[18],
+      controlId: trace[19],
+      onlineExpireTime: expireTime)
   except CatchableError as e:
     # assume db schema change?
     echo "internal error: ", e.msg
@@ -314,9 +365,17 @@ proc findByRecordProcessId*(pid: int, test: bool): Trace =
 
 proc findRecentTraces*(limit: int, test: bool): seq[Trace] =
   let db = ensureDB(test)
-  let traces = db.getAllRows(
-    sql("SELECT * FROM traces ORDER BY id DESC LIMIT ?"),
-    $limit)
+  let traces =
+    if limit == -1:
+      db.getAllRows(
+        sql("SELECT * FROM traces ORDER BY id DESC LIMIT ?"),
+        $limit
+      )
+    else:
+      db.getAllRows(
+        sql("SELECT * FROM traces ORDER BY id DESC")
+      )
+
   if traces.len > 0:
     result = traces.mapIt(it.loadTrace(test))
 

src/config/default_config.yaml

@@ -10,6 +10,7 @@ events: true
 # history: experimental support
 history: true
 repl: true
+traceSharingEnabled: false
 
 # === feature settings
 
@@ -54,7 +55,11 @@ defaultBuild: ""
 showMinimap: true
 
 # for now local setup
-webApiRoot: http://100.87.206.30:57103/api/codetracer
+baseUrl: http://localhost:55500/api/codetracer
+
+downloadApi: /download
+uploadApi: /upload
+deleteApi: /delete
 
 # # you can use KEY+OTHER
 # # use PageUp, PageDown, CTRL, ALT, SHIFT

src/ct/codetracerconf.nim

@@ -14,6 +14,7 @@ type
     install,
     upload,
     download,
+    cmdDelete,
     # build,
     record,
     console,
@@ -264,7 +265,16 @@ type
     of download:
       traceRegistryId* {.
         argument,
-        desc: "the trace registry unique id: <program-name>#<id> e.g. a.rb#5"
+        desc: "the trace registry unique id: <program-name>//<downloadId>//<password> e.g. noir//1234//asd"
+      .}: string
+    of cmdDelete:
+      traceId* {.
+        name: "trace-id"
+        desc: "trace trace unique id"
+      .}: int
+      controlId* {.
+        name: "control-id",
+        desc: "the trace control id to delete the online trace"
       .}: string
     of start_core:
       coreTraceArg* {.

src/ct/globals.nim

@@ -13,3 +13,10 @@ let
 var replayInteractive* = false
 var electronPid*: int = -1
 var rrPid* = -1 # global, so it can be updated immediately on starting a process, and then used in `onInterrupt` if needed
+
+# for now hardcode: files are usually useful and
+# probably much less perf/size compared to actual traces
+# it's still good to have an option/opt-out, so we leave that
+# as a flag in the internals, but not exposed to user yet
+# that's why for now it's hardcoded for db
+const DB_SELF_CONTAINED_DEFAULT* = true

src/ct/launch/launch.nim

@@ -2,6 +2,7 @@ import
   std/[strutils, strformat, osproc],
   ../../common/[ paths, types, intel_fix, install_utils, trace_index, start_utils ],
   ../utilities/[ git, env ],
+  ../online_sharing/trace_manager,
   ../cli/[ logging, list, help ],
   ../trace/[ replay, record, run, metadata ],
   ../codetracerconf,
@@ -101,15 +102,15 @@ proc runInitial*(conf: CodetracerConf) =
       notSupportedCommand($conf.cmd)
     of StartupCommand.upload:
       # similar to replay/console
-      notSupportedCommand($conf.cmd)
-      # eventually enable?
-      # uploadCommand(
-      #   conf.uploadLastTraceMatchingPattern,
-      #   conf.uploadTraceId,
-      #   conf.uploadTraceFolder,
-      #   replayInteractive)
+      uploadCommand(
+        conf.uploadLastTraceMatchingPattern,
+        conf.uploadTraceId,
+        conf.uploadTraceFolder,
+        replayInteractive)
     of StartupCommand.download:
-      notSupportedCommand($conf.cmd)
+      downloadTraceCommand(conf.traceRegistryId)
+    of StartupCommand.cmdDelete:
+      deleteTraceCommand(conf.traceId, conf.controlId)
       # eventually enable?
       # downloadCommand(conf.traceRegistryId)
     # of StartupCommand.build:

src/ct/online_sharing/security_upload.nim

@@ -0,0 +1,83 @@
+import nimcrypto, zip/zipfiles, std/[ sequtils, strutils, strformat, os, httpclient, mimetypes, uri ]
+import ../../common/[ config ]
+
+proc generateSecurePassword*(): string =
+  var key: array[32, byte]
+  discard randomBytes(key)
+
+  result = key.mapIt(it.toHex(2)).join("")
+  return result
+
+proc pkcs7Pad*(data: seq[byte], blockSize: int): seq[byte] =
+  let padLen = blockSize - (data.len mod blockSize)
+  result = data & repeat(cast[byte](padLen), padLen)
+
+proc pkcs7Unpad*(data: seq[byte]): seq[byte] =
+  if data.len == 0:
+    raise newException(ValueError, "Data is empty, cannot unpad")
+
+  let padLen = int64(data[^1])  # Convert last byte to int64 safely
+  if padLen <= 0 or padLen > data.len:
+    raise newException(ValueError, "Invalid padding")
+
+  result = data[0 ..< data.len - padLen]
+
+func toBytes*(s: string): seq[byte] =
+  ## Convert a string to the corresponding byte sequence - since strings in
+  ## nim essentially are byte sequences without any particular encoding, this
+  ## simply copies the bytes without a null terminator
+  when nimvm:
+    var r = newSeq[byte](s.len)
+    for i, c in s:
+      r[i] = cast[byte](c)
+    r
+  else:
+    @(s.toOpenArrayByte(0, s.high))
+
+proc encryptZip(zipFile, password: string) =
+  var iv: seq[byte] = password.toBytes()[0..15]
+
+  var aes: CBC[aes256]
+  aes.init(password.toOpenArrayByte(0, len(password) - 1), iv)
+
+  var zipData = readFile(zipFile).toBytes()
+  var paddedData = pkcs7Pad(zipData, 16)
+  var encrypted = newSeq[byte](paddedData.len)
+
+  aes.encrypt(paddedData, encrypted.toOpenArray(0, len(encrypted) - 1))
+  writeFile(zipFile & ".enc", encrypted)
+
+proc zipFileWithEncryption*(inputFile: string, outputZip: string, password: string) =
+  var zip: ZipArchive
+  if not zip.open(outputZip, fmWrite):
+    raise newException(IOError, "Failed to create zip file: " & outputZip)
+
+  for file in walkDirRec(inputFile):
+    let relPath = file.relativePath(inputFile)
+    zip.addFile(relPath, file)
+
+  zip.close()
+  encryptZip(outputZip, password)
+
+proc uploadEncyptedZip*(file: string): (string, int) =
+  let config = loadConfig(folder=getCurrentDir(), inTest=false)
+  var exitCode = 0
+  var response = ""
+
+  var client = newHttpClient()
+  let mimes = newMimetypes()
+  var data = newMultipartData()
+
+  data.addFiles({"file": file & ".enc"}, mimeDb = mimes)
+
+  try:
+    response = client.postContent(fmt"{parseUri(config.baseUrl) / config.uploadApi}", multipart=data)
+    exitCode = 0
+  except CatchableError as e:
+    echo fmt"error: can't upload to API: {e.msg}"
+    response = ""
+    exitCode = 1
+  finally:
+    client.close()
+
+  (response, exitCode)