encodes Nim PSym to NIF Symbol and SymbolDef

Author: demotomohiro

compiler/icnif/nifdecoder.nim

@@ -1,11 +1,12 @@
-import std / [assertions, tables]
-import "../../dist/nimony/src/lib" / [bitabs, nifreader, nifstreams, nifcursors]
-import ".." / [ast, astalgo, idents, lineinfos, options, modulegraphs, pathutils]
+import std / [assertions, tables, parseutils]
+import "../../dist/nimony/src/lib" / [bitabs, nifreader, nifstreams, nifcursors, symparser]
+import ".." / [ast, astalgo, idents, lineinfos, options, modulegraphs, msgs, pathutils]
 import enum2nif
 
 type
   DecodeContext = object
     graph: ModuleGraph
+    nifSymToPSym: Table[string, PSym] # foo.1.modsuffix -> PSym
 
 proc nodeKind(n: Cursor): TNodeKind {.inline.} =
   assert n.kind == ParLe
@@ -21,36 +22,87 @@ proc getSysType(typeKind: TTypeKind): PType =
   else:
     result = PType(itemId: ItemId(module: 0, item: typeKind.int32), kind: typeKind)
 
+type
+  SplittedNifSym = object
+    name: string
+    id: int
+    module: string
+
+proc splitNifSym(s: string): SplittedNifSym =
+  result = SplittedNifSym()
+  var i = s.len - 2
+  var mp = -1
+  while i > 0:
+    if s[i] == '.':
+      if s[i+1] in {'0'..'9'}:
+        var id = 0
+        discard parseutils.parseInt(s, id, i + 1)
+        return SplittedNifSym(
+          name: s.substr(0, i - 1),
+          id: id,
+          module: if mp < 0: "" else: s.substr(mp, s.high))
+      else:
+        mp = i + 1
+    dec i
+
 proc fromNif(c: var DecodeContext; n: var Cursor): PNode
 
-proc fromNifLocal(c: var DecodeContext; n: var Cursor; kind: TNodeKind): PNode =
-  result = newNodeI(kind, unknownLineInfo, 1)
-  inc n
-  assert n.nodeKind == nkIdentDefs
-  result[0] = newNodeI(nkIdentDefs, unknownLineInfo, 3)
-  inc n
+proc fromNifSymDef(c: var DecodeContext; n: var Cursor; kind: TNodeKind): PNode =
   assert n.nodeKind == nkSym
   let symKind = case kind:
     of nkVarSection: skVar
     of nkLetSection: skLet
+    of nkImportStmt: skModule
     else: skConst
   inc n
-  assert n.kind == Ident
-  let ident = c.graph.cache.getIdent(pool.strings[n.litId])
+  assert n.kind == SymbolDef
+  let nifSym = pool.syms[n.symId]
+  let symdef = nifSym.splitNifSym
+  assert symdef.name.len != 0
+  let ident = c.graph.cache.getIdent(symdef.name)
+  inc n
+  assert n.kind in {Ident, DotToken}, $n.kind
+  let flags = if n.kind == Ident: pool.strings[n.litId].parseSymFlags else: {}
   inc n
-  assert n.kind == IntLit
-  let id = pool.integers[n.intId]
-  result[0][0] = PSym(itemId: ItemId(module: 0, item: id.int32), kind: symKind, name: ident).newSymNode()
+  var position = 0
+  if symKind == skModule:
+    assert n.kind == StringLit
+    let path = pool.strings[n.litId].AbsoluteFile
+    position = fileInfoIdx(c.graph.config, path).int
+  else:
+    assert n.kind == IntLit
+    position = pool.integers[n.intId]
   inc n
+
+  let psym = PSym(itemId: ItemId(module: 0, item: symdef.id.int32), kind: symKind, name: ident, flags: flags, position: position)
+  result = newSymNode(psym)
+  let hasSym = c.nifSymToPSym.hasKeyOrPut(nifSym, psym)
+  assert not hasSym
+
   assert n.kind == ParRi
   inc n
+
+proc fromNifLocal(c: var DecodeContext; n: var Cursor; kind: TNodeKind): PNode =
+  result = newNodeI(kind, unknownLineInfo, 1)
+  inc n
+  assert n.nodeKind == nkIdentDefs
+  result[0] = newNodeI(nkIdentDefs, unknownLineInfo, 3)
+  inc n
+  result[0][0] = fromNifSymDef(c, n, kind)
   result[0][1] = fromNif(c, n)
   result[0][2] = fromNif(c, n)
-  assert n.kind == ParRi
+  assert n.kind == ParRi  # nkIdentDefs
   inc n
   assert n.kind == ParRi
   inc n
 
+proc fromNifImport(c: var DecodeContext; n: var Cursor): PNode =
+  result = newNode(nkImportStmt)
+  inc n
+  while n.kind != ParRi:
+    result.add fromNifSymDef(c, n, nkImportStmt)
+  inc n
+
 proc fromNif(c: var DecodeContext; n: var Cursor): PNode =
   result = nil
   case n.kind:
@@ -60,6 +112,10 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode =
   of IntLit:
     result = newIntTypeNode(pool.integers[n.intId], getSysType(tyInt))
     inc n
+  of Symbol:
+    let sym = c.nifSymToPSym[pool.syms[n.symId]]
+    result = newSymNode(sym)
+    inc n
   of ParLe:
     let kind = n.nodeKind
     case kind:
@@ -71,6 +127,8 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode =
       inc n
     of nkVarSection, nkLetSection:
       result = fromNifLocal(c, n, kind)
+    of nkImportStmt:
+      result = fromNifImport(c, n)
     else:
       assert false, "Not yet implemented " & $kind
   else:

compiler/icnif/nifencoder.nim

@@ -1,18 +1,63 @@
-import std / [assertions]
-import ".." / [ast, astalgo, idents]
+import std / [assertions, tables]
+import ".." / [ast, astalgo, idents, lineinfos, msgs, options]
 import "../../dist/nimony/src/lib" / nifbuilder
+import "../../dist/nimony/src/gear2" / modnames
 import enum2nif
 
 type
   EncodeContext = object
     b: Builder
+    conf: ConfigRef
+    toSuffix: Table[FileIndex, string]
+    moduleToNifSuffix: Table[ItemId, string] # module PSym -> module suffix
 
-proc toNifSymDef(c: var EncodeContext; n: PNode) =
+proc modname(c: var EncodeContext; idx: FileIndex): string =
+  # copied from ../nifgen.nim
+  result = c.toSuffix.getOrDefault(idx)
+  if result.len == 0:
+    let fp = toFullPath(c.conf, idx)
+    result = moduleSuffix(fp, cast[seq[string]](c.conf.searchPaths))
+    c.toSuffix[idx] = result
+
+proc toNifSym(c: var EncodeContext; sym: PSym): string =
+  result = sym.name.s & '.' & $sym.itemId.item
+  if sym.owner.kind == skModule:
+    result.add '.'
+    var modsuf = c.moduleToNifSuffix.getOrDefault(sym.owner.itemId)
+    if modsuf.len == 0:
+      modsuf = modname(c, sym.owner.position.FileIndex)
+      c.moduleToNifSuffix[sym.owner.itemId] = modsuf
+    result.add modsuf
+
+proc symdefToNif(c: var EncodeContext; n: PNode) =
   assert n.kind == nkSym
   let sym = n.sym
   c.b.addTree toNifTag(n.kind)
-  c.b.addIdent sym.name.s
-  c.b.addIntLit sym.itemId.item
+  var name = toNifSym(c, sym)
+  c.b.addSymbolDef name
+  if sym.flags == {}:
+    c.b.addEmpty()
+  else:
+    var flags: string = ""
+    genFlags(sym.flags, flags)
+    c.b.addIdent flags
+  if sym.kind == skModule:
+    # position is module's FileIndex but it cannot be directly encoded
+    # as the uniqueness of it can broke
+    # if any import/include statements are changed.
+    let path = toFullPath(c.conf, sym.position.FileIndex)
+    c.b.addStrLit path
+  else:
+    c.b.addIntLit sym.position
+  c.b.endTree()
+
+proc toNifImport(c: var EncodeContext; n: PNode) =
+  c.b.addTree toNifTag(n.kind)
+  for i in 0 ..< n.len:
+    assert n[i].kind == nkSym
+    let sym = n[i].sym
+    assert sym.kind == skModule
+    symdefToNif(c, n[i])
   c.b.endTree()
 
 proc toNif(c: var EncodeContext; n: PNode) =
@@ -24,10 +69,28 @@ proc toNif(c: var EncodeContext; n: PNode) =
   of nkIdentDefs:
     c.b.addTree toNifTag(n.kind)
     assert n.len == 3
-    toNifSymDef(c, n[0])
+    symdefToNif(c, n[0])
     toNif c, n[1]
     toNif c, n[2]
     c.b.endTree()
+  of nkSym:
+    when false:
+      echo "nkSym: ", n.sym.name.s
+      if n.sym.kind == skModule:
+        echo "position = ", n.sym.position
+      debug(n.sym)
+      var o = n.sym.owner
+      for i in 0 .. 20:
+        if o == nil:
+          break
+        echo "owner ", i, ":"
+        if o.kind == skModule:
+          echo "position = ", o.position
+        debug(o)
+        o = o.owner
+    c.b.addSymbol toNifSym(c, n.sym)
+  of nkImportStmt:
+    toNifImport(c, n)
   else:
     assert n.len > 0, $n.kind
     c.b.addTree toNifTag(n.kind)
@@ -37,20 +100,16 @@ proc toNif(c: var EncodeContext; n: PNode) =
 
 proc saveNif(c: var EncodeContext; n: PNode) =
   c.b.addHeader "nim2", "nim2-ic-nif"
-  c.b.addTree "stmts"
-  assert n.kind == nkStmtList
-  for i in 0 ..< n.len:
-    toNif c, n[i]
-  c.b.endTree()
+  toNif c, n
 
-proc saveNifFile*(module: PSym; n: PNode) =
+proc saveNifFile*(module: PSym; n: PNode; conf: ConfigRef) =
   let outfile = module.name.s & ".nif"
-  var c = EncodeContext(b: nifbuilder.open(outfile))
+  var c = EncodeContext(b: nifbuilder.open(outfile), conf: conf)
   saveNif(c, n)
   c.b.close()
 
-proc saveNifToBuffer*(n: PNode): string =
-  var c = EncodeContext(b: nifbuilder.open(100))
+proc saveNifToBuffer*(n: PNode; conf: ConfigRef): string =
+  var c = EncodeContext(b: nifbuilder.open(100), conf: conf)
   saveNif(c, n)
   c.b.close()
   result = c.b.extract

compiler/pipelines.nim

@@ -187,9 +187,9 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
         # encode the asts of specific modules to NIF files for debug or tests.
         if module.name.s.substr(0, 3) == "test":
           echo "saveNifFile: ", module.name.s
-          echo "n:"
-          debug(n)
-          nifencoder.saveNifFile(module, semNode)
+          echo "semNode:"
+          debug(semNode)
+          nifencoder.saveNifFile(module, semNode, graph.config)
 
     closeParser(p)
     if s.kind != llsStdIn: break

tests/icnif/tencode.nim

@@ -6,15 +6,17 @@ var graph = newModuleGraph(newIdentCache(), newConfigRef())
 
 block:
   const TestNif = """
-(.nif24)
-(.vendor "nim2")
-(.dialect "nim2-ic-nif")
-(stmts
- (var
-  (identdefs
-   (sym x +1) . +123))
- (var
-  (identdefs
-   (sym y +2) . +321)))"""
+    (.nif24)
+    (.vendor "nim2")
+    (.dialect "nim2-ic-nif")
+    (stmts
+     (import
+      (sym :testmod2.1 . "/home/nimdev/testnim/testmod2.nim"))
+     (var
+      (identdefs
+       (sym :x.2.tesvp2f3v ueg +0) . +123))
+     (var
+      (identdefs
+       (sym :y.3.tesvp2f3v g +0) . x.2.tesvp2f3v)))"""
 
-  assert loadNifFromBuffer(TestNif, graph).saveNifToBuffer() == TestNif
+  assert loadNifFromBuffer(TestNif, graph).saveNifToBuffer(graph.config) == TestNif