loads modules imported by system module and create module sym before decode

Author: demotomohiro

compiler/icnif/nifdecoder.nim

@@ -129,7 +129,7 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym =
   # PNode, PSym or PType type fields in PSym can have cycles.
   # Add PSym to `c.symbols` before parsing these fields so that 
   # they can refer this PSym.
-  let nifItemId = ItemId(module: nifModId, item: itemId)
+  let nifItemId = ItemId(module: itemIdModule.int32, item: itemId)
   assert nifItemId notin c.symbols
   c.symbols[nifItemId] = result
 
@@ -231,10 +231,10 @@ proc fromNifSymbol(c: var DecodeContext; n: var Cursor): PSym =
       result = c.fromNifSymDef n
     elif n.tagId == symTag:
       incExpect n, IntLit
-      let nifModId = pool.integers[n.intId].int32
+      let nifModId = c.modules[pool.integers[n.intId].int32]
       incExpect n, IntLit
       let item = pool.integers[n.intId].int32
-      let nifItemId = ItemId(module: nifModId, item: item)
+      let nifItemId = ItemId(module: nifModId.int32, item: item)
       result = c.symbols[nifItemId]
       inc n
       skipParRi n
@@ -356,9 +356,33 @@ proc loadNif(stream: var Stream; graph: ModuleGraph; prog: NifProgram): PNode =
 
   var buf = fromStream(stream)
   var n = beginRead(buf)
-
   var c = DecodeContext(graph: graph, prog: prog)
 
+  var n2 = n
+  var nested = 0
+  while true:
+    if n2.info != NoLineInfo:
+      break
+    elif n2.kind == EofToken:
+      break
+    elif n2.kind == ParLe:
+      inc nested
+    elif n2.kind == ParRi:
+      dec nested
+      if nested == 0: break
+    inc n2
+  assert n2.info != NoLineInfo
+  let info = pool.man.unpack(n2.info)
+  let fileIdx = c.graph.config.fileInfoIdx(pool.files[info.file].AbsoluteFile)
+  var currentModule = graph.newModule(fileIdx)
+  if currentModule.itemId.module == 0'i32:
+    currentModule.flags = {sfMainModule, sfSystemModule}
+
+  c.symbols[currentModule.itemId] = currentModule
+  let nifSym = toNifSym(currentModule, c.prog.moduleToNifSuffix, c.graph.config)
+  let symId = pool.syms.getOrIncl(nifSym)
+  c.prog.nifSymIdToPSym[symId] = currentModule
+
   result = fromNif(c, n)
 
   endRead(buf)

compiler/icnif/nifencoder.nim

@@ -17,6 +17,7 @@ proc initEncodeContext(conf: ConfigRef; currentModule: PSym): EncodeContext =
   result = EncodeContext(conf: conf,
                          currentModule: currentModule,
                          dest: createTokenBuf())
+  result.decodedSyms.incl(currentModule.itemId)
 
 template buildTree(dest: var TokenBuf; tag: TagId; body: untyped) =
   dest.addParLe tag

tests/icnif/tencode_node2node.nim

@@ -25,16 +25,16 @@ proc newModuleGraphForSem(cache: IdentCache; conf: ConfigRef): ModuleGraph =
   graph.config.cmd = oldCmd
   result = graph
 
-proc getSystemNif(graph: ModuleGraph): string =
-  assert graph.systemModule != nil
-  assert graph.systemModule.kind == skModule
-  assert graph.systemModule.ast != nil
-
-  let n = graph.systemModule.ast
-  # if nil is not assigned, it generates large NIF
-  graph.systemModule.ast = nil
-  result = saveNifToBuffer(n, graph.config, graph.systemModule)
-  #writeFile("system.nif", result)
+proc getSystemNif(graph: ModuleGraph): seq[string] =
+  result = newSeqOfCap[string](graph.ifaces.len)
+  for i, iface in graph.ifaces.mpairs:
+    if iface.module != nil:
+      let n = iface.module.ast
+      assert n != nil
+      # if nil is not assigned, it generates large NIF
+      iface.module.ast = nil
+      result.add saveNifToBuffer(n, graph.config, iface.module)
+      #writeFile(iface.module.name.s & ".nif", result[^1])
 
 proc sem(graph: ModuleGraph; path: AbsoluteFile): (PNode, PSym) =
   result = (nil, nil)
@@ -169,10 +169,12 @@ proc eql(x, y: PSym; c: var EqlContext): bool =
   elif x.magic != y.magic:
     echo "symbol magic mismatch: ", x.magic, "/", y.magic
     result = false
-  elif not eql(x.info, y.info, c):
+  elif x.kind != skPackage and not eql(x.info, y.info, c):
+    # fileIndex of info of skPackage is just a path of first semchecked module in the package
     echo "symbol line info mismatch"
     result = false
-  elif x.flags != y.flags:
+  elif x.kind != skModule and x.flags != y.flags:
+    # TODO: check the flag of skModule
     echo "symbol flag mismatch: ", x.flags, "/", y.flags
     result = false
   elif x.options != y.options:
@@ -360,10 +362,11 @@ proc eql(x, y: PNode; c: var EqlContext): bool =
     debug(y)
     result = false
 
-proc testNifEncDec(graph: ModuleGraph; src: string; systemNif: string) =
+proc testNifEncDec(graph: ModuleGraph; src: string; systemNif: openArray[string]) =
   let fullPath = TestCodeDir / RelativeFile(src)
   let (n, module) = sem(graph, fullPath)
   assert n != nil, "failed to sem " & $fullPath
+  assert module.owner.kind == skPackage
 
   #debug(n)
   let nif = saveNifToBuffer(n, graph.config, module)
@@ -374,7 +377,8 @@ proc testNifEncDec(graph: ModuleGraph; src: string; systemNif: string) =
   # Don't reuse the ModuleGraph used for semcheck when load NIF.
   var graphForLoad = newModuleGraph(newIdentCache(), newConfigRefForTest())
   var prog = NifProgram()
-  discard loadNifFromBuffer(systemNif, graphForLoad, prog)
+  for sysNif in systemNif:
+    discard loadNifFromBuffer(sysNif, graphForLoad, prog)
   let n2 = loadNifFromBuffer(nif, graphForLoad, prog)
   #debug(n2)
   var c = EqlContext(confX: graph.config, confY: graphForLoad.config)