import plugins (#1165)

Author: Araq

src/nimony/builtintypes.nim

@@ -4,7 +4,7 @@
 # See the file "license.txt", included in this
 # distribution, for details about the copyright.
 
-include nifprelude
+include ".." / lib / nifprelude
 import nimony_model
 
 type

src/nimony/decls.nim

@@ -7,7 +7,8 @@
 ## Helpers for declarative constructs like `let` statements or `proc` declarations.
 
 import std / assertions
-import nifstreams, nifcursors, nimony_model, programs
+import ".." / lib / [nifstreams, nifcursors]
+import ".." / nimony / [nimony_model, programs]
 
 proc isRoutine*(t: SymKind): bool {.inline.} =
   t in {ProcY, FuncY, IteratorY, MacroY, TemplateY, ConverterY, MethodY}

src/nimony/deps.nim

@@ -50,6 +50,7 @@ type
     id, parent: int
     active: int
     isSystem: bool
+    plugin: string
 
   Command* = enum
     DoCheck, # like `nim check`
@@ -68,6 +69,7 @@ type
     processedModules: HashSet[string]
     moduleFlags: set[ModuleFlag]
     isGeneratingFinal: bool
+    foundPlugins: HashSet[string]
 
 proc toPair(c: DepContext; f: string): FilePair =
   FilePair(nimFile: f, modname: moduleSuffix(f, c.config.paths))
@@ -88,6 +90,9 @@ proc processInclude(c: var DepContext; it: var Cursor; current: Node) =
       discard "ignore wrong `include` statement"
     else:
       for f1 in items(files):
+        if f1.plugin.len > 0:
+          discard "ignore plugin include file, will cause an error in sem.nim"
+          continue
         let f2 = resolveFileWrapper(c.config.paths, current.files[current.active].nimFile, f1.path)
         # check for recursive include files:
         var isRecursive = false
@@ -133,6 +138,16 @@ proc importSingleFile(c: var DepContext; f1: string; info: PackedLineInfo;
     else:
       current.deps.add p
 
+proc processPluginImport(c: var DepContext; f: ImportedFilename; info: PackedLineInfo; current: Node) =
+  let f2 = resolveFileWrapper(c.config.paths, current.files[current.active].nimFile, f.path)
+  if not semos.fileExists(f2): return
+  let p = c.toPair(f2)
+  if not c.processedModules.containsOrIncl(p.modname):
+    current.deps.add p
+    c.nodes.add Node(files: @[p], id: c.nodes.len,
+                     parent: current.id, isSystem: false, plugin: f.plugin)
+    c.foundPlugins.incl f.plugin
+
 proc processImport(c: var DepContext; it: var Cursor; current: Node) =
   let info = it.info
   var x = it
@@ -142,8 +157,8 @@ proc processImport(c: var DepContext; it: var Cursor; current: Node) =
   if x.stmtKind == WhenS: return
   while x.kind != ParRi:
     if x.kind == ParLe and x.exprKind == PragmaxX:
-      inc x
       var y = x
+      inc y
       skip y
       if y.substructureKind == PragmasU:
         inc y
@@ -157,7 +172,10 @@ proc processImport(c: var DepContext; it: var Cursor; current: Node) =
       discard "ignore wrong `import` statement"
     else:
       for f in files:
-        importSingleFile c, f.path, info, current, false
+        if f.plugin.len == 0:
+          importSingleFile c, f.path, info, current, false
+        else:
+          processPluginImport c, f, info, current
 
 proc processSingleImport(c: var DepContext; it: var Cursor; current: Node) =
   # process `from import` and `import except` which have a single module expression
@@ -174,7 +192,10 @@ proc processSingleImport(c: var DepContext; it: var Cursor; current: Node) =
     discard "ignore wrong `from` statement"
   else:
     for f in files:
-      importSingleFile c, f.path, info, current, false
+      if f.plugin.len == 0:
+        importSingleFile c, f.path, info, current, false
+      else:
+        processPluginImport c, f, info, current
       break
 
 proc processDep(c: var DepContext; n: var Cursor; current: Node) =
@@ -237,7 +258,7 @@ proc parseDeps(c: var DepContext; p: FilePair; current: Node) =
     nifstreams.close(stream)
 
 type
-  CFile = tuple
+  CFile = object
     name, obj, customArgs: string
 
 proc rootPath(c: DepContext): string =
@@ -253,7 +274,7 @@ proc toBuildList(c: DepContext): seq[CFile] =
       let path = i[1]
       let obj = splitFile(path).name & ".o"
       let customArgs = i[2]
-      result.add (path, obj, customArgs)
+      result.add CFile(name: path, obj: obj, customArgs: customArgs)
 
 proc generateFinalBuildFile(c: DepContext; commandLineArgsNifc: string; passC, passL: string): string =
   result = c.config.nifcachePath / c.rootNode.files[0].modname & ".final.build.nif"
@@ -387,6 +408,46 @@ proc generateFinalBuildFile(c: DepContext; commandLineArgsNifc: string; passC, p
 proc cachedConfigFile(config: NifConfig): string =
   config.nifcachePath / "cachedconfigfile.txt"
 
+proc generateSemInstructions(c: DepContext; v: Node; b: var Builder; isMain: bool) =
+  b.withTree "do":
+    b.addIdent "nimsem"
+    b.withTree "args":
+      if v.isSystem:
+        b.addStrLit "--isSystem"
+      elif isMain:
+        b.addStrLit "--isMain"
+    # Input: parsed file
+    var seenDeps = initHashSet[string]()
+    for f in v.files:
+      let pf = c.config.parsedFile(f)
+      if not seenDeps.containsOrIncl(pf):
+        b.withTree "input":
+          b.addStrLit pf
+    # Input: dependencies
+    for f in v.deps:
+      let idxFile = c.config.indexFile(f)
+      if not seenDeps.containsOrIncl(idxFile):
+        b.withTree "input":
+          b.addStrLit idxFile
+    # Input: cached config file
+    b.withTree "input":
+      b.addStrLit c.config.cachedConfigFile()
+    # Outputs: semmed file and index file
+    b.withTree "output":
+      b.addStrLit c.config.semmedFile(v.files[0])
+    b.withTree "output":
+      b.addStrLit c.config.indexFile(v.files[0])
+
+proc generatePluginSemInstructions(c: DepContext; v: Node; b: var Builder) =
+  b.withTree "do":
+    b.addIdent v.plugin
+    b.withTree "input":
+      b.addStrLit v.files[0].nimFile
+    b.withTree "output":
+      b.addStrLit c.config.semmedFile(v.files[0])
+    b.withTree "output":
+      b.addStrLit c.config.indexFile(v.files[0])
+
 proc generateFrontendBuildFile(c: DepContext; commandLineArgs: string): string =
   result = c.config.nifcachePath / c.rootNode.files[0].modname & ".build.nif"
   var b = nifbuilder.open(result)
@@ -420,44 +481,32 @@ proc generateFrontendBuildFile(c: DepContext; commandLineArgs: string): string =
       b.withTree "output":
         b.addIntLit 1  # index file output
 
-    # Build rules for semantic checking
-    var i = 0
-    for v in c.nodes:
-      b.withTree "do":
-        # Choose the right command based on flags
-        b.addIdent "nimsem"
-        b.withTree "args":
-          if v.isSystem:
-            b.addStrLit "--isSystem"
-          elif i == 0:  # first node is main
-            b.addStrLit "--isMain"
-
-        # Input: parsed file
-        var seenDeps = initHashSet[string]()
-        for f in v.files:
-          let pf = c.config.parsedFile(f)
-          if not seenDeps.containsOrIncl(pf):
-            b.withTree "input":
-              b.addStrLit pf
-        # Input: dependencies
-        for f in v.deps:
-          let idxFile = c.config.indexFile(f)
-          if not seenDeps.containsOrIncl(idxFile):
-            b.withTree "input":
-              b.addStrLit idxFile
-        # Input: cached config file
+    for plugin in c.foundPlugins:
+      b.withTree "cmd":
+        b.addSymbolDef plugin
+        b.addStrLit plugin
+        b.addKeyw "args"
         b.withTree "input":
-          b.addStrLit c.config.cachedConfigFile()
-        # Outputs: semmed file and index file
+          b.addIntLit 0  # main parsed file
         b.withTree "output":
-          b.addStrLit c.config.semmedFile(v.files[0])
+          b.addIntLit 0  # semmed file output
         b.withTree "output":
-          b.addStrLit c.config.indexFile(v.files[0])
+          b.addIntLit 1  # index file output
+
+    # Build rules for semantic checking
+    var i = 0
+    for v in c.nodes:
+      if v.plugin.len == 0:
+        generateSemInstructions c, v, b, i == 0
+      else:
+        generatePluginSemInstructions c, v, b
       inc i
 
     # Build rules for parsing
     var seenFiles = initHashSet[string]()
     for v in c.nodes:
+      if v.plugin.len > 0:
+        continue
       for i in 0..<v.files.len:
         let f = c.config.parsedFile(v.files[i])
         if not seenFiles.containsOrIncl(f):
@@ -481,6 +530,16 @@ proc generateCachedConfigFile(c: DepContext; passC, passL: string) =
   if needUpdate:
     writeFile path, configStr
 
+proc initDepContext(config: sink NifConfig; project, nifler: string; isFinal, forceRebuild: bool; moduleFlags: set[ModuleFlag]; cmd: Command): DepContext =
+  result = DepContext(nifler: nifler, config: config, rootNode: nil, includeStack: @[],
+    forceRebuild: forceRebuild, moduleFlags: moduleFlags, nimsem: findTool("nimsem"),
+    cmd: cmd, isGeneratingFinal: isFinal)
+  let p = result.toPair(project)
+  result.rootNode = Node(files: @[p], id: 0, parent: -1, active: 0, isSystem: IsSystem in moduleFlags)
+  result.nodes.add result.rootNode
+  result.processedModules.incl p.modname
+  parseDeps result, p, result.rootNode
+
 proc buildGraph*(config: sink NifConfig; project: string; forceRebuild, silentMake: bool;
     commandLineArgs, commandLineArgsNifc: string; moduleFlags: set[ModuleFlag]; cmd: Command;
     passC, passL: string) =
@@ -493,18 +552,7 @@ proc buildGraph*(config: sink NifConfig; project: string; forceRebuild, silentMa
       quoteShell(cfgNif)
     parseNifConfig cfgNif, config
 
-  template initDepContext(isFinal: bool): DepContext =
-    var c = DepContext(nifler: nifler, config: config, rootNode: nil, includeStack: @[],
-      forceRebuild: forceRebuild, moduleFlags: moduleFlags, nimsem: findTool("nimsem"),
-      cmd: cmd, isGeneratingFinal: isFinal)
-    let p = c.toPair(project)
-    c.rootNode = Node(files: @[p], id: 0, parent: -1, active: 0, isSystem: IsSystem in moduleFlags)
-    c.nodes.add c.rootNode
-    c.processedModules.incl p.modname
-    parseDeps c, p, c.rootNode
-    c
-
-  var c = initDepContext(false)
+  var c = initDepContext(config, project, nifler, false, forceRebuild, moduleFlags, cmd)
   generateCachedConfigFile c, passC, passL
   let buildFilename = generateFrontendBuildFile(c, commandLineArgs)
   #echo "run with: nifmake run ", buildFilename
@@ -519,7 +567,7 @@ proc buildGraph*(config: sink NifConfig; project: string; forceRebuild, silentMa
   # Parse `.2.deps.nif`.
   # It is generated by nimsem and doesn't contains modules imported under `when false:`.
   # https://github.com/nim-lang/nimony/issues/985
-  c = initDepContext(true)
+  c = initDepContext(config, project, nifler, true, forceRebuild, moduleFlags, cmd)
   let buildFinalFilename = generateFinalBuildFile(c, commandLineArgsNifc, passC, passL)
   exec nifmakeCommand & quoteShell(buildFinalFilename)
   if cmd == DoRun:

src/nimony/programs.nim

@@ -5,8 +5,9 @@
 # distribution, for details about the copyright.
 
 import std / [syncio, os, tables, sequtils, times, sets]
-include nifprelude
-import nifindexes, symparser, reporters, builtintypes
+include ".." / lib / nifprelude
+import ".." / lib / [nifindexes, symparser]
+import reporters, builtintypes
 import ".." / models / [nifindex_tags]
 
 type

src/nimony/reporters.nim

@@ -5,7 +5,7 @@
 # distribution, for details about the copyright.
 
 import std / [syncio, strutils, os, terminal, assertions, sets]
-import nifstreams, nifcursors, bitabs, lineinfos
+import ".." / lib / [nifstreams, nifcursors, bitabs, lineinfos]
 
 type
   MsgKind* = enum

src/nimony/sem.nim

@@ -4592,6 +4592,32 @@ proc buildIndexExports(c: var SemContext): TokenBuf =
         result.add identToken(s, NoLineInfo)
       result.addParRi()
 
+proc writeNewDepsFile(c: var SemContext; outfile: string) =
+  # Update .2.deps.nif file that doesn't contain modules imported under `when false:`
+  # so that Hexer and following phases doesn't read such modules.
+  var deps = createTokenBuf(16)
+  deps.buildTree StmtsS, NoLineInfo:
+    if c.importedModules.len != 0:
+      var usedPlugins = false
+      deps.buildTree ImportS, NoLineInfo:
+        for _, i in c.importedModules:
+          if not i.fromPlugin:
+            deps.addStrLit i.path.toAbsolutePath
+          else:
+            usedPlugins = true
+      if usedPlugins:
+        for _, i in c.importedModules:
+          if i.fromPlugin:
+            deps.buildTree ImportS, NoLineInfo:
+              deps.buildTree PragmaxX, NoLineInfo:
+                deps.addStrLit i.path.toAbsolutePath
+                deps.buildTree PragmasS, NoLineInfo:
+                  deps.buildTree KvU, NoLineInfo:
+                    deps.addIdent "plugin"
+                    deps.addStrLit "unknown"
+  let depsFile = changeFileExt(outfile, ".deps.nif")
+  writeFile depsFile, "(.nif24)\n" & toString(deps)
+
 proc writeOutput(c: var SemContext; outfile: string) =
   #var b = nifbuilder.open(outfile)
   #b.addHeader "nimony", "nim-sem"
@@ -4605,17 +4631,7 @@ proc writeOutput(c: var SemContext; outfile: string) =
       classes: move c.classIndexMap,
       toBuild: move c.toBuild,
       exportBuf: buildIndexExports(c))
-
-  # Update .2.deps.nif file that doesn't contain modules imported under `when false:`
-  # so that Hexer and following phases doesn't read such modules.
-  var deps = createTokenBuf(16)
-  deps.buildTree StmtsS, NoLineInfo:
-    if c.importedModules.len != 0:
-      deps.buildTree ImportS, NoLineInfo:
-        for _, i in c.importedModules:
-          deps.addStrLit i.path.toAbsolutePath
-  let depsFile = changeFileExt(outfile, ".deps.nif")
-  writeFile depsFile, "(.nif24)\n" & toString(deps)
+  writeNewDepsFile c, outfile
 
 proc phaseX(c: var SemContext; n: Cursor; x: SemPhase): TokenBuf =
   assert n.stmtKind == StmtsS

src/nimony/semdata.nim

@@ -34,6 +34,7 @@ type
     path*: string
     iface*: Iface
     exports*: Table[SymId, ImportFilter]
+    fromPlugin*: bool
 
   InstRequest* = object
     origin*: SymId

src/nimony/semimport.nim

@@ -51,8 +51,9 @@ proc importSingleFile(c: var SemContext; f1: ImportedFilename; origin: string;
   result = SymId(0)
   if not c.processedModules.contains(suffix):
     c.meta.importedFiles.add f2
-    if (c.canSelfExec or c.inWhen > 0) and needsRecompile(f2, suffixToNif suffix):
-      selfExec c, f2, (if f1.isSystem: " --isSystem" else: "")
+    if f1.plugin.len == 0:
+      if (c.canSelfExec or c.inWhen > 0) and needsRecompile(f2, suffixToNif suffix):
+        selfExec c, f2, (if f1.isSystem: " --isSystem" else: "")
 
     let moduleName = pool.strings.getOrIncl(f1.name)
     result = identToSym(c, moduleName, ModuleY)
@@ -66,7 +67,7 @@ proc importSingleFile(c: var SemContext; f1: ImportedFilename; origin: string;
     publish result, moduleDecl
   else:
     result = c.processedModules[suffix]
-  let module = addr c.importedModules.mgetOrPut(result, ImportedModule(path: f2))
+  let module = addr c.importedModules.mgetOrPut(result, ImportedModule(path: f2, fromPlugin: f1.plugin.len != 0))
   loadInterface suffix, module.iface, result, c.importTab, c.converters, c.methods, exports, mode
 
 proc importSingleFile(c: var SemContext; f1: ImportedFilename; origin: string;
@@ -99,8 +100,8 @@ proc doImports(c: var SemContext; files: seq[ImportedFilename]; mode: ImportFilt
 
 template maybeCyclic(c: var SemContext; x: var Cursor) =
   if x.kind == ParLe and x.exprKind == PragmaxX:
-    inc x
     var y = x
+    inc y
     skip y
     if y.substructureKind == PragmasU:
       inc y