First iteration of the features feature for the declarative parser (#1346)

* First iteration of the `features` feature for the declarative parser

There are a few things left to implement. Like adding a special syntax to requires.
This PR handles the scenarios for root packages.
See the `tdeclarativeparser` tests over the `features` packages.
Nimble file
```nim
# Dependencies

requires "nim >= 2.3.1"

feature "feature1":
  requires "stew"

```nim

when defined(features.features.feature1):
  echo "feature1 is enabled"
  import stew/byteutils #we should be able to import stew here as is its part of the feature1

else:
  echo "feature1 is disabled"
```

* dont require `nim` devel in test

* Removes log

Author: jmgomez

src/nimble.nim

@@ -87,6 +87,14 @@ proc processFreeDependenciesSAT(rootPkgInfo: PackageInfo, options: Options): Has
   var pkgsToInstall: seq[(string, Version)] = @[]
   var rootPkgInfo = rootPkgInfo
   rootPkgInfo.requires &= options.extraRequires
+  if options.useDeclarativeParser:
+    rootPkgInfo = rootPkgInfo.toRequiresInfo(options)
+    # displayInfo(&"Features: options: {options.features} pkg: {rootPkgInfo.features}", HighPriority)
+    for feature in options.features:
+      if feature in rootPkgInfo.features:
+        rootPkgInfo.requires &= rootPkgInfo.features[feature]
+        # displayInfo(&"Feature {feature} activated", LowPriority)
+    
   var pkgList = initPkgList(rootPkgInfo, options)
   if options.useDeclarativeParser:
     pkgList = pkgList.mapIt(it.toRequiresInfo(options))
@@ -290,6 +298,10 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[seq[string]],
     # Disable coloured output
     args.add("--colors:off")
 
+  for feature in options.features:
+    let featureStr = &"features.{pkgInfo.basicInfo.name}.{feature}"
+    args.add &"-d:{featureStr}"
+
   let binToBuild =
     # Only build binaries specified by user if any, but only if top-level package,
     # dependencies should have every binary built.
@@ -1802,7 +1814,8 @@ proc validateParsedDependencies(pkgInfo: PackageInfo, options: Options) =
 
   options.useDeclarativeParser = false
   let vmDeps = pkgInfo.toFullInfo(options).requires
-
+  displayInfo(&"Parsed declarative dependencies: {declDeps}", HighPriority)
+  displayInfo(&"Parsed VM dependencies: {vmDeps}", HighPriority)
   if declDeps != vmDeps:
     raise nimbleError(&"Parsed declarative and VM dependencies are not the same: {declDeps} != {vmDeps}")
 

src/nimblepkg/declarativeparser.nim

@@ -5,18 +5,31 @@ import std/strutils
 
 import compiler/[ast, idents, msgs, syntaxes, options, pathutils, lineinfos]
 import version, packageinfotypes, packageinfo, options, packageparser
+import std/[tables, sequtils]
 
 type NimbleFileInfo* = object
   requires*: seq[string]
   srcDir*: string
   version*: string
   tasks*: seq[(string, string)]
+  features*: Table[string, seq[string]]
   hasInstallHooks*: bool
   hasErrors*: bool
 
 proc eqIdent(a, b: string): bool {.inline.} =
   cmpIgnoreCase(a, b) == 0 and a[0] == b[0]
 
+proc extractRequires(n: PNode, conf: ConfigRef, result: var seq[string], hasErrors: var bool) =
+  for i in 1 ..< n.len:
+    var ch: PNode = n[i]
+    while ch.kind in {nkStmtListExpr, nkStmtList} and ch.len > 0:
+      ch = ch.lastSon
+    if ch.kind in {nkStrLit .. nkTripleStrLit}:
+      result.add ch.strVal
+    else:
+      localError(conf, ch.info, "'requires' takes string literals")
+      hasErrors = true
+
 proc extract(n: PNode, conf: ConfigRef, result: var NimbleFileInfo) =
   case n.kind
   of nkStmtList, nkStmtListExpr:
@@ -26,15 +39,19 @@ proc extract(n: PNode, conf: ConfigRef, result: var NimbleFileInfo) =
     if n[0].kind == nkIdent:
       case n[0].ident.s
       of "requires":
-        for i in 1 ..< n.len:
-          var ch = n[i]
-          while ch.kind in {nkStmtListExpr, nkStmtList} and ch.len > 0:
-            ch = ch.lastSon
-          if ch.kind in {nkStrLit .. nkTripleStrLit}:
-            result.requires.add ch.strVal
-          else:
-            localError(conf, ch.info, "'requires' takes string literals")
-            result.hasErrors = true
+        extractRequires(n, conf, result.requires, result.hasErrors)
+      of "feature":
+        if n.len >= 3 and n[1].kind in {nkStrLit .. nkTripleStrLit}:
+          let featureName = n[1].strVal
+          if not result.features.hasKey(featureName):
+            result.features[featureName] = @[]
+          if n[2].kind in {nkStmtList, nkStmtListExpr}:
+            for stmt in n[2]:
+              if stmt.kind in nkCallKinds and stmt[0].kind == nkIdent and 
+                 stmt[0].ident.s == "requires":
+                var requires: seq[string]
+                extractRequires(stmt, conf, requires, result.hasErrors)
+                result.features[featureName].add requires
       of "task":
         if n.len >= 3 and n[1].kind == nkIdent and
             n[2].kind in {nkStrLit .. nkTripleStrLit}:
@@ -170,16 +187,23 @@ proc getRequires*(nimbleFileInfo: NimbleFileInfo): seq[PkgTuple] =
   for require in nimbleFileInfo.requires:
     result.add(parseRequires(require))
 
+proc getFeatures*(nimbleFileInfo: NimbleFileInfo): Table[string, seq[PkgTuple]] =
+  result = initTable[string, seq[PkgTuple]]()
+  for feature, requires in nimbleFileInfo.features:
+    result[feature] = requires.map(parseRequires)    
+
 proc toRequiresInfo*(pkgInfo: PackageInfo, options: Options): PackageInfo =
   #For nim we only need the version. Since version is usually in the form of `version = $NimMajor & "." & $NimMinor & "." & $NimPatch
   #we need to use the vm to get the version. Another option could be to use the binary and ask for the version
   if pkgInfo.basicInfo.name.isNim:
     return pkgInfo.toFullInfo(options)
-    
+        
   let nimbleFileInfo = extractRequiresInfo(pkgInfo.myPath)
   result = pkgInfo
   result.requires = getRequires(nimbleFileInfo)
-  result.infoKind = pikRequires
+  if pkgInfo.infoKind != pikFull: #dont update as full implies pik requires
+    result.infoKind = pikRequires
+  result.features = getFeatures(nimbleFileInfo)
 
 when isMainModule:
   for x in tokenizeRequires("jester@#head >= 1.5 & <= 1.8"):

src/nimblepkg/nimscriptapi.nim

@@ -255,3 +255,6 @@ proc getPaths*(): seq[string] =
 proc getPathsClause*(): string =
   ## Returns the paths to the dependencies as consumed by the nim compiler.
   return getPaths().mapIt("--path:" & it).join(" ")
+
+template feature*(name: string, body: untyped): untyped =
+  discard

src/nimblepkg/options.nim

@@ -65,6 +65,7 @@ type
     disableNimBinaries*: bool # Whether to disable the use of nim binaries
     maxTaggedVersions*: int # Maximum number of tags to check for a package when discovering versions in a local repo
     useDeclarativeParser*: bool # Whether to use the declarative parser for parsing nimble files (only when solver is SAT)
+    features*: seq[string] # Features to be activated. Only used when using the declarative parser
 
   ActionType* = enum
     actionNil, actionRefresh, actionInit, actionDump, actionPublish, actionUpgrade
@@ -278,6 +279,7 @@ Nimble Options:
       --disableNimBinaries        Disable the use of nim precompiled binaries. Note in some platforms precompiled binaries are not available but the flag can still be used to avoid compile the Nim version once and reuse it.
       --maximumTaggedVersions     Maximum number of tags to check for a package when discovering versions for the SAT solver. 0 means all.
       --parser:declarative|nimvm  Use the declarative parser or the nimvm parser (default).
+      --features                  Activate features. Only used when using the declarative parser.
 For more information read the GitHub readme:
   https://github.com/nim-lang/nimble#readme
 """
@@ -680,6 +682,8 @@ proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) =
       result.maxTaggedVersions = parseUInt(val).int
     except ValueError:
       raise nimbleError(&"{val} is not a valid value")
+  of "features":
+    result.features = val.split(";").mapIt(it.strip)
   else: isGlobalFlag = false
 
   var wasFlagHandled = true

src/nimblepkg/packageinfotypes.nim

@@ -77,6 +77,7 @@ type
     isLink*: bool
     paths*: seq[string] 
     entryPoints*: seq[string] #useful for tools like the lsp.
+    features*: Table[string, seq[PkgTuple]] #features requires defined in the nimble file. Declarative parser + SAT solver only
 
   Package* = object ## Definition of package from packages.json.
     # Required fields in a package.

tests/features/features.nimble

@@ -0,0 +1,15 @@
+# Package
+
+version       = "0.1.0"
+author        = "jmgomez"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+bin = @["features"]
+
+# Dependencies
+
+requires "nim"
+
+feature "feature1":
+  requires "stew"

tests/features/src/features.nim

@@ -0,0 +1,18 @@
+# This is just an example to get you started. A typical library package
+# exports the main API in this file. Note that you cannot rename this file
+# but you can remove it if you wish.
+
+proc add*(x, y: int): int =
+  ## Adds two numbers together.
+  return x + y
+
+
+
+when defined(features.features.feature1):
+  echo "feature1 is enabled"
+  import stew/byteutils #we should be able to import stew here as is its part of the feature1
+  
+else:
+  echo "feature1 is disabled"
+
+echo ""

tests/features/src/features/submodule.nim

@@ -0,0 +1,12 @@
+# This is just an example to get you started. Users of your library will
+# import this file by writing ``import features/submodule``. Feel free to rename or
+# remove this file altogether. You may create additional modules alongside
+# this file as required.
+
+type
+  Submodule* = object
+    name*: string
+
+proc initSubmodule*(): Submodule =
+  ## Initialises a new ``Submodule`` object.
+  Submodule(name: "Anonymous")

tests/features/tests/test1.nim

@@ -0,0 +1,12 @@
+# This is just an example to get you started. You may wish to put all of your
+# tests into a single file, or separate them into multiple `test1`, `test2`
+# etc. files (better names are recommended, just make sure the name starts with
+# the letter 't').
+#
+# To run these tests, simply execute `nimble test`.
+
+import unittest
+
+import features
+test "can add":
+  check add(5, 5) == 10

tests/tdeclarativeparser.nim

@@ -3,7 +3,7 @@ import unittest
 import testscommon
 import std/[options, tables, sequtils, os]
 import
-  nimblepkg/[packageinfotypes, version, options, config, nimblesat, declarativeparser, cli]
+  nimblepkg/[packageinfotypes, version, options, config, nimblesat, declarativeparser, cli, common]
 
 proc getNimbleFileFromPkgNameHelper(pkgName: string): string =
   let pv: PkgTuple = (pkgName, VersionRange(kind: verAny))
@@ -72,4 +72,35 @@ suite "Declarative parsing":
     check exitCode == QuitSuccess
 
 
-  # suite "Declarative parser features":
+suite "Declarative parser features":
+  test "should be able to parse features from a nimble file":
+    let nimbleFile =  "./features/features.nimble"
+    let nimbleFileInfo = extractRequiresInfo(nimbleFile)
+    let features = nimbleFileInfo.features
+    check features.len == 1
+    check features["feature1"] == @["stew"]
+
+  test "should be able to install a package using the declarative parser with a feature":
+    cd "features":
+      #notice it imports stew, which will fail to compile if feature1 is not activated although it only imports it in the when part
+      let (output, exitCode) = execNimble("--parser:declarative", "--features:feature1", "run")      
+      check exitCode == QuitSuccess
+      check output.processOutput.inLines("feature1 is enabled")
+
+  test "should not enable features if not specified":
+    cd "features":
+      let (output, exitCode) = execNimble("run")
+      check exitCode == QuitSuccess
+      check output.processOutput.inLines("feature1 is disabled")
+
+
+  #[NEXT Tests:
+
+    TODO:
+    - compile time nimble parser detection so we can warn when using the vm parser with features
+    - add enable features to nimble.paths
+    - dependencies syntax i.e. requires "stew > 0.3.4[feature1; feature2]"
+
+
+]#
+