Normalization (#1443)

* Normalizes requirements

* normalize should also take alias into account

* progress

* Use original requirements for installing packages

* Internally normalizes deps

* [OK] should ignore features specified in `requires` when using the vmparser

* adds urls to the packageToDependency table

* removes comment

* Stores the package version table in SATResult. Uses it to do a lookup fallback

* temporary comments an "easy" to fix test

* [OK] cannot remove package with develop reverse dependency

* [OK] can uninstall

* [OK] Forge alias is generated inside lockfile

* Removes normalizedRequirements lookup table

* Revert "Removes normalizedRequirements lookup table"

* Reenables last test

* Replaces direct comparisons with `cmpIgnoreCase`

Author: jmgomez

src/nimble.nim

@@ -2122,10 +2122,14 @@ proc lock(options: var Options) =
           vcsRevision = getVcsRevision(pkgInfo.getRealDir())
         except CatchableError:
           discard
+      var lockUrl = pkgInfo.metaData.url
+      if lockUrl == "":
+        lockUrl = nimblesat.getUrlFromPkgName(solvedPkg.pkgName, options.satResult.pkgVersionTable, options)
+
       lockDeps[noTask][pkgInfo.basicInfo.name] = LockFileDep(
         version: solvedPkg.version,
         vcsRevision: vcsRevision,
-        url: pkgInfo.metaData.url,
+        url: lockUrl,
         downloadMethod: pkgInfo.metaData.downloadMethod,
         dependencies: solvedPkg.requirements.mapIt(it.name), 
         checksums: Checksums(sha1: pkgInfo.basicInfo.checksum))
@@ -2662,6 +2666,15 @@ proc runVNext*(options: var Options) =
       options.satResult = initSATResult(satNimSelection)      
       var rootPackage = downloadPkInfoForPv(pkg, options, doPrompt = true)
       solvePkgs(rootPackage, options)
+    
+      let rootSolvedPkg = SolvedPackage(
+        pkgName: rootPackage.basicInfo.name,
+        version: rootPackage.basicInfo.version,
+        requirements: rootPackage.requires,
+        deps: options.satResult.solvedPkgs.filterIt(it.pkgName.toLower != rootPackage.basicInfo.name.toLower)  
+      )
+      options.satResult.solvedPkgs.add(rootSolvedPkg)
+      
   # echo "BEFORE INSTALL PKGS"
   # options.debugSATResult()
   options.satResult.installPkgs(options)

src/nimblepkg/declarativeparser.nim

@@ -469,4 +469,5 @@ proc getPkgInfoFromDirWithDeclarativeParser*(dir: string, options: Options): Pac
   let nimbleFileInfo = extractRequiresInfo(nimbleFile)
   result = initPackageInfo()
   fillPkgBasicInfo(result, nimbleFileInfo)
+  result.metadata = loadMetaData(result.getNimbleFileDir(), raiseIfNotFound = false, options)
   result = toRequiresInfo(result, options, some nimbleFileInfo)

src/nimblepkg/download.nim

@@ -7,7 +7,7 @@ import parseutils, os, osproc, strutils, tables, uri, strformat,
 from algorithm import SortOrder, sorted
 
 import packageinfotypes, packageparser, version, tools, common, options, cli,
-       sha1hashes, vcstools, displaymessages, packageinfo, config, declarativeparser
+       sha1hashes, vcstools, displaymessages, packageinfo, config, declarativeparser, packagemetadatafile
 
 type
   DownloadPkgResult* = tuple
@@ -523,6 +523,11 @@ proc downloadPkg*(url: string, verRange: VersionRange,
   (result.version, result.vcsRevision) = doDownload(
     modUrl, downloadDir, verRange, downMethod, options, vcsRevision)
 
+  var metaData = initPackageMetaData()
+  metaData.url = modUrl
+  metaData.vcsRevision = result.vcsRevision
+  saveMetaData(metaData, result.dir)
+
   var pkgInfo: PackageInfo
   if validateRange and verRange.kind notin {verSpecial, verAny} or not options.isLegacy:
     ## Makes sure that the downloaded package's version satisfies the requested

src/nimblepkg/nimblesat.nim

@@ -116,6 +116,17 @@ type
     pkg*: Option[PackageInfo] #when none, we need to install it
     version*: Version
 
+  PackageMinimalInfo* = object
+    name*: string
+    version*: Version
+    requires*: seq[PkgTuple]
+    isRoot*: bool
+    url*: string
+
+  PackageVersions* = object
+    pkgName*: string
+    versions*: seq[PackageMinimalInfo]
+    
   SATResult* = ref object
     rootPackage*: PackageInfo 
     pkgsToInstall*: seq[(string, Version)] #Packages to install
@@ -129,6 +140,8 @@ type
     declarativeParseFailed*: bool
     declarativeParserErrorLines*: seq[string]
     nimResolved*: NimResolved
+    normalizedRequirements*: Table[string, string] #normalized -> old. Some packages are not published as nimble packages, we keep the url for installation.
+    pkgVersionTable*: Table[string, PackageVersions]
 
 proc `==`*(a, b: SolvedPackage): bool =
   a.pkgName == b.pkgName and
@@ -157,4 +170,7 @@ proc getGloballyActiveFeatures*(): seq[string] =
       result.add(&"features.{pkgName}.{feature}")
 
 proc initSATResult*(pass: SATPass): SATResult =
-  SATResult(pkgsToInstall: @[], solvedPkgs: @[], output: "", pkgs: initHashSet[PackageInfo](), pass: pass, installedPkgs: @[], declarativeParseFailed: false, declarativeParserErrorLines: @[])
+  SATResult(pkgsToInstall: @[], solvedPkgs: @[], output: "", pkgs: initHashSet[PackageInfo](), 
+    pass: pass, installedPkgs: @[], declarativeParseFailed: false, declarativeParserErrorLines: @[],
+    normalizedRequirements: initTable[string, string]()
+    )

src/nimblepkg/packageinfotypes.nim

@@ -19,46 +19,48 @@ import nimblesat, packageinfotypes, options, version, declarativeparser, package
 
 proc debugSATResult*(options: Options) =
   # return
+  let satResult = options.satResult
+  let color = "\e[32m"
+  let reset = "\e[0m"
   echo "=== DEBUG SAT RESULT ==="
   echo "Called from: ", getStackTrace()[^2]
-  let satResult = options.satResult
   echo "--------------------------------"
-  echo "Pass: ", satResult.pass
+  echo color, "Pass: ", reset, satResult.pass
   if satResult.nimResolved.pkg.isSome:
-    echo "Selected Nim: ", satResult.nimResolved.pkg.get.basicInfo.name, " ", satResult.nimResolved.version
+    echo color, "Selected Nim: ", reset, satResult.nimResolved.pkg.get.basicInfo.name, " ", satResult.nimResolved.version
   else:
     echo "No Nim selected"
-  echo "Declarative parser failed: ", satResult.declarativeParseFailed
+  echo color, "Declarative parser failed: ", reset, satResult.declarativeParseFailed
   if satResult.declarativeParseFailed:
-    echo "Declarative parser error lines: ", satResult.declarativeParserErrorLines
+    echo color, "Declarative parser error lines: ", reset, satResult.declarativeParserErrorLines
 
   if satResult.rootPackage.hasLockFile(options):
     echo "Root package has lock file: ", satResult.rootPackage.myPath.parentDir() / "nimble.lock"
   else:
     echo "Root package does not have lock file"
-  echo "Root package: ", satResult.rootPackage.basicInfo.name, " ", satResult.rootPackage.basicInfo.version, " ", satResult.rootPackage.myPath
-  echo "Root requires: ", satResult.rootPackage.requires.mapIt(it.name & " " & $it.ver)
-  echo "Solved packages: ", satResult.solvedPkgs.mapIt(it.pkgName & " " & $it.version & " " & $it.deps.mapIt(it.pkgName))
-  echo "Solution as Packages Info: ", satResult.pkgs.mapIt(it.basicInfo.name & " " & $it.basicInfo.version)
+  echo color, "Root package: ", reset, satResult.rootPackage.basicInfo.name, " ", satResult.rootPackage.basicInfo.version, " ", satResult.rootPackage.myPath
+  echo color, "Root requires: ", reset, satResult.rootPackage.requires.mapIt(it.name & " " & $it.ver)
+  echo color, "Solved packages: ", reset, satResult.solvedPkgs.mapIt(it.pkgName & " " & $it.version & " " & $it.deps.mapIt(it.pkgName))
+  echo color, "Solution as Packages Info: ", reset, satResult.pkgs.mapIt(it.basicInfo.name & " " & $it.basicInfo.version)
   if options.action.typ == actionUpgrade:
-    echo "Upgrade versions: ", options.action.packages.mapIt(it.name & " " & $it.ver)
-    echo "RESULT REVISIONS ", satResult.pkgs.mapIt(it.basicInfo.name & " " & $it.metaData.vcsRevision)
-    echo "PKG LIST REVISIONS ", satResult.pkgList.mapIt(it.basicInfo.name & " " & $it.metaData.vcsRevision)
-  echo "Packages to install: ", satResult.pkgsToInstall
-  echo "Installed pkgs: ", satResult.pkgs.mapIt(it.basicInfo.name)
-  echo "Build pkgs: ", satResult.buildPkgs.mapIt(it.basicInfo.name)
-  echo "Packages url: ", satResult.pkgs.mapIt(it.metaData.url)
-  echo "Package list: ", satResult.pkgList.mapIt(it.basicInfo.name)
-  echo "PkgList path: ", satResult.pkgList.mapIt(it.myPath.parentDir)
-  echo "Nimbledir: ", options.getNimbleDir()
-  echo "Nimble Action: ", options.action.typ
+    echo color, "Upgrade versions: ", reset, options.action.packages.mapIt(it.name & " " & $it.ver)
+    echo color, "RESULT REVISIONS ", reset, satResult.pkgs.mapIt(it.basicInfo.name & " " & $it.metaData.vcsRevision)
+    echo color, "PKG LIST REVISIONS ", reset, satResult.pkgList.mapIt(it.basicInfo.name & " " & $it.metaData.vcsRevision)
+  echo color, "Packages to install: ", reset, satResult.pkgsToInstall
+  echo color, "Installed pkgs: ", reset, satResult.pkgs.mapIt(it.basicInfo.name)
+  echo color, "Build pkgs: ", reset, satResult.buildPkgs.mapIt(it.basicInfo.name)
+  echo color, "Packages url: ", reset, satResult.pkgs.mapIt(it.metaData.url)
+  echo color, "Package list: ", reset, satResult.pkgList.mapIt(it.basicInfo.name)
+  echo color, "PkgList path: ", reset, satResult.pkgList.mapIt(it.myPath.parentDir)
+  echo color, "Nimbledir: ", reset, options.getNimbleDir()
+  echo color, "Nimble Action: ", reset, options.action.typ
   if options.action.typ == actionDevelop:
-    echo "Path: ", options.action.packages.mapIt(it.name)
-    echo "Dev actions: ", options.action.devActions.mapIt(it.actionType)
-    echo "Dependencies: ", options.action.packages.mapIt(it.name)
+    echo color, "Path: ", reset, options.action.packages.mapIt(it.name)
+    echo color, "Dev actions: ", reset, options.action.devActions.mapIt(it.actionType)
+    echo color, "Dependencies: ", reset, options.action.packages.mapIt(it.name)
     for devAction in options.action.devActions:
-      echo "Dev action: ", devAction.actionType
-      echo "Argument: ", devAction.argument
+      echo color, "Dev action: ", reset, devAction.actionType
+      echo color, "Argument: ", reset, devAction.argument
   echo "--------------------------------"
 
 proc nameMatches(pkg: PackageInfo, pv: PkgTuple, options: Options): bool =
@@ -890,7 +892,23 @@ proc installPkgs*(satResult: var SATResult, options: Options) =
         installedPkgInfo = installFromDirDownloadInfo(satResult.rootPackage.getNimbleFileDir(), satResult.rootPackage.metaData.url, pv, options).toRequiresInfo(options)
         wasNewlyInstalled = oldPkg.isNone
     else:      
-      var dlInfo = getPackageDownloadInfo(pv, options, doPrompt = true)
+      # echo "NORMALIZING REQUIREMENT: ", pv.name
+      # echo "ROOT PACKAGE: ", satResult.rootPackage.basicInfo.name, " ", $satResult.rootPackage.basicInfo.version, " ", satResult.rootPackage.metaData.url
+      # options.debugSATResult()
+      if pv.name in options.satResult.normalizedRequirements:
+        pv.name = options.satResult.normalizedRequirements[pv.name]    
+      
+      var dlInfo: PackageDownloadInfo
+      try:
+        dlInfo = getPackageDownloadInfo(pv, options, doPrompt = true)
+      except CatchableError as e:
+        #if we fail, we try to find the url for the req:
+        let url = getUrlFromPkgName(pv.name, options.satResult.pkgVersionTable, options)
+        if url != "":
+          pv.name = url
+          dlInfo = getPackageDownloadInfo(pv, options, doPrompt = true)
+        else:
+          raise e
       var downloadDir = dlInfo.downloadDir / dlInfo.subdir       
       if not dirExists(dlInfo.downloadDir):        
         #The reason for this is that the download cache may have a constrained version

src/nimblepkg/vnext.nim

@@ -52,14 +52,33 @@ proc downloadAllPackages() {.used.} =
   let ignorePackages = ["rpgsheet", 
   "arturo", "argument_parser", "murmur", "nimgame", "locale", "nim-locale",
   "nim-ao", "ao", "termbox", "linagl", "kwin", "yahooweather", "noaa",
-  "nimwc",
+  "nimwc", "pylib",
   "artemis"]
-  let toDownload = importantPackages.filterIt(it notin ignorePackages)
-  for pkg in toDownload:
+  let startAt = 0#importantPackages.find("rbtree")
+  let toDownload = importantPackages
+  for i, pkg in toDownload:
+    if i >= startAt or pkg in ignorePackages:
+      continue
     echo "Downloading ", pkg
     downloadAndStorePackageVersionTableFor(pkg, options)
     echo "Done with ", pkg
 
+proc fromJsonHook(pv: var PkgTuple, jsonNode: JsonNode, opt = Joptions()) =
+  if jsonNode.kind == Jstring:
+    pv = parseRequires(jsonNode.getStr())
+  else:
+    raise newException(ValueError, "Expected a string for PkgTuple found: " & $jsonNode.kind & " val: " & $jsonNode)
+
+# proc fromJsonHook(pm: var PackageMinimalInfo, jsonNode: JsonNode, opt = Joptions()) =
+#   pm.name = jsonNode["name"].getStr().toLower
+#   pm.version = newVersion(jsonNode["version"].getStr())
+#   for req in jsonNode["requires"]:
+#     var pv: PkgTuple
+#     fromJson(pv, req)
+#     pm.requires.add((name: pv.name, ver: pv.ver))
+#   pm.isRoot = jsonNode["isRoot"].getBool()
+
+
 suite "SAT solver":
   test "can solve simple SAT":
     let pkgVersionTable = {
@@ -181,7 +200,8 @@ suite "SAT solver":
     var pks = 0
     for jsonFile in walkPattern("packageMinimal/*.json"):
       inc pks
-      var pkgVersionTable = parseJson(readFile(jsonFile)).to(Table[string, PackageVersions])
+      var pkgVersionTable = parseJson(readFile(jsonFile)).jsonTo(Table[string, PackageVersions], Joptions(allowMissingKeys: true))
+      pkgVersionTable.normalizeRequirements(initOptions())
       var graph = pkgVersionTable.toDepGraph()
       let form = toFormular(graph)
       var packages = initTable[string, Version]()