Improves nim installation by using csources (same as atlas) (#1233)

* draft: improves nim installation by using csources (same as atlas)

Notice when SAT is enabled, csources is stored in the cache.
Missing:
-- Proper reporting
-- Handle special versions (assume csources_2?). We could have special cases for #version-x
- Refactor to another file (nimble.nim is crazy big already)

* progress

* refactor: extract file nimenv

* Fixes an issue where the packageCache wasnt being shared in local mode

* [Green]Should be able to install different Nim versions

* fix warnings

* missing display

* fix typo

* Remove nimble from nim compilation Fixes #1175

Author: jmgomez

src/nimble.nim

@@ -21,7 +21,7 @@ import nimblepkg/packageinfotypes, nimblepkg/packageinfo, nimblepkg/version,
        nimblepkg/nimscriptwrapper, nimblepkg/developfile, nimblepkg/paths,
        nimblepkg/nimbledatafile, nimblepkg/packagemetadatafile,
        nimblepkg/displaymessages, nimblepkg/sha1hashes, nimblepkg/syncfile,
-       nimblepkg/deps, nimblepkg/nimblesat, nimblepkg/forge_aliases
+       nimblepkg/deps, nimblepkg/nimblesat, nimblepkg/forge_aliases, nimblepkg/nimenv
 
 const
   nimblePathsFileName* = "nimble.paths"
@@ -754,35 +754,6 @@ proc processLockedDependencies(pkgInfo: PackageInfo, options: Options):
 
   return res.toHashSet
 
-proc compileNim(realDir: string) =
-  let command = when defined(windows): "build_all.bat" else: "./build_all.sh"
-  cd realDir:
-    display("Info:", "compiling nim in $1" % realDir, priority = HighPriority)
-    tryDoCmdEx(command)
-
-proc useNimFromDir(options: var Options, realDir: string, tryCompiling = false) =
-  const binaryName = when defined(windows): "nim.exe" else: "nim"
-
-  let
-    nim = realDir / "bin" / binaryName
-    fileExists = fileExists(options.nimBin)
-
-  if not fileExists(nim):
-    if tryCompiling and options.prompt("Develop version of nim was found but it is not compiled. Compile it now?"):
-      compileNim(realDir)
-    else:
-      raise nimbleError("Trying to use nim from $1 " % realDir,
-                        "If you are using develop mode nim make sure to compile it.")
-
-  options.nimBin = nim
-  let separator = when defined(windows): ";" else: ":"
-
-  putEnv("PATH", realDir / "bin" & separator & getEnv("PATH"))
-  if fileExists:
-    display("Info:", "switching to $1 for compilation" % options.nim, priority = HighPriority)
-  else:
-    display("Info:", "using $1 for compilation" % options.nim, priority = HighPriority)
-
 proc install(packages: seq[PkgTuple], options: Options,
              doPrompt, first, fromLockFile: bool,
              preferredPackages: seq[PackageInfo] = @[]): PackageDependenciesInfo =
@@ -817,6 +788,9 @@ proc install(packages: seq[PkgTuple], options: Options,
       var downloadPath = ""
       if options.useSatSolver and subdir == "": #Ignore the cache if subdir is set
           downloadPath =  getCacheDownloadDir(url, pv.ver, options)
+      # if pv.name.isNim:
+      #   downloadPath = "/Volumes/Store/Projects/nim/nimble/temptest/nimtest"
+
 
       let (downloadDir, downloadVersion, vcsRevision) =
         if not isAlias:
@@ -831,8 +805,8 @@ proc install(packages: seq[PkgTuple], options: Options,
       try:
         var opt = options
         if pv.name.isNim:
-          compileNim(downloadDir)
-          opt.useNimFromDir(downloadDir, true)
+          compileNim(opt, downloadDir, pv.ver)
+          opt.useNimFromDir(downloadDir, pv.ver, true)
         result = installFromDir(downloadDir, pv.ver, opt, url,
                                 first, fromLockFile, vcsRevision,
                                 preferredPackages = preferredPackages)
@@ -2403,16 +2377,17 @@ proc setNimBin*(options: var Options) =
   if lockFile.fileExists and not options.disableLockFile and not options.useSystemNim:
     for name, dep in lockFile.getLockedDependencies.lockedDepsFor(options):
       if name.isNim:
+        let v = dep.version.toVersionRange()
         if isInstalled(name, dep, options):
-          options.useNimFromDir(getDependencyDir(name, dep, options))
+          options.useNimFromDir(getDependencyDir(name, dep, options), v)
         elif not options.offline:
           let depsOnly = options.depsOnly
           options.depsOnly = false
           let downloadResult = downloadDependency(name, dep, options, false)
-          compileNim(downloadResult.downloadDir)
-          options.useNimFromDir(downloadResult.downloadDir)
+          compileNim(options, downloadResult.downloadDir, v)
+          options.useNimFromDir(downloadResult.downloadDir, v)
           let pkgInfo = installDependency(initTable[string, LockFileDep](), downloadResult, options, @[])
-          options.useNimFromDir(pkgInfo.getRealDir)
+          options.useNimFromDir(pkgInfo.getRealDir, v)
           options.depsOnly = depsOnly
         break
 
@@ -2429,13 +2404,13 @@ proc setNimBin*(options: var Options) =
     let installedPkgs = getInstalledPkgsMin(options.getPkgsDir(), options)
     var pkg = initPackageInfo()
     if findPkg(installedPkgs, nimVersion, pkg):
-      options.useNimFromDir(pkg.getRealDir)
+      options.useNimFromDir(pkg.getRealDir, pkg.basicInfo.version.toVersionRange())
     else:
       # It still no nim found then download and install one to allow parsing of
       # other packages.
       if options.nimBin.len == 0 and not options.offline and options.prompt("No nim found. Download it now?"):
         for pkg in install(nimVersion, options):
-          options.useNimFromDir(pkg.getRealDir)
+          options.useNimFromDir(pkg.getRealDir, pkg.basicInfo.version.toVersionRange())
 
   if options.nimBin.len == 0:
     raise nimbleError("Unable to find nim")
@@ -2446,7 +2421,7 @@ proc setNimBin*(options: var Options) =
     pkgInfo = getPkgInfo(getCurrentDir(), options)
     for pkg in pkgInfo.processDevelopDependencies(options):
       if pkg.name.isNim:
-        options.useNimFromDir(pkg.getRealDir, true)
+        options.useNimFromDir(pkg.getRealDir, pkg.basicInfo.version.toVersionRange(), true)
         return
     options.pkgInfoCache.clear()
   except NimbleError:
@@ -2464,11 +2439,11 @@ proc setNimBin*(options: var Options) =
       let installedPkgs = getInstalledPkgsMin(options.getPkgsDir(), options)
       var pkg = initPackageInfo()
       if findPkg(installedPkgs, require, pkg):
-        options.useNimFromDir(pkg.getRealDir)
+        options.useNimFromDir(pkg.getRealDir, require.ver)
       else:
         if not options.offline and options.prompt("No nim version matching $1. Download it now?" % $require.ver):
           for pkg in install(require, options):
-            options.useNimFromDir(pkg.getRealDir)
+            options.useNimFromDir(pkg.getRealDir, require.ver)
         else:
           let msg = "Unsatisfied dependency: " & require.name & " (" & $require.ver & ")"
           raise nimbleError(msg)

src/nimblepkg/nimblesat.nim

@@ -105,22 +105,22 @@ proc hasVersion*(packagesVersions: Table[string, PackageVersions], name: string,
         return true
   false
 
-proc getNimVersion*(pvs: seq[PkgTuple]): Version =
-  proc getVersion(ver: VersionRange): Version =
-    case ver.kind:
-    of verLater, verEarlier, verEqLater, verEqEarlier, verEq:
-      ver.ver
-    of verSpecial:
-      ver.spe
-    of verIntersect, verTilde, verCaret:
-      getVersion(ver.verILeft)
-    of verAny:
-      newVersion "0.0.0"
+proc getNimVersion*(ver: VersionRange): Version =
+  case ver.kind:
+  of verLater, verEarlier, verEqLater, verEqEarlier, verEq:
+    ver.ver
+  of verSpecial:
+    ver.spe
+  of verIntersect, verTilde, verCaret:
+    getNimVersion(ver.verILeft)
+  of verAny:
+    newVersion "0.0.0"
 
+proc getNimVersion*(pvs: seq[PkgTuple]): Version =
   result = newVersion("0.0.0")
   for pv in pvs:
     if pv.name == "nim":
-      result = getVersion(pv.ver)
+      result = getNimVersion(pv.ver)
 
 proc findDependencyForDep(g: DepGraph; dep: string): int {.inline.} =
   assert g.packageToDependency.hasKey(dep), dep & " not found"

src/nimblepkg/nimenv.nim

@@ -0,0 +1,110 @@
+import std/[strscans, os, strutils, strformat]
+import version, nimblesat, cli, common, options
+
+when defined(windows):
+  const
+    BatchFile = """
+  @echo off
+  set PATH="$1";%PATH%
+  """
+else:
+  const
+    ShellFile = "export PATH=$1:$$PATH\n"
+
+const ActivationFile = 
+  when defined(windows): "activate.bat" else: "activate.sh"
+
+proc infoAboutActivation(nimDest, nimVersion: string) =
+  when defined(windows):
+    display("Info", nimDest & "installed; activate with 'nim-" & nimVersion & "activate.bat'")
+  else:
+    display("Info", nimDest & "installed; activate with 'source nim-" & nimVersion & "activate.sh'")
+
+proc compileNim*(options: Options, nimDest: string, v: VersionRange) =
+  let keepCsources = options.useSatSolver #SAT Solver has a cache instead of a temp dir for downloads
+  template exec(command: string) =
+    let cmd = command # eval once
+    if os.execShellCmd(cmd) != 0:
+      display("Error", "Failed to execute: $1" % cmd, Error, HighPriority)
+      return
+  let nimVersion = v.getNimVersion()
+  let workspace = nimDest.parentDir()
+  if dirExists(workspace / nimDest):
+    if not fileExists(nimDest / ActivationFile):
+      display("Info", &"Directory {nimDest} already exists; remove or rename and try again")
+    else:
+      infoAboutActivation nimDest, $nimVersion
+    return
+
+  var major, minor, patch: int
+  if not nimVersion.isSpecial: 
+    if not scanf($nimVersion, "$i.$i.$i", major, minor, patch):
+      display("Error", "cannot parse version requirement", Error)
+      return
+  let csourcesVersion =
+    #TODO We could test special against the special versionn-x branch to get the right csources
+    if nimVersion.isSpecial or (major == 1 and minor >= 9) or major >= 2:
+      # already uses csources_v2
+      "csources_v2"
+    elif major == 0:
+      "csources" # has some chance of working
+    else:
+      "csources_v1"
+  cd workspace:
+    echo "Entering CSOURCES", csourcesVersion, " exists ", dirExists(csourcesVersion)
+    if not dirExists(csourcesVersion):
+      exec "git clone https://github.com/nim-lang/" & csourcesVersion
+
+  cd workspace / csourcesVersion:
+    when defined(windows):
+      exec "build.bat"
+    else:
+      let makeExe = findExe("make")
+      if makeExe.len == 0:
+        exec "sh build.sh"
+      else:
+        exec "make"
+  let nimExe0 = ".." / csourcesVersion / "bin" / "nim".addFileExt(ExeExt)
+  cd nimDest:
+    let nimExe = "bin" / "nim".addFileExt(ExeExt)
+    copyFileWithPermissions nimExe0, nimExe
+    exec nimExe & " c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch"
+    let kochExe = when defined(windows): "koch.exe" else: "./koch"
+    exec kochExe & " boot -d:release --skipUserCfg --skipParentCfg --hints:off"
+    exec kochExe & " tools --skipUserCfg --skipParentCfg --hints:off"
+    # unless --keep is used delete the csources because it takes up about 2GB and
+    # is not necessary afterwards:
+    if not keepCsources:
+      removeDir workspace / csourcesVersion / "c_code"
+    let pathEntry = workspace / nimDest / "bin"
+    #remove nimble so it doesnt interfer with the current one:
+    removeFile "bin" / "nimble".addFileExt(ExeExt)
+    when defined(windows):
+      writeFile "activate.bat", BatchFile % pathEntry.replace('/', '\\')
+    else:
+      writeFile "activate.sh", ShellFile % pathEntry
+    infoAboutActivation nimDest, $nimVersion
+
+
+proc useNimFromDir*(options: var Options, realDir: string, v: VersionRange, tryCompiling = false) =
+  const binaryName = when defined(windows): "nim.exe" else: "nim"
+
+  let
+    nim = realDir / "bin" / binaryName
+    fileExists = fileExists(options.nimBin)
+
+  if not fileExists(nim):
+    if tryCompiling and options.prompt("Develop version of nim was found but it is not compiled. Compile it now?"):
+      compileNim(options, realDir, v)
+    else:
+      raise nimbleError("Trying to use nim from $1 " % realDir,
+                        "If you are using develop mode nim make sure to compile it.")
+
+  options.nimBin = nim
+  let separator = when defined(windows): ";" else: ":"
+
+  putEnv("PATH", realDir / "bin" & separator & getEnv("PATH"))
+  if fileExists:
+    display("Info:", "switching to $1 for compilation" % options.nim, priority = HighPriority)
+  else:
+    display("Info:", "using $1 for compilation" % options.nim, priority = HighPriority)

src/nimblepkg/options.nim

@@ -390,6 +390,11 @@ proc getPkgsLinksDir*(options: Options): string =
 proc getBinDir*(options: Options): string =
   options.getNimbleDir() / nimbleBinariesDirName
 
+proc setPackageCache(options: var Options, baseDir: string) = 
+  if options.useSatSolver:
+    options.pkgCachePath = baseDir / "pkgcache"
+    display("Info:", "Package cache path " & options.pkgCachePath, priority = HighPriority)
+
 proc setNimbleDir*(options: var Options) =
   var
     nimbleDir = options.config.nimbleDir
@@ -407,13 +412,15 @@ proc setNimbleDir*(options: var Options) =
     # --nimbleDir:<dir> takes priority...
     nimbleDir = options.nimbleDir
     propagate = true
+    setPackageCache(options, nimbleDir)
   else:
     # ...followed by the environment variable.
     let env = getEnv("NIMBLE_DIR")
     if env.len != 0:
       display("Info:", "Using the environment variable: NIMBLE_DIR='" &
               env & "'", Success, priority = HighPriority)
       nimbleDir = env
+      setPackageCache(options, nimbleDir)
     else:
       # ...followed by project local deps mode
       if dirExists(nimbledeps) or (options.localdeps and not options.developLocaldeps):
@@ -422,6 +429,7 @@ proc setNimbleDir*(options: var Options) =
         nimbleDir = nimbledeps
         options.localdeps = true
         propagate = true
+        setPackageCache(options, options.config.nimbleDir) #We want to use the nimbleDir from the config so it can be shared
 
   options.nimbleDir = expandTilde(nimbleDir).absolutePath()
   if propagate:
@@ -438,8 +446,6 @@ proc setNimbleDir*(options: var Options) =
     let pkgsDir = options.getPkgsDir()
     if not dirExists(pkgsDir):
       createDir(pkgsDir)
-  if options.useSatSolver:
-    options.pkgCachePath = options.getNimbleDir() / "pkgcache"
 
 proc parseCommand*(key: string, result: var Options) =
   result.action = Action(typ: parseActionType(key))

tests/nimnimble/nim1.6.20/nim1620.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "0.1.0"
+author        = "jmgomez"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+
+
+# Dependencies
+
+requires "nim == 1.6.20"

tests/nimnimble/nim1.6.20/src/nim1620.nim

@@ -0,0 +1,7 @@
+# This is just an example to get you started. A typical library package
+# exports the main API in this file. Note that you cannot rename this file
+# but you can remove it if you wish.
+
+proc add*(x, y: int): int =
+  ## Adds two numbers together.
+  return x + y

tests/nimnimble/nim1.6.20/src/nim1620/submodule.nim

@@ -0,0 +1,12 @@
+# This is just an example to get you started. Users of your library will
+# import this file by writing ``import nim1620/submodule``. Feel free to rename or
+# remove this file altogether. You may create additional modules alongside
+# this file as required.
+
+type
+  Submodule* = object
+    name*: string
+
+proc initSubmodule*(): Submodule =
+  ## Initialises a new ``Submodule`` object.
+  Submodule(name: "Anonymous")

tests/nimnimble/nim1.6.20/tests/test1.nim

@@ -0,0 +1,12 @@
+# This is just an example to get you started. You may wish to put all of your
+# tests into a single file, or separate them into multiple `test1`, `test2`
+# etc. files (better names are recommended, just make sure the name starts with
+# the letter 't').
+#
+# To run these tests, simply execute `nimble test`.
+
+import unittest
+
+import nim1620
+test "can add":
+  check add(5, 5) == 10

tests/nimnimble/nim2.0.4/nim204.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "0.1.0"
+author        = "jmgomez"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+
+
+# Dependencies
+
+requires "nim == 2.0.4"

tests/nimnimble/nim2.0.4/src/nim204.nim

@@ -0,0 +1,7 @@
+# This is just an example to get you started. A typical library package
+# exports the main API in this file. Note that you cannot rename this file
+# but you can remove it if you wish.
+
+proc add*(x, y: int): int =
+  ## Adds two numbers together.
+  return x + y