Improve CI (fixes failure and improves isolation) (#924)

* Clear Nimble dir at the start of each test suite.

* Run CI every day at 7pm.

* Let's try debugging.

* Remove config file if backup doesn't exist in tester.

Author: dom96

.github/workflows/test.yml

@@ -1,6 +1,8 @@
 on:
   push:
   pull_request:
+  schedule:
+  - cron: "0 19 * * *"
 
 jobs:
   test:

tests/tester.nim

@@ -16,10 +16,6 @@ const stringNotFound = -1
 # Set env var to propagate nimble binary path
 putEnv("NIMBLE_TEST_BINARY_PATH", nimblePath)
 
-# Clear nimble dir.
-removeDir(installDir)
-createDir(installDir)
-
 # Always recompile.
 doAssert execCmdEx("nim c -d:danger " & path).exitCode == QuitSuccess
 
@@ -111,8 +107,19 @@ template testRefresh(body: untyped) =
   # Restore config
   if fileExists(configBakFile):
     safeMoveFile(configBakFile, configFile)
+  else:
+    # If the old config doesn't exist, we should still get rid of this new
+    # config to not screw up the other tests.
+    removeFile(configFile)
+
+proc beforeSuite() =
+  # Clear nimble dir.
+  removeDir(installDir)
+  createDir(installDir)
 
 suite "nimble refresh":
+  beforeSuite()
+
   test "can refresh with default urls":
     let (output, exitCode) = execNimble(["refresh"])
     checkpoint(output)
@@ -182,6 +189,8 @@ suite "nimble refresh":
       check exitCode == QuitFailure
 
 suite "nimscript":
+  beforeSuite()
+
   test "can install nimscript package":
     cd "nimscript":
       let
@@ -287,6 +296,8 @@ suite "nimscript":
       check execNimble("c", "-d:release", "nimscript.nim").exitCode == QuitSuccess
 
 suite "uninstall":
+  beforeSuite()
+
   test "can install packagebin2":
     let args = ["install", "https://github.com/nimble-test/packagebin2.git"]
     check execNimbleYes(args).exitCode == QuitSuccess
@@ -345,9 +356,9 @@ suite "uninstall":
         QuitSuccess
     check(not dirExists(installDir / "pkgs" / "PackageA-0.2.0"))
 
-    check execNimbleYes("uninstall", "nimscript").exitCode == QuitSuccess
-
 suite "nimble dump":
+  beforeSuite()
+
   test "can dump for current project":
     cd "testdump":
       let (outp, exitCode) = execNimble("dump")
@@ -425,6 +436,8 @@ backend: "c"
     check: outp == outpExpected
 
 suite "can handle two binary versions":
+  beforeSuite()
+
   setup:
     cd "binaryPackage/v1":
       check execNimbleYes("install").exitCode == QuitSuccess
@@ -452,6 +465,8 @@ suite "can handle two binary versions":
     check output.strip() == "v2"
 
 suite "reverse dependencies":
+  beforeSuite()
+
   test "basic test":
     cd "revdep/mydep":
       verify execNimbleYes("install")
@@ -496,7 +511,7 @@ suite "reverse dependencies":
     verify execNimbleYes("remove", "mydep")
 
   test "remove skips packages with revDeps (#504)":
-    check execNimbleYes("install", "[email protected]", "[email protected]").exitCode == QuitSuccess
+    check execNimbleYes("--debug", "install", "[email protected]", "[email protected]").exitCode == QuitSuccess
 
     var (output, exitCode) = execNimble("uninstall", "nimboost", "nimfp", "-n")
     var lines = output.strip.processOutput()
@@ -510,6 +525,8 @@ suite "reverse dependencies":
     check execNimble("path", "nimfp").exitCode != QuitSuccess
 
 suite "develop feature":
+  beforeSuite()
+
   test "can reject binary packages":
     cd "develop/binary":
       let (output, exitCode) = execNimble("develop")
@@ -533,7 +550,7 @@ suite "develop feature":
 
   test "can develop with srcDir":
     cd "develop/srcdirtest":
-      let (output, exitCode) = execNimble("develop")
+      let (output, exitCode) = execNimbleYes("develop")
       checkpoint output
       check(not output.processOutput.inLines("will not be compiled"))
       check exitCode == QuitSuccess
@@ -547,7 +564,7 @@ suite "develop feature":
       check split[1].endsWith("develop" / "srcdirtest" / "src")
 
     cd "develop/dependent":
-      let (output, exitCode) = execNimble("c", "-r", "src" / "dependent.nim")
+      let (output, exitCode) = execNimbleYes("c", "-r", "src" / "dependent.nim")
       checkpoint output
       check(output.processOutput.inLines("hello"))
       check exitCode == QuitSuccess
@@ -591,6 +608,8 @@ suite "develop feature":
     check output.strip() == installDir / "pkgs" / "srcdirtest-1.0"
 
 suite "test command":
+  beforeSuite()
+
   test "Runs passing unit tests":
     cd "testCommand/testsPass":
       # Pass flags to test #726, #757
@@ -630,6 +649,8 @@ suite "test command":
       check outp.processOutput.inLines(getCurrentDir())
 
 suite "check command":
+  beforeSuite()
+
   test "can succeed package":
     cd "binaryPackage/v1":
       let (outp, exitCode) = execNimble("check")
@@ -664,6 +685,8 @@ suite "check command":
       check outp.processOutput.inLines("package 'x' has an incorrect structure")
 
 suite "multi":
+  beforeSuite()
+
   test "can install package from git subdir":
     var
       args = @["install", "https://github.com/nimble-test/multi?subdir=alpha"]
@@ -683,6 +706,8 @@ suite "multi":
     check execNimbleYes(args).exitCode == QuitSuccess
 
 suite "Module tests":
+  beforeSuite()
+
   test "version":
     cd "..":
       check execCmdEx("nim c -r src/nimblepkg/version").exitCode == QuitSuccess
@@ -708,6 +733,8 @@ suite "Module tests":
       check execCmdEx("nim c -r src/nimblepkg/download").exitCode == QuitSuccess
 
 suite "nimble run":
+  beforeSuite()
+
   test "Invalid binary":
     cd "run":
       let (output, exitCode) = execNimble(
@@ -806,6 +833,8 @@ suite "nimble run":
       check output.contains("""Testing `nimble run`: @["--test"]""")
 
 suite "project local deps mode":
+  beforeSuite()
+
   test "nimbledeps exists":
     cd "localdeps":
       removeDir("nimbledeps")
@@ -831,6 +860,8 @@ suite "project local deps mode":
     check not dirExists("nimbledeps")
 
 suite "misc tests":
+  beforeSuite()
+
   test "depsOnly + flag order test":
     let (output, exitCode) = execNimbleYes(
       "--depsOnly", "install", "https://github.com/nimble-test/packagebin2"
@@ -926,9 +957,11 @@ suite "misc tests":
     check execNimble("list", "-i").exitCode == QuitSuccess
 
 suite "issues":
+  beforeSuite()
+
   test "issue 801":
     cd "issue801":
-      let (output, exitCode) = execNimbleYes("test")
+      let (output, exitCode) = execNimbleYes("--debug", "test")
       check exitCode == QuitSuccess
 
       # Verify hooks work
@@ -961,22 +994,22 @@ suite "issues":
 
   test "issue 727":
     cd "issue727":
-      var (output, exitCode) = execNimbleYes("c", "src/abc")
+      var (output, exitCode) = execNimbleYes("--debug", "c", "src/abc")
       check exitCode == QuitSuccess
       check fileExists(buildTests / "abc".addFileExt(ExeExt))
       check not fileExists("src/def".addFileExt(ExeExt))
       check not fileExists(buildTests / "def".addFileExt(ExeExt))
 
-      (output, exitCode) = execNimbleYes("uninstall", "-i", "timezones")
+      (output, exitCode) = execNimbleYes("--debug", "uninstall", "-i", "timezones")
       check exitCode == QuitSuccess
 
-      (output, exitCode) = execNimbleYes("run", "def")
+      (output, exitCode) = execNimbleYes("--debug", "run", "def")
       check exitCode == QuitSuccess
       check output.contains("def727")
       check not fileExists("abc".addFileExt(ExeExt))
       check fileExists("def".addFileExt(ExeExt))
 
-      (output, exitCode) = execNimbleYes("uninstall", "-i", "timezones")
+      (output, exitCode) = execNimbleYes("--debug", "uninstall", "-i", "timezones")
       check exitCode == QuitSuccess
 
   test "issue 708":
@@ -1285,6 +1318,8 @@ suite "issues":
       check inLines(lines, "Nothing to build")
 
 suite "nimble tasks":
+  beforeSuite()
+
   test "can list tasks even with no tasks defined in nimble file":
     cd "tasks/empty":
       let (_, exitCode) = execNimble("tasks")