Merge branch 'devel' into araq-quirky-exceptions

7 files changed, 170 insertions, 65 deletions

diff --git a/testament/categories.nim b/testament/categories.nim
index 55b9e5ed2..77769743d 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -10,6 +10,8 @@
 ## Include for the tester that contains test suites that test special features
 ## of the compiler.
 
+import important_packages
+
 const
   specialCategories = [
     "assert",
@@ -97,7 +99,7 @@ proc compileRodFiles(r: var TResults, cat: Category, options: string) =
 
 proc flagTests(r: var TResults, cat: Category, options: string) =
   # --genscript
-  const filename = "tests"/"flags"/"tgenscript"
+  const filename = testsDir/"flags"/"tgenscript"
   const genopts = " --genscript"
   let nimcache = nimcacheDir(filename, genopts, targetC)
   testSpec r, makeTest(filename, genopts, cat)
@@ -352,13 +354,13 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) =
     "8b5d28e985c0542163927d253a3e4fc9",
     "783299b98179cc725f9c46b5e3b5381f",
     "1a2b3fba1187c68d6a9bfa66854f3318",
-    "80f9c3e594a798225046e8a42e990daf"
+    "391ff57b38d9ea6f3eeb3fe69ab539d3"
   ]
 
   for i, test in tests:
-    let filename = "tests" / test.addFileExt("nim")
+    let filename = testsDir / test.addFileExt("nim")
     let testHash = getMD5(readFile(filename).string)
-    doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed."
+    doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed: " & $(i: i, testHash: testHash, refHash: refHashes[i])
   # Run the tests.
   for testfile in tests:
     test "tests/" & testfile & ".nim"
@@ -425,15 +427,12 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
     let contents = readFile(testFile).string
     var testObj = makeTest(testFile, options, cat)
     #[
-    TODO:
+    todo:
     this logic is fragile:
     false positives (if appears in a comment), or false negatives, eg
     `when defined(osx) and isMainModule`.
-    Instead of fixing this, a much better way, is to extend
-    https://github.com/nim-lang/Nim/issues/9581 to stdlib modules as follows:
-    * add these to megatest
-    * patch compiler so `isMainModule` is true when -d:isMainModuleIsAlwaysTrue
-    That'll give speedup benefit, and we don't have to patch stdlib files.
+    Instead of fixing this, see https://github.com/nim-lang/Nim/issues/10045
+    for a much better way.
     ]#
     if "when isMainModule" notin contents:
       testObj.spec.action = actionCompile
@@ -451,7 +450,7 @@ if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
 let
   nimbleExe = findExe("nimble")
   #packageDir = nimbleDir / "pkgs" # not used
-  packageIndex = nimbleDir / "packages.json"
+  packageIndex = nimbleDir / "packages_official.json"
 
 proc waitForExitEx(p: Process): int =
   var outp = outputStream(p)
@@ -466,28 +465,35 @@ proc waitForExitEx(p: Process): int =
 
 proc getPackageDir(package: string): string =
   ## TODO - Replace this with dom's version comparison magic.
-  var commandOutput = execCmdEx("nimble path $#" % package)
+  let commandOutput = execCmdEx("nimble path $#" % package)
   if commandOutput.exitCode != QuitSuccess:
     return ""
   else:
     result = commandOutput[0].string
 
-iterator listPackages(filter: PackageFilter): tuple[name, url: string] =
+iterator listPackages(filter: PackageFilter): tuple[name, url, cmd: string] =
+  const defaultCmd = "nimble test"
   let packageList = parseFile(packageIndex)
-  for package in packageList.items():
-    let
-      name = package["name"].str
-      url = package["url"].str
-      isCorePackage = "nim-lang" in normalize(url)
-    case filter:
-    of pfCoreOnly:
-      if isCorePackage:
-        yield (name, url)
-    of pfExtraOnly:
-      if not isCorePackage:
-        yield (name, url)
-    of pfAll:
-      yield (name, url)
+  for package in packageList.items:
+    if package.hasKey("url"):
+      let name = package["name"].str
+      if name notin ["nimble", "compiler"]:
+        let url = package["url"].str
+        case filter
+        of pfCoreOnly:
+          if "nim-lang" in normalize(url):
+            yield (name, url, defaultCmd)
+        of pfExtraOnly:
+          for n, cmd, commit in important_packages.packages.items:
+            if name == n: yield (name, url, cmd)
+        of pfAll:
+          yield (name, url, defaultCmd)
+
+proc makeSupTest(test, options: string, cat: Category): TTest =
+  result.cat = cat
+  result.name = test
+  result.options = options
+  result.startTime = epochTime()
 
 proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
   if nimbleExe == "":
@@ -498,31 +504,34 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
     echo("[Warning] - Cannot run nimble tests: Nimble update failed.")
     return
 
-  let packageFileTest = makeTest("PackageFileParsed", "", cat)
+  let packageFileTest = makeSupTest("PackageFileParsed", "", cat)
+  var keepDir = false
   try:
-    for name, url in listPackages(filter):
-      var test = makeTest(name, "", cat)
-      echo(url)
-      let
-        installProcess = startProcess(nimbleExe, "", ["install", "-y", name])
-        installStatus = waitForExitEx(installProcess)
+    for name, url, cmd in listPackages(filter):
+      var test = makeSupTest(url, "", cat)
+      let buildPath = "pkgstemp" / name
+      let installProcess = startProcess("git", "", ["clone", url, buildPath])
+      let installStatus = waitForExitEx(installProcess)
       installProcess.close
       if installStatus != QuitSuccess:
         r.addResult(test, targetC, "", "", reInstallFailed)
-        continue
-
-      let
-        buildPath = getPackageDir(name).strip
-        buildProcess = startProcess(nimbleExe, buildPath, ["build"])
-        buildStatus = waitForExitEx(buildProcess)
-      buildProcess.close
-      if buildStatus != QuitSuccess:
-        r.addResult(test, targetC, "", "", reBuildFailed)
-      r.addResult(test, targetC, "", "", reSuccess)
+        keepDir = true
+      else:
+        let cmdArgs = parseCmdLine(cmd)
+        let buildProcess = startProcess(cmdArgs[0], buildPath, cmdArgs[1..^1])
+        let buildStatus = waitForExitEx(buildProcess)
+        buildProcess.close
+        if buildStatus != QuitSuccess:
+          r.addResult(test, targetC, "", "", reBuildFailed)
+          keepDir = true
+        else:
+          r.addResult(test, targetC, "", "", reSuccess)
     r.addResult(packageFileTest, targetC, "", "", reSuccess)
   except JsonParsingError:
     echo("[Warning] - Cannot run nimble tests: Invalid package file.")
     r.addResult(packageFileTest, targetC, "", "", reBuildFailed)
+  finally:
+    if not keepDir: removeDir("pkgstemp")
 
 
 # ----------------------------------------------------------------------------
@@ -534,7 +543,7 @@ proc `&.?`(a, b: string): string =
   result = if b.startswith(a): b else: a & b
 
 proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
-  let test = "tests" & DirSep &.? cat.string / test
+  let test = testsDir &.? cat.string / test
   let target = if cat.string.normalize == "js": targetJS else: targetC
   if existsFile(test):
     testSpec r, makeTest(test, options, cat), {target}
@@ -559,6 +568,7 @@ proc isJoinableSpec(spec: TSpec): bool =
     (spec.targets == {} or spec.targets == {targetC})
 
 proc norm(s: var string) =
+  # equivalent of s/\n+/\n/g (could use a single pass over input if needed)
   while true:
     let tmp = s.replace("\n\n", "\n")
     if tmp == s: break
@@ -566,9 +576,13 @@ proc norm(s: var string) =
   s = s.strip
 
 proc isTestFile*(file: string): bool =
-  let (dir, name, ext) = splitFile(file)
+  let (_, name, ext) = splitFile(file)
   result = ext == ".nim" and name.startsWith("t")
 
+proc quoted(a: string): string =
+  # todo: consider moving to system.nim
+  result.addQuoted(a)
+
 proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
   ## returs a list of tests that have problems
   var specs: seq[TSpec] = @[]
@@ -582,6 +596,8 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
         if isJoinableSpec(spec):
           specs.add spec
 
+  proc cmp(a: TSpec, b:TSpec): auto = cmp(a.file, b.file)
+  sort(specs, cmp=cmp) # reproducible order
   echo "joinable specs: ", specs.len
 
   if simulate:
@@ -591,19 +607,36 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
     return
 
   var megatest: string
-  for runSpec in specs:
-    megatest.add "import r\""
-    megatest.add runSpec.file
-    megatest.add "\"\n"
+  #[
+  TODO(minor):
+  get from Nim cmd
+  put outputGotten.txt, outputGotten.txt, megatest.nim there too
+  delete upon completion, maybe
+  ]#
+  var outDir = nimcacheDir(testsDir / "megatest", "", targetC)
+  const marker = "megatest:processing: "
+
+  for i, runSpec in specs:
+    let file = runSpec.file
+    let file2 = outDir / ("megatest_" & $i & ".nim")
+    # `include` didn't work with `trecmod2.nim`, so using `import`
+    let code = "echo \"" & marker & "\", " & quoted(file) & "\n"
+    createDir(file2.parentDir)
+    writeFile(file2, code)
+    megatest.add "import " & quoted(file2) & "\n"
+    megatest.add "import " & quoted(file) & "\n"
 
   writeFile("megatest.nim", megatest)
 
-  const args = ["c", "-d:testing", "--listCmd", "megatest.nim"]
-  var (buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, options = {poStdErrToStdOut, poUsePath}, input = "")
+  let args = ["c", "--nimCache:" & outDir, "-d:testing", "--listCmd", "megatest.nim"]
+  proc onStdout(line: string) = echo line
+  var (buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, options = {poStdErrToStdOut, poUsePath}, input = "",
+    onStdout = if verboseMegatest: onStdout else: nil)
   if exitCode != 0:
     echo buf
     quit("megatest compilation failed")
 
+  # Could also use onStdout here.
   (buf, exitCode) = execCmdEx("./megatest")
   if exitCode != 0:
     echo buf
@@ -613,6 +646,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
   writeFile("outputGotten.txt", buf)
   var outputExpected = ""
   for i, runSpec in specs:
+    outputExpected.add marker & runSpec.file & "\n"
     outputExpected.add runSpec.output.strip
     outputExpected.add '\n'
   norm outputExpected
@@ -621,6 +655,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
     writeFile("outputExpected.txt", outputExpected)
     discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt")
     echo "output different!"
+    # outputGotten.txt, outputExpected.txt not removed on purpose for debugging.
     quit 1
   else:
     echo "output OK"
@@ -684,10 +719,9 @@ proc processCategory(r: var TResults, cat: Category, options, testsDir: string,
     runJoinedTest(r, cat, testsDir)
   else:
     var testsRun = 0
-
     var files: seq[string]
-    for file in walkDirRec("tests" & DirSep &.? cat.string):
-        if isTestFile(file): files.add file
+    for file in walkDirRec(testsDir &.? cat.string):
+      if isTestFile(file): files.add file
     files.sort # give reproducible order
 
     for i, name in files:
diff --git a/testament/htmlgen.nim b/testament/htmlgen.nim
index 34902f71e..641c1c32c 100644
--- a/testament/htmlgen.nim
+++ b/testament/htmlgen.nim
@@ -11,7 +11,7 @@
 
 import cgi, backend, strutils, json, os, tables, times
 
-import "testamenthtml.templ"
+import "testamenthtml.nimf"
 
 proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) =
   let
diff --git a/testament/important_packages.nim b/testament/important_packages.nim
new file mode 100644
index 000000000..6607783ab
--- /dev/null
+++ b/testament/important_packages.nim
@@ -0,0 +1,28 @@
+import strutils
+
+template pkg(name: string; cmd = "nimble test"; version = ""): untyped =
+  packages.add((name, cmd, version))
+
+var packages*: seq[tuple[name, cmd, version: string]] = @[]
+
+pkg "karax"
+pkg "cligen"
+pkg "glob"
+pkg "regex"
+pkg "freeimage", "nim c freeimage.nim"
+pkg "zero_functional"
+pkg "nimpy", "nim c nimpy.nim"
+pkg "nimongo", "nimble test_ci"
+pkg "inim"
+
+pkg "sdl1", "nim c src/sdl.nim"
+pkg "iterutils"
+pkg "gnuplot"
+pkg "c2nim"
+
+#[
+    arraymancer
+    nimpb
+    jester
+    nimx
+]#
diff --git a/testament/lib/stdtest/specialpaths.nim b/testament/lib/stdtest/specialpaths.nim
new file mode 100644
index 000000000..3c8b2338c
--- /dev/null
+++ b/testament/lib/stdtest/specialpaths.nim
@@ -0,0 +1,31 @@
+#[
+todo: move findNimStdLibCompileTime, findNimStdLib here
+]#
+
+import os
+
+# Note: all the const paths defined here are known at compile time and valid
+# so long Nim repo isn't relocated after compilation.
+# This means the binaries they produce will embed hardcoded paths, which
+# isn't appropriate for some applications that need to be relocatable.
+
+const sourcePath = currentSourcePath()
+  # robust way to derive other paths here
+  # We don't depend on PATH so this is robust to having multiple nim
+  # binaries
+
+const nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir
+  ## root of Nim repo
+
+const stdlibDir* = nimRootDir / "lib"
+  # todo: make nimeval.findNimStdLibCompileTime use this
+
+const systemPath* = stdlibDir / "system.nim"
+
+const buildDir* = nimRootDir / "build"
+  ## refs #10268: all testament generated files should go here to avoid
+  ## polluting .gitignore
+
+static:
+  # sanity check
+  doAssert fileExists(systemPath)
diff --git a/testament/specs.nim b/testament/specs.nim
index 22ad7d2f8..903e3e282 100644
--- a/testament/specs.nim
+++ b/testament/specs.nim
@@ -9,7 +9,7 @@
 
 import parseutils, strutils, os, osproc, streams, parsecfg
 
-var compilerPrefix* = "compiler" / "nim"  ## built via ./koch tests
+var compilerPrefix* = "nim"
 
 let isTravis* = existsEnv("TRAVIS")
 let isAppVeyor* = existsEnv("APPVEYOR")
diff --git a/testament/testamenthtml.templ b/testament/testamenthtml.nimf
index 9190f370e..9190f370e 100644
--- a/testament/testamenthtml.templ
+++ b/testament/testamenthtml.nimf
diff --git a/testament/tester.nim b/testament/tester.nim
index 9e5fe5830..ad0d22742 100644
--- a/testament/tester.nim
+++ b/testament/tester.nim
@@ -17,8 +17,11 @@ import
 var useColors = true
 var backendLogging = true
 var simulate = false
+var verboseMegatest = false # very verbose but can be useful
+var verboseCommands = false
 
 const
+  testsDir = "tests" & DirSep
   resultsFile = "testresults.html"
   #jsonFile = "testresults.json" # not used
   Usage = """Usage:
@@ -34,6 +37,8 @@ Arguments:
   arguments are passed to the compiler
 Options:
   --print                   also print results to the console
+  --verboseMegatest         log to stdout megatetest compilation
+  --verboseCommands         log to stdout info about commands being run
   --simulate                see what tests would be run but don't run them (for debugging)
   --failing                 only show failing/ignored tests
   --targets:"c c++ js objc" run tests for specified targets (default: all)
@@ -83,7 +88,7 @@ proc getFileDir(filename: string): string =
   if not result.isAbsolute():
     result = getCurrentDir() / result
 
-proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOption], input: string): tuple[
+proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOption], input: string, onStdout: proc(line: string) = nil): tuple[
                 output: TaintedString,
                 exitCode: int] {.tags:
                 [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
@@ -101,14 +106,21 @@ proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOp
   var line = newStringOfCap(120).TaintedString
   while true:
     if outp.readLine(line):
-      result[0].string.add(line.string)
-      result[0].string.add("\n")
+      result.output.string.add(line.string)
+      result.output.string.add("\n")
+      if onStdout != nil: onStdout(line.string)
     else:
-      result[1] = peekExitCode(p)
-      if result[1] != -1: break
+      result.exitCode = peekExitCode(p)
+      if result.exitCode != -1: break
   close(p)
 
-
+  if verboseCommands:
+    var command2 = command
+    if args.len > 0: command2.add " " & args.quoteShellCommand
+    echo (msg: "execCmdEx2",
+      command: command2,
+      options: options,
+      exitCode: result.exitCode)
 
 proc nimcacheDir(filename, options: string, target: TTarget): string =
   ## Give each test a private nimcache dir so they don't clobber each other's.
@@ -517,8 +529,6 @@ else:
 
 include categories
 
-const testsDir = "tests" & DirSep
-
 proc main() =
   os.putenv "NIMTEST_COLOR", "never"
   os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
@@ -533,6 +543,8 @@ proc main() =
   while p.kind == cmdLongoption:
     case p.key.string.normalize
     of "print", "verbose": optPrintResults = true
+    of "verbosemegatest": verboseMegatest = true
+    of "verbosecommands": verboseCommands = true
     of "failing": optFailing = true
     of "pedantic": discard "now always enabled"
     of "targets":