summary refs log tree commit diff stats
path: root/testament
diff options
context:
space:
mode:
Diffstat (limited to 'testament')
-rw-r--r--testament/backend.nim11
-rw-r--r--testament/caasdriver.nim10
-rw-r--r--testament/categories.nim649
-rw-r--r--testament/htmlgen.nim4
-rw-r--r--testament/important_packages.nim200
-rw-r--r--testament/lib/readme.md4
-rw-r--r--testament/lib/stdtest/netutils.nim13
-rw-r--r--testament/lib/stdtest/specialpaths.nim56
-rw-r--r--testament/lib/stdtest/testutils.nim126
-rw-r--r--testament/lib/stdtest/unittest_light.nim37
-rw-r--r--testament/specs.nim429
-rw-r--r--testament/testament.nim592
-rw-r--r--testament/testament.nim.cfg4
-rw-r--r--testament/tests/shouldfail/tccodecheck.nim2
-rw-r--r--testament/tests/shouldfail/tcolumn.nim6
-rw-r--r--testament/tests/shouldfail/terrormsg.nim6
-rw-r--r--testament/tests/shouldfail/texitcode1.nim2
-rw-r--r--testament/tests/shouldfail/tfile.nim4
-rw-r--r--testament/tests/shouldfail/tline.nim6
-rw-r--r--testament/tests/shouldfail/tmaxcodesize.nim2
-rw-r--r--testament/tests/shouldfail/tmsg.nim6
-rw-r--r--testament/tests/shouldfail/tnimout.nim4
-rw-r--r--testament/tests/shouldfail/tnimoutfull.nim14
-rw-r--r--testament/tests/shouldfail/toutput.nim6
-rw-r--r--testament/tests/shouldfail/toutputsub.nim2
-rw-r--r--testament/tests/shouldfail/treject.nim7
-rw-r--r--testament/tests/shouldfail/tsortoutput.nim4
-rw-r--r--testament/tests/shouldfail/tvalgrind.nim17
28 files changed, 1438 insertions, 785 deletions
diff --git a/testament/backend.nim b/testament/backend.nim
index 0a6d7d8b9..1770c6657 100644
--- a/testament/backend.nim
+++ b/testament/backend.nim
@@ -13,7 +13,6 @@ type
   CommitId = distinct string
 
 proc `$`*(id: MachineId): string {.borrow.}
-#proc `$`(id: CommitId): string {.borrow.} # not used
 
 var
   thisMachine: MachineId
@@ -21,10 +20,10 @@ var
   thisBranch: string
 
 proc getMachine*(): MachineId =
-  var name = execProcess("hostname").string.strip
+  var name = execProcess("hostname").strip
   if name.len == 0:
-    name = when defined(posix): getEnv("HOSTNAME").string
-           else: getEnv("COMPUTERNAME").string
+    name = when defined(posix): getEnv("HOSTNAME")
+           else: getEnv("COMPUTERNAME")
   if name.len == 0:
     quit "cannot determine the machine name"
 
@@ -32,8 +31,8 @@ proc getMachine*(): MachineId =
 
 proc getCommit(): CommitId =
   const commLen = "commit ".len
-  let hash = execProcess("git log -n 1").string.strip[commLen..commLen+10]
-  thisBranch = execProcess("git symbolic-ref --short HEAD").string.strip
+  let hash = execProcess("git log -n 1").strip[commLen..commLen+10]
+  thisBranch = execProcess("git symbolic-ref --short HEAD").strip
   if hash.len == 0 or thisBranch.len == 0: quit "cannot determine git HEAD"
   result = CommitId(hash)
 
diff --git a/testament/caasdriver.nim b/testament/caasdriver.nim
index 30383bddb..01e402e07 100644
--- a/testament/caasdriver.nim
+++ b/testament/caasdriver.nim
@@ -62,7 +62,7 @@ proc doCaasCommand(session: var NimSession, command: string): string =
   result = ""
 
   while true:
-    var line = TaintedString("")
+    var line = ""
     if session.nim.outputStream.readLine(line):
       if line.string == "": break
       result.add(line.string & "\n")
@@ -78,7 +78,7 @@ proc doProcCommand(session: var NimSession, command: string): string =
   var
     process = startProcess(NimBin, args = session.replaceVars(command).split)
     stream = outputStream(process)
-    line = TaintedString("")
+    line = ""
 
   result = ""
   while stream.readLine(line):
@@ -113,12 +113,12 @@ proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool):
   result = true
 
   var f = open(script)
-  var project = TaintedString("")
+  var project = ""
 
   if f.readLine(project):
     var
       s = startNimSession(script.parentDir / project.string, script, mode)
-      tline = TaintedString("")
+      tline = ""
       ln = 1
 
     while f.readLine(tline):
@@ -175,7 +175,7 @@ when isMainModule:
     verbose = false
 
   for i in 0..paramCount() - 1:
-    let param = string(paramStr(i + 1))
+    let param = paramStr(i + 1)
     case param
     of "verbose": verbose = true
     else: filter = param
diff --git a/testament/categories.nim b/testament/categories.nim
index f8cc625c4..843bef3f9 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -10,9 +10,11 @@
 ## Include for the tester that contains test suites that test special features
 ## of the compiler.
 
-# included from tester.nim
+# included from testament.nim
 
 import important_packages
+import std/[strformat, strutils]
+from std/sequtils import filterIt
 
 const
   specialCategories = [
@@ -21,26 +23,21 @@ const
     "debugger",
     "dll",
     "examples",
-    "flags",
     "gc",
     "io",
     "js",
     "ic",
     "lib",
-    "longgc",
     "manyloc",
     "nimble-packages",
     "niminaction",
-    "rodfiles",
     "threads",
-    "untestable",
-    "stdlib",
+    "untestable", # see trunner_special
     "testdata",
     "nimcache",
     "coroutines",
     "osproc",
     "shouldfail",
-    "dir with space",
     "destructor"
   ]
 
@@ -48,72 +45,9 @@ proc isTestFile*(file: string): bool =
   let (_, name, ext) = splitFile(file)
   result = ext == ".nim" and name.startsWith("t")
 
-# ---------------- IC tests ---------------------------------------------
-
-proc icTests(r: var TResults; testsDir: string, cat: Category, options: string) =
-  const
-    tooltests = ["compiler/nim.nim", "tools/nimgrep.nim"]
-    writeOnly = " --incremental:writeonly "
-    readOnly = " --incremental:readonly "
-    incrementalOn = " --incremental:on "
-
-  template test(x: untyped) =
-    testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache)
-
-  template editedTest(x: untyped) =
-    var test = makeTest(file, x & options, cat)
-    test.spec.targets = {getTestSpecTarget()}
-    testSpecWithNimcache(r, test, nimcache)
-
-  const tempExt = "_temp.nim"
-  for it in walkDirRec(testsDir / "ic"):
-    if isTestFile(it) and not it.endsWith(tempExt):
-      let nimcache = nimcacheDir(it, options, getTestSpecTarget())
-      removeDir(nimcache)
-
-      let content = readFile(it)
-      for fragment in content.split("#!EDIT!#"):
-        let file = it.replace(".nim", tempExt)
-        writeFile(file, fragment)
-        let oldPassed = r.passed
-        editedTest incrementalOn
-        if r.passed != oldPassed+1: break
-
-  for file in tooltests:
-    let nimcache = nimcacheDir(file, options, getTestSpecTarget())
-    removeDir(nimcache)
-
-    let oldPassed = r.passed
-    test writeOnly
-
-    if r.passed == oldPassed+1:
-      test readOnly
-      if r.passed == oldPassed+2:
-        test readOnly & "-d:nimBackendAssumesChange "
-
-# --------------------- flags tests -------------------------------------------
-
-proc flagTests(r: var TResults, cat: Category, options: string) =
-  # --genscript
-  const filename = testsDir/"flags"/"tgenscript"
-  const genopts = " --genscript"
-  let nimcache = nimcacheDir(filename, genopts, targetC)
-  testSpec r, makeTest(filename, genopts, cat)
-
-  when defined(windows):
-    testExec r, makeTest(filename, " cmd /c cd " & nimcache &
-                         " && compile_tgenscript.bat", cat)
-
-  elif defined(posix):
-    testExec r, makeTest(filename, " sh -c \"cd " & nimcache &
-                         " && sh compile_tgenscript.sh\"", cat)
-
-  # Run
-  testExec r, makeTest(filename, " " & nimcache / "tgenscript", cat)
-
 # --------------------- DLL generation tests ----------------------------------
 
-proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
+proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = false) =
   const rpath = when defined(macosx):
       " --passL:-rpath --passL:@loader_path"
     else:
@@ -125,42 +59,53 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
   var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat)
   test2.spec.action = actionCompile
   testSpec c, test2
-  var test3 = makeTest("lib/nimhcr.nim", options & " --outdir:tests/dll" & rpath, cat)
+
+  var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat)
   test3.spec.action = actionCompile
   testSpec c, test3
-  var test4 = makeTest("tests/dll/visibility.nim", options & " --app:lib" & rpath, cat)
+  var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat)
   test4.spec.action = actionCompile
   testSpec c, test4
 
   # windows looks in the dir of the exe (yay!):
-  when not defined(Windows):
+  when not defined(windows):
     # posix relies on crappy LD_LIBRARY_PATH (ugh!):
     const libpathenv = when defined(haiku): "LIBRARY_PATH"
                        else: "LD_LIBRARY_PATH"
-    var libpath = getEnv(libpathenv).string
+    var libpath = getEnv(libpathenv)
     # Temporarily add the lib directory to LD_LIBRARY_PATH:
     putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: ""))
     defer: putEnv(libpathenv, libpath)
 
   testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat)
-  testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & rpath, cat)
-  testSpec r, makeTest("tests/dll/visibility.nim", options & rpath, cat)
+  testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat)
+  testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat)
 
   if "boehm" notin options:
+    # hcr tests
+    
+    var basicHcrTest = makeTest("tests/dll/nimhcr_basic.nim", options & " --threads:off --forceBuild --hotCodeReloading:on " & rpath, cat)
+    # test segfaults for now but compiles:
+    if isOrc: basicHcrTest.spec.action = actionCompile
+    testSpec r, basicHcrTest
+
     # force build required - see the comments in the .nim file for more details
     var hcri = makeTest("tests/dll/nimhcr_integration.nim",
-                                   options & " --forceBuild --hotCodeReloading:on" & rpath, cat)
+                                   options & " --threads:off --forceBuild --hotCodeReloading:on" & rpath, cat)
     let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget())
-    hcri.args = prepareTestArgs(hcri.spec.getCmd, hcri.name,
+    let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name,
                                 hcri.options, nimcache, getTestSpecTarget())
+    hcri.testArgs = cmd.parseCmdLine
     testSpec r, hcri
 
 proc dllTests(r: var TResults, cat: Category, options: string) =
   # dummy compile result:
   var c = initResults()
 
-  runBasicDLLTest c, r, cat, options
-  runBasicDLLTest c, r, cat, options & " -d:release"
+  runBasicDLLTest c, r, cat, options & " --mm:refc"
+  runBasicDLLTest c, r, cat, options & " -d:release --mm:refc"
+  runBasicDLLTest c, r, cat, options, isOrc = true
+  runBasicDLLTest c, r, cat, options & " -d:release", isOrc = true
   when not defined(windows):
     # still cannot find a recent Windows version of boehm.dll:
     runBasicDLLTest c, r, cat, options & " --gc:boehm"
@@ -169,18 +114,10 @@ proc dllTests(r: var TResults, cat: Category, options: string) =
 # ------------------------------ GC tests -------------------------------------
 
 proc gcTests(r: var TResults, cat: Category, options: string) =
-  template testWithNone(filename: untyped) =
-    testSpec r, makeTest("tests/gc" / filename, options &
-                  " --gc:none", cat)
-    testSpec r, makeTest("tests/gc" / filename, options &
-                  " -d:release --gc:none", cat)
-
   template testWithoutMs(filename: untyped) =
-    testSpec r, makeTest("tests/gc" / filename, options, cat)
-    testSpec r, makeTest("tests/gc" / filename, options &
-                  " -d:release", cat)
+    testSpec r, makeTest("tests/gc" / filename, options & "--mm:refc", cat)
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " -d:release -d:useRealtimeGC", cat)
+                  " -d:release -d:useRealtimeGC --mm:refc", cat)
     when filename != "gctest":
       testSpec r, makeTest("tests/gc" / filename, options &
                     " --gc:orc", cat)
@@ -193,6 +130,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) =
                   " --gc:markAndSweep", cat)
     testSpec r, makeTest("tests/gc" / filename, options &
                   " -d:release --gc:markAndSweep", cat)
+
   template test(filename: untyped) =
     testWithoutBoehm filename
     when not defined(windows) and not defined(android):
@@ -210,7 +148,6 @@ proc gcTests(r: var TResults, cat: Category, options: string) =
   test "gcleak"
   test "gcleak2"
   testWithoutBoehm "gctest"
-  testWithNone "gctest"
   test "gcleak3"
   test "gcleak4"
   # Disabled because it works and takes too long to run:
@@ -225,19 +162,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) =
 
   test "stackrefleak"
   test "cyclecollector"
-
-proc longGCTests(r: var TResults, cat: Category, options: string) =
-  when defined(windows):
-    let cOptions = "-ldl -DWIN"
-  else:
-    let cOptions = "-ldl"
-
-  var c = initResults()
-  # According to ioTests, this should compile the file
-  testSpec c, makeTest("tests/realtimeGC/shared", options, cat)
-  #        ^- why is this not appended to r? Should this be discarded?
-  testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun
-  testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat)
+  testWithoutBoehm "trace_globals"
 
 # ------------------------- threading tests -----------------------------------
 
@@ -256,6 +181,11 @@ proc ioTests(r: var TResults, cat: Category, options: string) =
   # dummy compile result:
   var c = initResults()
   testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat)
+  #        ^- why is this not appended to r? Should this be discarded?
+  # EDIT: this should be replaced by something like in D20210524T180826,
+  # likewise in similar instances where `testSpec c` is used, or more generally
+  # when a test depends on another test, as it makes tests non-independent,
+  # creating complications for batching and megatest logic.
   testSpec r, makeTest("tests/system/tio", options, cat)
 
 # ------------------------- async tests ---------------------------------------
@@ -268,25 +198,29 @@ proc asyncTests(r: var TResults, cat: Category, options: string) =
 # ------------------------- debugger tests ------------------------------------
 
 proc debuggerTests(r: var TResults, cat: Category, options: string) =
-  var t = makeTest("tools/nimgrep", options & " --debugger:on", cat)
-  t.spec.action = actionCompile
-  testSpec r, t
+  if fileExists("tools/nimgrep.nim"):
+    var t = makeTest("tools/nimgrep", options & " --debugger:on", cat)
+    t.spec.action = actionCompile
+    # force target to C because of MacOS 10.15 SDK headers bug
+    # https://github.com/nim-lang/Nim/pull/15612#issuecomment-712471879
+    t.spec.targets = {targetC}
+    testSpec r, t
 
 # ------------------------- JS tests ------------------------------------------
 
 proc jsTests(r: var TResults, cat: Category, options: string) =
   template test(filename: untyped) =
-    testSpec r, makeTest(filename, options & " -d:nodejs", cat), {targetJS}
-    testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat), {targetJS}
+    testSpec r, makeTest(filename, options, cat), {targetJS}
+    testSpec r, makeTest(filename, options & " -d:release", cat), {targetJS}
 
   for t in os.walkFiles("tests/js/t*.nim"):
     test(t)
   for testfile in ["exception/texceptions", "exception/texcpt1",
                    "exception/texcsub", "exception/tfinally",
                    "exception/tfinally2", "exception/tfinally3",
-                   "actiontable/tactiontable", "method/tmultimjs",
+                   "collections/tactiontable", "method/tmultimjs",
                    "varres/tvarres0", "varres/tvarres3", "varres/tvarres4",
-                   "varres/tvartup", "misc/tints", "misc/tunsignedinc",
+                   "varres/tvartup", "int/tints", "int/tunsignedinc",
                    "async/tjsandnativeasync"]:
     test "tests/" & testfile & ".nim"
 
@@ -296,8 +230,6 @@ proc jsTests(r: var TResults, cat: Category, options: string) =
 # ------------------------- nim in action -----------
 
 proc testNimInAction(r: var TResults, cat: Category, options: string) =
-  let options = options & " --nilseqs:on"
-
   template test(filename: untyped) =
     testSpec r, makeTest(filename, options, cat)
 
@@ -388,7 +320,7 @@ proc findMainFile(dir: string): string =
   var nimFiles = 0
   for kind, file in os.walkDir(dir):
     if kind == pcFile:
-      if file.endsWith(cfgExt): return file[.. ^(cfgExt.len+1)] & ".nim"
+      if file.endsWith(cfgExt): return file[0..^(cfgExt.len+1)] & ".nim"
       elif file.endsWith(".nim"):
         if result.len == 0: result = file
         inc nimFiles
@@ -418,7 +350,7 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
   proc isValid(file: string): bool =
     for dir in parentDirs(file, inclusive = false):
       if dir.lastPathPart in ["includes", "nimcache"]:
-        # eg: lib/pure/includes/osenv.nim gives: Error: This is an include file for os.nim!
+        # e.g.: lib/pure/includes/osenv.nim gives: Error: This is an include file for os.nim!
         return false
     let name = extractFilename(file)
     if name.splitFile.ext != ".nim": return false
@@ -433,12 +365,12 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
 
   files.sort # reproducible order
   for testFile in files:
-    let contents = readFile(testFile).string
+    let contents = readFile(testFile)
     var testObj = makeTest(testFile, options, cat)
     #[
     todo:
     this logic is fragile:
-    false positives (if appears in a comment), or false negatives, eg
+    false positives (if appears in a comment), or false negatives, e.g.
     `when defined(osx) and isMainModule`.
     Instead of fixing this, see https://github.com/nim-lang/Nim/issues/10045
     for a much better way.
@@ -448,114 +380,183 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
     testSpec r, testObj
 
 # ----------------------------- nimble ----------------------------------------
-
-var nimbleDir = getEnv("NIMBLE_DIR").string
-if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
-let
-  nimbleExe = findExe("nimble")
-  packageIndex = nimbleDir / "packages_official.json"
-
-iterator listPackages(): tuple[name, url, cmd: string, hasDeps: bool] =
-  let defaultCmd = "nimble test"
+proc listPackagesAll(): seq[NimblePackage] =
+  var nimbleDir = getEnv("NIMBLE_DIR")
+  if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
+  let packageIndex = nimbleDir / "packages_official.json"
   let packageList = parseFile(packageIndex)
-  for n, cmd, hasDeps, url in important_packages.packages.items:
-    let cmd = if cmd.len == 0: defaultCmd else: cmd
-    if url.len != 0:
-      yield (n, url, cmd, hasDeps)
+  proc findPackage(name: string): JsonNode =
+    for a in packageList:
+      if a["name"].str == name: return a
+  for pkg in important_packages.packages.items:
+    var pkg = pkg
+    if pkg.url.len == 0:
+      let pkg2 = findPackage(pkg.name)
+      if pkg2 == nil:
+        raise newException(ValueError, "Cannot find package '$#'." % pkg.name)
+      pkg.url = pkg2["url"].str
+    result.add pkg
+
+proc listPackages(packageFilter: string): seq[NimblePackage] =
+  let pkgs = listPackagesAll()
+  if packageFilter.len != 0:
+    # xxx document `packageFilter`, seems like a bad API,
+    # at least should be a regex; a substring match makes no sense.
+    result = pkgs.filterIt(packageFilter in it.name)
+  else:
+    if testamentData0.batchArg == "allowed_failures":
+      result = pkgs.filterIt(it.allowFailure)
+    elif testamentData0.testamentNumBatch == 0:
+      result = pkgs
     else:
-      var found = false
-      for package in packageList.items:
-        let name = package["name"].str
-        if name == n:
-          found = true
-          let pUrl = package["url"].str
-          yield (name, pUrl, cmd, hasDeps)
-          break
-      if not found:
-        raise newException(ValueError, "Cannot find package '$#'." % n)
-
-proc makeSupTest(test, options: string, cat: Category): TTest =
+      let pkgs2 = pkgs.filterIt(not it.allowFailure)
+      for i in 0..<pkgs2.len:
+        if i mod testamentData0.testamentNumBatch == testamentData0.testamentBatch:
+          result.add pkgs2[i]
+
+proc makeSupTest(test, options: string, cat: Category, debugInfo = ""): TTest =
   result.cat = cat
   result.name = test
   result.options = options
+  result.debugInfo = debugInfo
   result.startTime = epochTime()
 
-proc testNimblePackages(r: var TResults, cat: Category) =
-  if nimbleExe == "":
-    echo "[Warning] - Cannot run nimble tests: Nimble binary not found."
-    return
-  if execCmd("$# update" % nimbleExe) == QuitFailure:
-    echo "[Warning] - Cannot run nimble tests: Nimble update failed."
-    return
+import std/private/gitutils
 
+proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) =
+  let nimbleExe = findExe("nimble")
+  doAssert nimbleExe != "", "Cannot run nimble tests: Nimble binary not found."
+  doAssert execCmd("$# update" % nimbleExe) == 0, "Cannot run nimble tests: Nimble update failed."
   let packageFileTest = makeSupTest("PackageFileParsed", "", cat)
   let packagesDir = "pkgstemp"
+  createDir(packagesDir)
   var errors = 0
   try:
-    for name, url, cmd, hasDep in listPackages():
+    let pkgs = listPackages(packageFilter)
+    for i, pkg in pkgs:
       inc r.total
-      var test = makeSupTest(url, "", cat)
-      let buildPath = packagesDir / name
-      if not existsDir(buildPath):
-        if hasDep:
-          let installName = if url.len != 0: url else: name
-          let (nimbleCmdLine, nimbleOutput, nimbleStatus) = execCmdEx2("nimble", ["install", "-y", installName])
-          if nimbleStatus != QuitSuccess:
-            let message = "nimble install failed:\n$ " & nimbleCmdLine & "\n" & nimbleOutput
-            r.addResult(test, targetC, "", message, reInstallFailed)
-            continue
-
-        let (installCmdLine, installOutput, installStatus) = execCmdEx2("git", ["clone", url, buildPath])
-        if installStatus != QuitSuccess:
-          let message = "git clone failed:\n$ " & installCmdLine & "\n" & installOutput
-          r.addResult(test, targetC, "", message, reInstallFailed)
+      var test = makeSupTest(pkg.name, "", cat, "[$#/$#] " % [$i, $pkgs.len])
+      let buildPath = packagesDir / pkg.name
+      template tryCommand(cmd: string, workingDir2 = buildPath, reFailed = reInstallFailed, maxRetries = 1): string =
+        var outp: string
+        let ok = retryCall(maxRetry = maxRetries, backoffDuration = 10.0):
+          var status: int
+          (outp, status) = execCmdEx(cmd, workingDir = workingDir2)
+          status == QuitSuccess
+        if not ok:
+          if pkg.allowFailure:
+            inc r.passed
+            inc r.failedButAllowed
+          addResult(r, test, targetC, "", "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure)
           continue
-
-      let cmdArgs = parseCmdLine(cmd)
-
-      let (buildCmdLine, buildOutput, buildStatus) = execCmdEx2(cmdArgs[0], cmdArgs[1..^1], workingDir=buildPath)
-      if buildStatus != QuitSuccess:
-        let message = "package test failed\n$ " & buildCmdLine & "\n" & buildOutput
-        r.addResult(test, targetC, "", message, reBuildFailed)
-      else:
-        inc r.passed
-        r.addResult(test, targetC, "", "", reSuccess)
+        outp
+
+      if not dirExists(buildPath):
+        discard tryCommand("git clone $# $#" % [pkg.url.quoteShell, buildPath.quoteShell], workingDir2 = ".", maxRetries = 3)
+        if not pkg.useHead:
+          discard tryCommand("git fetch --tags", maxRetries = 3)
+          let describeOutput = tryCommand("git describe --tags --abbrev=0")
+          discard tryCommand("git checkout $#" % [describeOutput.strip.quoteShell])
+        discard tryCommand("nimble install --depsOnly -y", maxRetries = 3)
+      let cmds = pkg.cmd.split(';')
+      for i in 0 ..< cmds.len - 1:
+        discard tryCommand(cmds[i], maxRetries = 3)
+      discard tryCommand(cmds[^1], reFailed = reBuildFailed)
+      inc r.passed
+      r.addResult(test, targetC, "", "", "", reSuccess, allowFailure = pkg.allowFailure)
 
     errors = r.total - r.passed
     if errors == 0:
-      r.addResult(packageFileTest, targetC, "", "", reSuccess)
+      r.addResult(packageFileTest, targetC, "", "", "", reSuccess)
     else:
-      r.addResult(packageFileTest, targetC, "", "", reBuildFailed)
+      r.addResult(packageFileTest, targetC, "", "", "", reBuildFailed)
 
   except JsonParsingError:
-    echo "[Warning] - Cannot run nimble tests: Invalid package file."
-    r.addResult(packageFileTest, targetC, "", "Invalid package file", reBuildFailed)
+    errors = 1
+    r.addResult(packageFileTest, targetC, "", "", "Invalid package file", reBuildFailed)
+    raise
   except ValueError:
-    echo "[Warning] - $#" % getCurrentExceptionMsg()
-    r.addResult(packageFileTest, targetC, "", "Unknown package", reBuildFailed)
+    errors = 1
+    r.addResult(packageFileTest, targetC, "", "", "Unknown package", reBuildFailed)
+    raise # bug #18805
   finally:
     if errors == 0: removeDir(packagesDir)
 
+# ---------------- IC tests ---------------------------------------------
+
+proc icTests(r: var TResults; testsDir: string, cat: Category, options: string;
+             isNavigatorTest: bool) =
+  const
+    tooltests = ["compiler/nim.nim"]
+    writeOnly = " --incremental:writeonly "
+    readOnly = " --incremental:readonly "
+    incrementalOn = " --incremental:on -d:nimIcIntegrityChecks "
+    navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint:Conf:off --warnings:off "
+
+  template test(x: untyped) =
+    testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache)
+
+  template editedTest(x: untyped) =
+    var test = makeTest(file, x & options, cat)
+    if isNavigatorTest:
+      test.spec.action = actionCompile
+    test.spec.targets = {getTestSpecTarget()}
+    testSpecWithNimcache(r, test, nimcache)
+
+  template checkTest() =
+    var test = makeRawTest(file, options, cat)
+    test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:on $options " & file
+    testSpecWithNimcache(r, test, nimcache)
+
+  if not isNavigatorTest:
+    for file in tooltests:
+      let nimcache = nimcacheDir(file, options, getTestSpecTarget())
+      removeDir(nimcache)
+
+      let oldPassed = r.passed
+      checkTest()
+
+      if r.passed == oldPassed+1:
+        checkTest()
+        if r.passed == oldPassed+2:
+          checkTest()
+
+  const tempExt = "_temp.nim"
+  for it in walkDirRec(testsDir):
+  # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test
+    if isTestFile(it) and not it.endsWith(tempExt):
+      let nimcache = nimcacheDir(it, options, getTestSpecTarget())
+      removeDir(nimcache)
+
+      let content = readFile(it)
+      for fragment in content.split("#!EDIT!#"):
+        let file = it.replace(".nim", tempExt)
+        writeFile(file, fragment)
+        let oldPassed = r.passed
+        editedTest(if isNavigatorTest: navTestConfig else: incrementalOn)
+        if r.passed != oldPassed+1: break
 
 # ----------------------------------------------------------------------------
 
-const AdditionalCategories = ["debugger", "examples", "lib", "ic"]
+const AdditionalCategories = ["debugger", "examples", "lib", "ic", "navigator"]
 const MegaTestCat = "megatest"
 
 proc `&.?`(a, b: string): string =
   # candidate for the stdlib?
   result = if b.startsWith(a): b else: a & b
 
-proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
-  let test = testsDir &.? cat.string / test
-  let target = if cat.string.normalize == "js": targetJS else: targetC
-  if existsFile(test):
-    testSpec r, makeTest(test, options, cat), {target}
-  else:
-    echo "[Warning] - ", test, " test does not exist"
+proc processSingleTest(r: var TResults, cat: Category, options, test: string, targets: set[TTarget], targetsSet: bool) =
+  var targets = targets
+  if not targetsSet:
+    let target = if cat.string.normalize == "js": targetJS else: targetC
+    targets = {target}
+  doAssert fileExists(test), test & " test does not exist"
+  testSpec r, makeTest(test, options, cat), targets
 
 proc isJoinableSpec(spec: TSpec): bool =
-  result = not spec.sortoutput and
+  # xxx simplify implementation using a whitelist of fields that are allowed to be
+  # set to non-default values (use `fieldPairs`), to avoid issues like bug #16576.
+  result = useMegatest and not spec.sortoutput and
     spec.action == actionRun and
     not fileExists(spec.file.changeFileExt("cfg")) and
     not fileExists(spec.file.changeFileExt("nims")) and
@@ -567,36 +568,48 @@ proc isJoinableSpec(spec: TSpec): bool =
     spec.exitCode == 0 and
     spec.input.len == 0 and
     spec.nimout.len == 0 and
+    spec.nimoutFull == false and
+      # so that tests can have `nimoutFull: true` with `nimout.len == 0` with
+      # the meaning that they expect empty output.
+    spec.matrix.len == 0 and
     spec.outputCheck != ocSubstr and
     spec.ccodeCheck.len == 0 and
     (spec.targets == {} or spec.targets == {targetC})
-
-proc norm(s: var string) =
-  while true:
-    let tmp = s.replace("\n\n", "\n")
-    if tmp == s: break
-    s = tmp
-  s = s.strip
+  if result:
+    if spec.file.readFile.contains "when isMainModule":
+      result = false
 
 proc quoted(a: string): string =
   # todo: consider moving to system.nim
   result.addQuoted(a)
 
-proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
+proc runJoinedTest(r: var TResults, cat: Category, testsDir: string, options: string) =
   ## returns a list of tests that have problems
+  #[
+  xxx create a reusable megatest API after abstracting out testament specific code,
+  refs https://github.com/timotheecour/Nim/issues/655
+  and https://github.com/nim-lang/gtk2/pull/28; it's useful in other contexts.
+  ]#
   var specs: seq[TSpec] = @[]
   for kind, dir in walkDir(testsDir):
-    assert testsDir.startsWith(testsDir)
+    assert dir.startsWith(testsDir)
     let cat = dir[testsDir.len .. ^1]
     if kind == pcDir and cat notin specialCategories:
       for file in walkDirRec(testsDir / cat):
         if isTestFile(file):
-          let spec = parseSpec(file)
+          var spec: TSpec
+          try:
+            spec = parseSpec(file)
+          except ValueError:
+            # e.g. for `tests/navigator/tincludefile.nim` which have multiple
+            # specs; this will be handled elsewhere
+            echo "parseSpec raised ValueError for: '$1', assuming this will be handled outside of megatest" % file
+            continue
           if isJoinableSpec(spec):
             specs.add spec
 
-  proc cmp(a: TSpec, b:TSpec): auto = cmp(a.file, b.file)
-  sort(specs, cmp=cmp) # reproducible order
+  proc cmp(a: TSpec, b: TSpec): auto = cmp(a.file, b.file)
+  sort(specs, cmp = cmp) # reproducible order
   echo "joinable specs: ", specs.len
 
   if simulate:
@@ -606,142 +619,156 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
     return
 
   var megatest: string
-  #[
-  TODO(minor):
-  get from Nim cmd
-  put outputGotten.txt, outputGotten.txt, megatest.nim there too
-  delete upon completion, maybe
-  ]#
+  # xxx (minor) put outputExceptedFile, outputGottenFile, megatestFile under here or `buildDir`
   var outDir = nimcacheDir(testsDir / "megatest", "", targetC)
-  const marker = "megatest:processing: "
-
+  template toMarker(file, i): string =
+    "megatest:processing: [$1] $2" % [$i, file]
   for i, runSpec in specs:
     let file = runSpec.file
-    let file2 = outDir / ("megatest_" & $i & ".nim")
+    let file2 = outDir / ("megatest_a_$1.nim" % $i)
     # `include` didn't work with `trecmod2.nim`, so using `import`
-    let code = "echo \"" & marker & "\", " & quoted(file) & "\n"
+    let code = "echo $1\nstatic: echo \"CT:\", $1\n" % [toMarker(file, i).quoted]
     createDir(file2.parentDir)
     writeFile(file2, code)
-    megatest.add "import " & quoted(file2) & "\n"
-    megatest.add "import " & quoted(file) & "\n"
+    megatest.add "import $1\nimport $2 as megatest_b_$3\n" % [file2.quoted, file.quoted, $i]
 
-  writeFile("megatest.nim", megatest)
+  let megatestFile = testsDir / "megatest.nim" # so it uses testsDir / "config.nims"
+  writeFile(megatestFile, megatest)
 
-  let args = ["c", "--nimCache:" & outDir, "-d:testing", "--listCmd",
-              "--listFullPaths:off", "--excessiveStackTrace:off", "megatest.nim"]
+  let root = getCurrentDir()
 
+  var args = @["c", "--nimCache:" & outDir, "-d:testing", "-d:nimMegatest", "--listCmd",
+              "--path:" & root]
+  args.add options.parseCmdLine
+  args.add megatestFile
   var (cmdLine, buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, input = "")
   if exitCode != 0:
-    echo "$ ", cmdLine
-    echo buf.string
-    quit("megatest compilation failed")
+    echo "$ " & cmdLine & "\n" & buf
+    quit(failString & "megatest compilation failed")
 
-  (buf, exitCode) = execCmdEx("./megatest")
+  (buf, exitCode) = execCmdEx(megatestFile.changeFileExt(ExeExt).dup normalizeExe)
   if exitCode != 0:
-    echo buf.string
-    quit("megatest execution failed")
+    echo buf
+    quit(failString & "megatest execution failed")
 
-  norm buf.string
-  writeFile("outputGotten.txt", buf.string)
+  const outputExceptedFile = "outputExpected.txt"
+  const outputGottenFile = "outputGotten.txt"
+  writeFile(outputGottenFile, buf)
   var outputExpected = ""
   for i, runSpec in specs:
-    outputExpected.add marker & runSpec.file & "\n"
-    outputExpected.add runSpec.output.strip
-    outputExpected.add '\n'
-  norm outputExpected
-
-  if buf.string != outputExpected:
-    writeFile("outputExpected.txt", outputExpected)
-    discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt")
-    echo "output different!"
-    # outputGotten.txt, outputExpected.txt not removed on purpose for debugging.
+    outputExpected.add toMarker(runSpec.file, i) & "\n"
+    if runSpec.output.len > 0:
+      outputExpected.add runSpec.output
+      if not runSpec.output.endsWith "\n":
+        outputExpected.add '\n'
+
+  if buf != outputExpected:
+    writeFile(outputExceptedFile, outputExpected)
+    echo diffFiles(outputGottenFile, outputExceptedFile).output
+    echo failString & "megatest output different, see $1 vs $2" % [outputGottenFile, outputExceptedFile]
+    # outputGottenFile, outputExceptedFile not removed on purpose for debugging.
     quit 1
   else:
-    echo "output OK"
-    removeFile("outputGotten.txt")
-    removeFile("megatest.nim")
-  #testSpec r, makeTest("megatest", options, cat)
+    echo "megatest output OK"
+
 
 # ---------------------------------------------------------------------------
 
 proc processCategory(r: var TResults, cat: Category,
                      options, testsDir: string,
                      runJoinableTests: bool) =
-  case cat.string.normalize
-  of "rodfiles":
-    when false:
-      compileRodFiles(r, cat, options)
-      runRodFiles(r, cat, options)
-  of "ic":
-    when false:
-      icTests(r, testsDir, cat, options)
-  of "js":
-    # only run the JS tests on Windows or Linux because Travis is bad
-    # and other OSes like Haiku might lack nodejs:
-    if not defined(linux) and isTravis:
+  let cat2 = cat.string.normalize
+  var handled = false
+  if isNimRepoTests():
+    handled = true
+    case cat2
+    of "js":
+      # only run the JS tests on Windows or Linux because Travis is bad
+      # and other OSes like Haiku might lack nodejs:
+      if not defined(linux) and isTravis:
+        discard
+      else:
+        jsTests(r, cat, options)
+    of "dll":
+      dllTests(r, cat, options & " -d:nimDebugDlOpen")
+    of "gc":
+      gcTests(r, cat, options)
+    of "debugger":
+      debuggerTests(r, cat, options)
+    of "manyloc":
+      manyLoc r, cat, options
+    of "threads":
+      threadTests r, cat, options & " --threads:on"
+    of "io":
+      ioTests r, cat, options
+    of "async":
+      asyncTests r, cat, options
+    of "lib":
+      testStdlib(r, "lib/pure/", options, cat)
+      testStdlib(r, "lib/packages/docutils/", options, cat)
+    of "examples":
+      compileExample(r, "examples/*.nim", options, cat)
+      compileExample(r, "examples/gtk/*.nim", options, cat)
+      compileExample(r, "examples/talk/*.nim", options, cat)
+    of "nimble-packages":
+      testNimblePackages(r, cat, options)
+    of "niminaction":
+      testNimInAction(r, cat, options)
+    of "ic":
+      icTests(r, testsDir / cat2, cat, options, isNavigatorTest=false)
+    of "navigator":
+      icTests(r, testsDir / cat2, cat, options, isNavigatorTest=true)
+    of "untestable":
+      # These require special treatment e.g. because they depend on a third party
+      # dependency; see `trunner_special` which runs some of those.
       discard
     else:
-      jsTests(r, cat, options)
-  of "dll":
-    dllTests(r, cat, options)
-  of "flags":
-    flagTests(r, cat, options)
-  of "gc":
-    gcTests(r, cat, options)
-  of "longgc":
-    longGCTests(r, cat, options)
-  of "debugger":
-    debuggerTests(r, cat, options)
-  of "manyloc":
-    manyLoc r, cat, options
-  of "threads":
-    threadTests r, cat, options & " --threads:on"
-  of "io":
-    ioTests r, cat, options
-  of "async":
-    asyncTests r, cat, options
-  of "lib":
-    testStdlib(r, "lib/pure/", options, cat)
-    testStdlib(r, "lib/packages/docutils/", options, cat)
-  of "examples":
-    compileExample(r, "examples/*.nim", options, cat)
-    compileExample(r, "examples/gtk/*.nim", options, cat)
-    compileExample(r, "examples/talk/*.nim", options, cat)
-  of "nimble-packages":
-    testNimblePackages(r, cat)
-  of "niminaction":
-    testNimInAction(r, cat, options)
-  of "untestable":
-    # We can't test it because it depends on a third party.
-    discard # TODO: Move untestable tests to someplace else, i.e. nimble repo.
-  of "megatest":
-    runJoinedTest(r, cat, testsDir)
-  else:
-    var testsRun = 0
-    var files: seq[string]
-    for file in walkDirRec(testsDir &.? cat.string):
-      if isTestFile(file): files.add file
-    files.sort # give reproducible order
-
-    for i, name in files:
-      var test = makeTest(name, options, cat)
-      if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories:
-        discard "run the test"
-      else:
-        test.spec.err = reJoined
-      testSpec r, test
-      inc testsRun
-    if testsRun == 0:
-      echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run"
+      handled = false
+  if not handled:
+    case cat2
+    of "megatest":
+      runJoinedTest(r, cat, testsDir, options)
+      if isNimRepoTests():
+        runJoinedTest(r, cat, testsDir, options & " --mm:refc")
+    else:
+      var testsRun = 0
+      var files: seq[string]
+      for file in walkDirRec(testsDir &.? cat.string):
+        if isTestFile(file): files.add file
+      files.sort # give reproducible order
+      for i, name in files:
+        var test = makeTest(name, options, cat)
+        if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories:
+          discard "run the test"
+        else:
+          test.spec.err = reJoined
+        testSpec r, test
+        inc testsRun
+      if testsRun == 0:
+        const whiteListedDirs = ["deps", "htmldocs", "pkgs"]
+          # `pkgs` because bug #16556 creates `pkgs` dirs and this can affect some users
+          # that try an old version of choosenim.
+        doAssert cat.string in whiteListedDirs,
+          "Invalid category specified: '$#' not in whilelist: $#" % [cat.string, $whiteListedDirs]
 
 proc processPattern(r: var TResults, pattern, options: string; simulate: bool) =
   var testsRun = 0
-  for name in walkPattern(pattern):
-    if simulate:
-      echo "Detected test: ", name
-    else:
-      var test = makeTest(name, options, Category"pattern")
-      testSpec r, test
-    inc testsRun
+  if dirExists(pattern):
+    for k, name in walkDir(pattern):
+      if k in {pcFile, pcLinkToFile} and name.endsWith(".nim"):
+        if simulate:
+          echo "Detected test: ", name
+        else:
+          var test = makeTest(name, options, Category"pattern")
+          testSpec r, test
+        inc testsRun
+  else:
+    for name in walkPattern(pattern):
+      if simulate:
+        echo "Detected test: ", name
+      else:
+        var test = makeTest(name, options, Category"pattern")
+        testSpec r, test
+      inc testsRun
   if testsRun == 0:
     echo "no tests were found for pattern: ", pattern
diff --git a/testament/htmlgen.nim b/testament/htmlgen.nim
index ee3f3bad7..174f36d0b 100644
--- a/testament/htmlgen.nim
+++ b/testament/htmlgen.nim
@@ -16,7 +16,7 @@ import "testamenthtml.nimf"
 proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) =
   let
     trId = htmlQuote(testResultRow["category"].str & "_" & testResultRow["name"].str).
-        multiReplace({".": "_", " ": "_", ":": "_"})
+        multiReplace({".": "_", " ": "_", ":": "_", "/": "_"})
     name = testResultRow["name"].str.htmlQuote()
     category = testResultRow["category"].str.htmlQuote()
     target = testResultRow["target"].str.htmlQuote()
@@ -52,7 +52,7 @@ proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) =
     trId, name, target, category, action, resultDescription,
     timestamp, result, resultSign, panelCtxClass, textCtxClass, bgCtxClass
   )
-  if expected.isNilOrWhitespace() and gotten.isNilOrWhitespace():
+  if expected.isEmptyOrWhitespace() and gotten.isEmptyOrWhitespace():
     outfile.generateHtmlTestresultOutputNone()
   else:
     outfile.generateHtmlTestresultOutputDetails(
diff --git a/testament/important_packages.nim b/testament/important_packages.nim
index f2ce8fb25..efec04b3c 100644
--- a/testament/important_packages.nim
+++ b/testament/important_packages.nim
@@ -1,95 +1,193 @@
+##[
+## note 1
+`useHead` should ideally be used as the default but lots of packages (e.g. `chronos`)
+don't have release tags (or have really old ones compared to HEAD), making it
+impossible to test them reliably here.
 
-template pkg(name: string; cmd = "nimble test"; hasDeps = false; url = ""): untyped =
-  packages.add((name, cmd, hasDeps, url))
+packages listed here should ideally have regularly updated release tags, so that:
+* we're testing recent versions of the package
+* the version that's tested is stable enough even if HEAD may occasionally break
 
-var packages*: seq[tuple[name, cmd: string; hasDeps: bool; url: string]] = @[]
+## note 2: D20210308T165435:here
+nimble packages should be testable as follows:
+git clone $url $dir && cd $dir
+NIMBLE_DIR=$TMP_NIMBLE_DIR XDG_CONFIG_HOME= nimble install --depsOnly -y
+NIMBLE_DIR=$TMP_NIMBLE_DIR XDG_CONFIG_HOME= nimble test
 
+if this fails (e.g. nimcrypto), it could be because a package lacks a `tests/nim.cfg` with `--path:..`,
+so the above commands would've worked by accident with `nimble install` but not with `nimble install --depsOnly`.
+When this is the case, a workaround is to test this package here by adding `--path:$srcDir` on the test `cmd`.
+]##
 
+type NimblePackage* = object
+  name*, cmd*, url*: string
+  useHead*: bool
+  allowFailure*: bool
+    ## When true, we still run the test but the test is allowed to fail.
+    ## This is useful for packages that currently fail but that we still want to
+    ## run in CI, e.g. so that we can monitor when they start working again and
+    ## are reminded about those failures without making CI fail for unrelated PRs.
+
+var packages*: seq[NimblePackage]
+
+proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true, allowFailure = false) =
+  packages.add NimblePackage(name: name, cmd: cmd, url: url, useHead: useHead, allowFailure: allowFailure)
+
+pkg "alea"
 pkg "argparse"
-pkg "arraymancer", "nim c tests/tests_cpu.nim", true
-pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim"
-pkg "asyncmysql", "", true
+pkg "arraymancer", "nim c tests/tests_cpu.nim"
+pkg "ast_pattern_matching", "nim c -r tests/test1.nim"
+pkg "asyncftpclient", "nimble compileExample"
+pkg "asyncthreadpool", "nimble test --mm:refc"
+pkg "awk"
 pkg "bigints"
 pkg "binaryheap", "nim c -r binaryheap.nim"
-pkg "blscurve", "", true
-pkg "bncurve", "", true
+pkg "BipBuffer"
+pkg "blscurve", allowFailure = true
+pkg "bncurve"
+pkg "brainfuck", "nim c -d:release -r tests/compile.nim"
+pkg "bump", "nim c --mm:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump", allowFailure = true
 pkg "c2nim", "nim c testsuite/tester.nim"
 pkg "cascade"
+pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true
+pkg "checksums"
 pkg "chroma"
-pkg "chronicles", "nim c -o:chr -r chronicles.nim", true
-pkg "chronos", "", true
-pkg "cligen", "nim c -o:cligenn -r cligen.nim"
-pkg "coco", "", true
-pkg "combparser"
+pkg "chronicles", "nim c -o:chr -r chronicles.nim"
+pkg "chronos", "nim c -r -d:release tests/testall"
+pkg "cligen", "nim c --path:. -r cligen.nim"
+pkg "combparser", "nimble test --mm:orc"
 pkg "compactdict"
-pkg "comprehension", "", false, "https://github.com/alehander42/comprehension"
-pkg "criterion"
+pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehension"
+pkg "constantine", "nimble make_lib"
+pkg "cowstrings"
+pkg "criterion", allowFailure = true # needs testing binary
+pkg "datamancer"
 pkg "dashing", "nim c tests/functional.nim"
+pkg "delaunay"
+pkg "dnsclient", allowFailure = true # super fragile
 pkg "docopt"
-pkg "easygl", "nim c -o:egl -r src/easygl.nim", true, "https://github.com/jackmott/easygl"
+pkg "dotenv"
+# when defined(linux): pkg "drchaos"
+pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl"
 pkg "elvis"
-pkg "fragments", "nim c -r fragments/dsl.nim"
+pkg "faststreams"
+pkg "fidget"
+pkg "fragments", "nim c -r fragments/dsl.nim", allowFailure = true # pending https://github.com/nim-lang/packages/issues/2115 
+pkg "fusion"
 pkg "gara"
 pkg "glob"
-pkg "gnuplot"
-# pkg "godot", "nim c -r godot/godot.nim" # not yet compatible with Nim 0.19
+pkg "ggplotnim", "nim c -d:noCairo -r tests/tests.nim"
+pkg "gittyup", "nimble test", "https://github.com/disruptek/gittyup", allowFailure = true
+pkg "gnuplot", "nim c gnuplot.nim"
+# pkg "gram", "nim c -r --mm:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram"
+  # pending https://github.com/nim-lang/Nim/issues/16509
 pkg "hts", "nim c -o:htss src/hts.nim"
+pkg "httpauth"
+pkg "httputils"
 pkg "illwill", "nimble examples"
 pkg "inim"
 pkg "itertools", "nim doc src/itertools.nim"
 pkg "iterutils"
+pkg "json_rpc"
+pkg "json_serialization"
 pkg "jstin"
 pkg "karax", "nim c -r tests/tester.nim"
+pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree"
 pkg "loopfusion"
-pkg "msgpack4nim"
+pkg "lockfreequeues"
+pkg "macroutils"
+pkg "manu"
+pkg "markdown"
+pkg "measuremancer", "nimble testDeps; nimble -y test"
+pkg "memo"
+pkg "msgpack4nim", "nim c -r tests/test_spec.nim"
 pkg "nake", "nim c nakefile.nim"
-pkg "neo", "nim c -d:blas=openblas tests/all.nim", true
-# pkg "nico", "", true
-pkg "nicy", "nim c src/nicy.nim"
+pkg "neo", "nim c -d:blas=openblas --mm:refc tests/all.nim"
+pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true, allowFailure = true
+  # inactive, tests not adapted to #23096
+pkg "netty"
+pkg "nico", allowFailure = true
+pkg "nicy", "nim c -r src/nicy.nim"
 pkg "nigui", "nim c -o:niguii -r src/nigui.nim"
-pkg "nimcrypto", "nim c -r tests/testall.nim"
-pkg "NimData", "nim c -o:nimdataa src/nimdata.nim", true
-pkg "nimes", "nim c src/nimes.nim", true
-pkg "nimfp", "nim c -o:nfp -r src/fp.nim", true
-pkg "nimgame2", "nim c nimgame2/nimgame.nim", true
-pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim", true
-# pkg "nimlsp", "", true
-pkg "nimly", "", true
-# pkg "nimongo", "nimble test_ci", true
+pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround needed, see D20210308T165435
+pkg "NimData", "nim c -o:nimdataa src/nimdata.nim"
+pkg "nimes", "nim c src/nimes.nim"
+pkg "nimfp", "nim c -o:nfp -r src/fp.nim"
+pkg "nimgame2", "nim c --mm:refc nimgame2/nimgame.nim"
+pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim"
+pkg "nimib"
+pkg "nimlsp"
+pkg "nimly", "nim c -r tests/test_readme_example.nim"
+pkg "nimongo", "nimble test_ci", allowFailure = true
+pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true
+pkg "nimPNG", useHead = true
 pkg "nimpy", "nim c -r tests/nimfrompy.nim"
 pkg "nimquery"
-pkg "nimsl", "", true
+pkg "nimsl"
 pkg "nimsvg"
-# pkg "nimterop", "", true
-pkg "nimx", "nim c --threads:on test/main.nim", true
-pkg "norm", "nim c -r tests/tsqlite.nim", true
-pkg "npeg"
-pkg "ormin", "nim c -o:orminn ormin.nim", true
+pkg "nimterop", "nimble minitest", url = "https://github.com/nim-lang/nimterop"
+pkg "nimwc", "nim c nimwc.nim"
+pkg "nimx", "nim c test/main.nim", allowFailure = true
+pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter"
+pkg "norm", "testament r tests/common/tmodel.nim"
+pkg "normalize"
+pkg "npeg", "nimble testarc"
+pkg "numericalnim", "nimble nimCI"
+pkg "optionsutils"
+pkg "ormin", "nim c -o:orminn ormin.nim"
 pkg "parsetoml"
 pkg "patty"
-pkg "plotly", "nim c --oldgensym:on examples/all.nim", true
+pkg "pixie"
+pkg "plotly", "nim c examples/all.nim"
 pkg "pnm"
 pkg "polypbren"
-pkg "protobuf", "nim c -o:protobuff -r src/protobuf.nim", true
+pkg "presto"
+pkg "prologue", "nimble tcompile"
+# remove fork after https://github.com/PMunch/combparser/pull/7 is merged:
+pkg "protobuf", "nimble install -y https://github.com/metagn/combparser@#HEAD; nim c -o:protobuff -r src/protobuf.nim"
 pkg "rbtree"
 pkg "react", "nimble example"
-pkg "regex", "nim c src/regex", true
-pkg "result", "nim c -r result.nim"
+pkg "regex", "nim c src/regex"
+pkg "results", "nim c -r results.nim"
+pkg "RollingHash", "nim c -r tests/test_cyclichash.nim"
 pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim"
 pkg "sdl1", "nim c -r src/sdl.nim"
 pkg "sdl2_nim", "nim c -r sdl2/sdl.nim"
-pkg "snip", "", false, "https://github.com/genotrance/snip"
-pkg "stint", "nim c -o:stintt -r stint.nim"
-pkg "strunicode", "nim c -r src/strunicode.nim", true
-pkg "telebot", "nim c -o:tbot --oldgensym:on -r telebot.nim", true
+pkg "serialization"
+pkg "sigv4", "nim c --mm:arc -r sigv4.nim", "https://github.com/disruptek/sigv4"
+pkg "sim"
+pkg "smtp", "nimble compileExample"
+pkg "snip", "nimble test", "https://github.com/genotrance/snip"
+pkg "ssostrings"
+pkg "stew"
+pkg "stint", "nim c stint.nim"
+pkg "strslice"
+pkg "strunicode", "nim c -r --mm:refc src/strunicode.nim"
+pkg "supersnappy"
+pkg "synthesis"
+pkg "taskpools"
+pkg "telebot", "nim c -o:tbot -r src/telebot.nim"
 pkg "tempdir"
-pkg "tensordsl", "nim c -r tests/tests.nim", false, "https://krux02@bitbucket.org/krux02/tensordslnim.git"
+pkg "templates"
+pkg "tensordsl", "nim c -r --mm:refc tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git"
+pkg "terminaltables", "nim c src/terminaltables.nim"
+pkg "termstyle", "nim c -r termstyle.nim"
+pkg "testutils"
+pkg "timeit"
+pkg "timezones"
 pkg "tiny_sqlite"
-pkg "unicodedb"
-pkg "unicodeplus", "", true
+pkg "unicodedb", "nim c -d:release -r tests/tests.nim"
+pkg "unicodeplus", "nim c -d:release -r tests/tests.nim"
+pkg "union", "nim c -r tests/treadme.nim", url = "https://github.com/alaviss/union"
+pkg "unittest2"
 pkg "unpack"
-# pkg "winim", "", true
+pkg "weave", "nimble install -y cligen@#HEAD; nimble test_gc_arc", useHead = true
+pkg "websock"
+pkg "websocket", "nim c websocket.nim"
+# pkg "winim", allowFailure = true
 pkg "with"
-pkg "ws"
+pkg "ws", allowFailure = true
 pkg "yaml"
 pkg "zero_functional", "nim c -r test.nim"
+pkg "zippy"
+pkg "zxcvbn"
diff --git a/testament/lib/readme.md b/testament/lib/readme.md
new file mode 100644
index 000000000..20e866338
--- /dev/null
+++ b/testament/lib/readme.md
@@ -0,0 +1,4 @@
+This directory contains helper files used by several tests, to avoid
+code duplication, akin to a std extension tailored for testament.
+
+Some of these could later migrate to stdlib.
diff --git a/testament/lib/stdtest/netutils.nim b/testament/lib/stdtest/netutils.nim
new file mode 100644
index 000000000..5115390e0
--- /dev/null
+++ b/testament/lib/stdtest/netutils.nim
@@ -0,0 +1,13 @@
+import std/[nativesockets, asyncdispatch, os]
+
+proc bindAvailablePort*(handle: SocketHandle, port = Port(0)): Port =
+  ## See also `asynchttpserver.getPort`.
+  block:
+    var name: Sockaddr_in
+    name.sin_family = typeof(name.sin_family)(toInt(AF_INET))
+    name.sin_port = htons(uint16(port))
+    name.sin_addr.s_addr = htonl(INADDR_ANY)
+    if bindAddr(handle, cast[ptr SockAddr](addr(name)),
+                sizeof(name).Socklen) < 0'i32:
+      raiseOSError(osLastError(), $port)
+  result = getLocalAddr(handle, AF_INET)[1]
diff --git a/testament/lib/stdtest/specialpaths.nim b/testament/lib/stdtest/specialpaths.nim
index 3c8b2338c..e214d113d 100644
--- a/testament/lib/stdtest/specialpaths.nim
+++ b/testament/lib/stdtest/specialpaths.nim
@@ -1,30 +1,54 @@
 #[
 todo: move findNimStdLibCompileTime, findNimStdLib here
+xxx: factor pending https://github.com/timotheecour/Nim/issues/616
+
+## note: $lib vs $nim
+note: these can resolve to 3 different paths if running via `nim c --lib:lib foo`,
+eg if compiler was installed via nimble (or is in nim path), and nim is external
+(ie not in `$lib/../bin/` dir)
+
+import "$lib/../compiler/nimpaths" # <- most robust if you want to favor --lib:lib
+import "$nim/compiler/nimpaths"
+import compiler/nimpaths
 ]#
 
 import os
+when defined(nimPreviewSlimSystem):
+  import std/assertions
 
 # Note: all the const paths defined here are known at compile time and valid
 # so long Nim repo isn't relocated after compilation.
 # This means the binaries they produce will embed hardcoded paths, which
 # isn't appropriate for some applications that need to be relocatable.
 
-const sourcePath = currentSourcePath()
-  # robust way to derive other paths here
-  # We don't depend on PATH so this is robust to having multiple nim
-  # binaries
-
-const nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir
-  ## root of Nim repo
-
-const stdlibDir* = nimRootDir / "lib"
-  # todo: make nimeval.findNimStdLibCompileTime use this
-
-const systemPath* = stdlibDir / "system.nim"
-
-const buildDir* = nimRootDir / "build"
-  ## refs #10268: all testament generated files should go here to avoid
-  ## polluting .gitignore
+const
+  sourcePath = currentSourcePath()
+    # robust way to derive other paths here
+    # We don't depend on PATH so this is robust to having multiple nim binaries
+  nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir ## root of Nim repo
+  testsFname* = "tests"
+  stdlibDir* = nimRootDir / "lib"
+  systemPath* = stdlibDir / "system.nim"
+  testsDir* = nimRootDir / testsFname
+  buildDir* = nimRootDir / "build"
+    ## refs #10268: all testament generated files should go here to avoid
+    ## polluting .gitignore
+
+proc splitTestFile*(file: string): tuple[cat: string, path: string] =
+  ## At least one directory is required in the path, to use as a category name
+  runnableExamples:
+    doAssert splitTestFile("tests/fakedir/tfakename.nim") == ("fakedir", "tests/fakedir/tfakename.nim".unixToNativePath)
+  for p in file.parentDirs(inclusive = false):
+    let parent = p.parentDir
+    if parent.lastPathPart == testsFname:
+      result.cat = p.lastPathPart
+      let dir = getCurrentDir()
+      if file.isRelativeTo(dir):
+        result.path = file.relativePath(dir)
+      else:
+        result.path = file
+      return result
+  raiseAssert "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'"
 
 static:
   # sanity check
diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim
new file mode 100644
index 000000000..a490b17c8
--- /dev/null
+++ b/testament/lib/stdtest/testutils.nim
@@ -0,0 +1,126 @@
+import std/private/miscdollars
+when defined(nimscript):
+  import std/os # xxx investigate why needed
+else:
+  from std/os import getEnv
+import std/[macros, genasts]
+
+template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) =
+  ## API to deal with flaky or failing tests. This avoids disabling entire tests
+  ## altogether so that at least the parts that are working are kept being
+  ## tested. This also avoids making CI fail periodically for tests known to
+  ## be flaky. Finally, for known failures, passing `notifySuccess = true` will
+  ## log that the test succeeded, which may indicate that a bug was fixed
+  ## "by accident" and should be looked into.
+  const info = instantiationInfo(-1, true)
+  const expr = astToStr(cond)
+  if cond and not notifySuccess:
+    discard # silent success
+  else:
+    var msg2 = ""
+    toLocation(msg2, info.filename, info.line, info.column)
+    if cond:
+      # a flaky test is failing, we still report it but we don't fail CI
+      msg2.add " FLAKY_SUCCESS "
+    else:
+      # a previously failing test is now passing, a pre-existing bug might've been
+      # fixed by accidend
+      msg2.add " FLAKY_FAILURE "
+    msg2.add $expr & " " & msg
+    echo msg2
+
+when not defined(js) and not defined(nimscript):
+  import std/strutils
+
+  proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool =
+    ## Returns true if each stripped line in `lhs` appears in rhs, using a greedy matching.
+    # xxx improve error reporting by showing the last matched pair
+    iterator splitLinesClosure(): string {.closure.} =
+      for line in splitLines(rhs.strip):
+        yield line
+    template isMatch(lhsi, rhsi): bool =
+      if allowPrefixMatch:
+        startsWith(rhsi, lhsi)
+      else:
+        lhsi == rhsi
+
+    var rhsIter = splitLinesClosure
+    var currentLine = strip(rhsIter())
+
+    for line in lhs.strip.splitLines:
+      let line = line.strip
+      if line.len != 0:
+        while not isMatch(line, currentLine):
+          currentLine = strip(rhsIter())
+          if rhsIter.finished:
+            return false
+
+      if rhsIter.finished:
+        return false
+    return true
+
+template enableRemoteNetworking*: bool =
+  ## Allows contolling whether to run some test at a statement-level granularity.
+  ## Using environment variables simplifies propagating this all the way across
+  ## process calls, e.g. `testament all` calls itself, which in turns invokes
+  ## a `nim` invocation (possibly via additional intermediate processes).
+  getEnv("NIM_TESTAMENT_REMOTE_NETWORKING") == "1"
+
+template disableSSLTesting*: bool =
+  ## TODO: workaround for GitHub Action gcc 14 matrix; remove this
+  ## matrix and the flag after Azure agent supports ubuntu 24.04
+  getEnv("NIM_TESTAMENT_DISABLE_SSL") == "1"
+
+template whenRuntimeJs*(bodyIf, bodyElse) =
+  ##[
+  Behaves as `when defined(js) and not nimvm` (which isn't legal yet).
+  pending improvements to `nimvm`, this sugar helps; use as follows:
+
+  whenRuntimeJs:
+    doAssert defined(js)
+    when nimvm: doAssert false
+    else: discard
+  do:
+    discard
+  ]##
+  when nimvm: bodyElse
+  else:
+    when defined(js): bodyIf
+    else: bodyElse
+
+template whenVMorJs*(bodyIf, bodyElse) =
+  ## Behaves as: `when defined(js) or nimvm`
+  when nimvm: bodyIf
+  else:
+    when defined(js): bodyIf
+    else: bodyElse
+
+template accept*(a) =
+  doAssert compiles(a)
+
+template reject*(a) =
+  doAssert not compiles(a)
+
+template disableVm*(body) =
+  when nimvm: discard
+  else: body
+
+macro assertAll*(body) =
+  ## works in VM, unlike `check`, `require`
+  runnableExamples:
+    assertAll:
+      1+1 == 2
+      var a = @[1, 2] # statements work
+      a.len == 2
+  # remove this once these support VM, pending #10129 (closed but not yet fixed)
+  result = newStmtList()
+  for a in body:
+    result.add genAst(a, a2 = a.repr, info = lineInfo(a)) do:
+      # D20210421T014713:here
+      # xxx pending https://github.com/nim-lang/Nim/issues/12030,
+      # `typeof` should introduce its own scope, so that this
+      # is sufficient: `typeof(a)` instead of `typeof(block: a)`
+      when typeof(block: a) is void: a
+      else:
+        if not a:
+          raise newException(AssertionDefect, info & " " & a2)
diff --git a/testament/lib/stdtest/unittest_light.nim b/testament/lib/stdtest/unittest_light.nim
new file mode 100644
index 000000000..4ab1d7543
--- /dev/null
+++ b/testament/lib/stdtest/unittest_light.nim
@@ -0,0 +1,37 @@
+import std/assertions
+
+
+proc mismatch*[T](lhs: T, rhs: T): string =
+  ## Simplified version of `unittest.require` that satisfies a common use case,
+  ## while avoiding pulling too many dependencies. On failure, diagnostic
+  ## information is provided that in particular makes it easy to spot
+  ## whitespace mismatches and where the mismatch is.
+  proc replaceInvisible(s: string): string =
+    for a in s:
+      case a
+      of '\n': result.add "\\n\n"
+      else: result.add a
+
+  proc quoted(s: string): string = result.addQuoted s
+
+  result.add '\n'
+  result.add "lhs:{" & replaceInvisible(
+      $lhs) & "}\nrhs:{" & replaceInvisible($rhs) & "}\n"
+  when compiles(lhs.len):
+    if lhs.len != rhs.len:
+      result.add "lhs.len: " & $lhs.len & " rhs.len: " & $rhs.len & '\n'
+    when compiles(lhs[0]):
+      var i = 0
+      while i < lhs.len and i < rhs.len:
+        if lhs[i] != rhs[i]: break
+        i.inc
+      result.add "first mismatch index: " & $i & '\n'
+      if i < lhs.len and i < rhs.len:
+        result.add "lhs[i]: {" & quoted($lhs[i]) & "}\nrhs[i]: {" & quoted(
+            $rhs[i]) & "}\n"
+      result.add "lhs[0..<i]:{" & replaceInvisible($lhs[
+          0..<i]) & '}'
+
+proc assertEquals*[T](lhs: T, rhs: T, msg = "") =
+  if lhs != rhs:
+    doAssert false, mismatch(lhs, rhs) & '\n' & msg
diff --git a/testament/specs.nim b/testament/specs.nim
index 70be6990c..c3040c1d8 100644
--- a/testament/specs.nim
+++ b/testament/specs.nim
@@ -7,7 +7,17 @@
 #    distribution, for details about the copyright.
 #
 
-import sequtils, parseutils, strutils, os, streams, parsecfg
+import sequtils, parseutils, strutils, os, streams, parsecfg,
+  tables, hashes, sets
+import compiler/platform
+
+type TestamentData* = ref object
+  # better to group globals under 1 object; could group the other ones here too
+  batchArg*: string
+  testamentNumBatch*: int
+  testamentBatch*: int
+
+let testamentData0* = TestamentData()
 
 var compilerPrefix* = findExe("nim")
 
@@ -25,60 +35,77 @@ type
 
   TOutputCheck* = enum
     ocIgnore = "ignore"
-    ocEqual  = "equal"
+    ocEqual = "equal"
     ocSubstr = "substr"
 
   TResultEnum* = enum
-    reNimcCrash,     # nim compiler seems to have crashed
-    reMsgsDiffer,       # error messages differ
-    reFilesDiffer,      # expected and given filenames differ
-    reLinesDiffer,      # expected and given line numbers differ
+    reNimcCrash,       # nim compiler seems to have crashed
+    reMsgsDiffer,      # error messages differ
+    reFilesDiffer,     # expected and given filenames differ
+    reLinesDiffer,     # expected and given line numbers differ
     reOutputsDiffer,
-    reExitcodesDiffer,
+    reExitcodesDiffer, # exit codes of program or of valgrind differ
     reTimeout,
     reInvalidPeg,
     reCodegenFailure,
     reCodeNotFound,
     reExeNotFound,
-    reInstallFailed     # package installation failed
-    reBuildFailed       # package building failed
-    reDisabled,         # test is disabled
-    reJoined,           # test is disabled because it was joined into the megatest
-    reSuccess           # test was successful
-    reInvalidSpec       # test had problems to parse the spec
+    reInstallFailed    # package installation failed
+    reBuildFailed      # package building failed
+    reDisabled,        # test is disabled
+    reJoined,          # test is disabled because it was joined into the megatest
+    reSuccess          # test was successful
+    reInvalidSpec      # test had problems to parse the spec
 
   TTarget* = enum
-    targetC = "C"
-    targetCpp = "C++"
-    targetObjC = "ObjC"
-    targetJS = "JS"
+    targetC = "c"
+    targetCpp = "cpp"
+    targetObjC = "objc"
+    targetJS = "js"
+
+  InlineError* = object
+    kind*: string
+    msg*: string
+    line*, col*: int
+
+  ValgrindSpec* = enum
+    disabled, enabled, leaking
 
   TSpec* = object
+    # xxx make sure `isJoinableSpec` takes into account each field here.
     action*: TTestAction
     file*, cmd*: string
+    filename*: string ## Test filename (without path).
     input*: string
     outputCheck*: TOutputCheck
     sortoutput*: bool
     output*: string
     line*, column*: int
-    tfile*: string
-    tline*, tcolumn*: int
     exitCode*: int
     msg*: string
-    ccodeCheck*: string
+    ccodeCheck*: seq[string]
     maxCodeSize*: int
     err*: TResultEnum
+    inCurrentBatch*: bool
     targets*: set[TTarget]
+    matrix*: seq[string]
     nimout*: string
-    parseErrors*: string # when the spec definition is invalid, this is not empty.
+    nimoutFull*: bool # whether nimout is all compiler output or a subset
+    parseErrors*: string            # when the spec definition is invalid, this is not empty.
     unjoinable*: bool
-    useValgrind*: bool
+    unbatchable*: bool
+      # whether this test can be batchable via `NIM_TESTAMENT_BATCH`; only very
+      # few tests are not batchable; the ones that are not could be turned batchable
+      # by making the dependencies explicit
+    useValgrind*: ValgrindSpec
     timeout*: float # in seconds, fractions possible,
-                    # but don't rely on much precision
+                      # but don't rely on much precision
+    inlineErrors*: seq[InlineError] # line information to error message
+    debugInfo*: string # debug info to give more context
 
 proc getCmd*(s: TSpec): string =
   if s.cmd.len == 0:
-    result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file"
+    result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:build/deps/pkgs2 $options $file"
   else:
     result = s.cmd
 
@@ -86,6 +113,13 @@ const
   targetToExt*: array[TTarget, string] = ["nim.c", "nim.cpp", "nim.m", "js"]
   targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"]
 
+proc defaultOptions*(a: TTarget): string =
+  case a
+  of targetJS: "-d:nodejs"
+    # once we start testing for `nim js -d:nimbrowser` (eg selenium or similar),
+    # we can adapt this logic; or a given js test can override with `-u:nodejs`.
+  else: ""
+
 when not declared(parseCfgBool):
   # candidate for the stdlib:
   proc parseCfgBool(s: string): bool =
@@ -94,21 +128,174 @@ when not declared(parseCfgBool):
     of "n", "no", "false", "0", "off": result = false
     else: raise newException(ValueError, "cannot interpret as a bool: " & s)
 
-proc extractSpec(filename: string): string =
-  const tripleQuote = "\"\"\""
-  var x = readFile(filename).string
-  var a = x.find(tripleQuote)
-  var b = x.find(tripleQuote, a+3)
-  # look for """ only in the first section
-  if a >= 0 and b > a and a < 40:
-    result = x.substr(a+3, b-1).replace("'''", tripleQuote)
+proc addLine*(self: var string; pieces: varargs[string]) =
+  for piece in pieces:
+    self.add piece
+  self.add "\n"
+
+
+const
+  inlineErrorKindMarker = "tt."
+  inlineErrorMarker = "#[" & inlineErrorKindMarker
+
+proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int =
+  ## Extract inline error messages.
+  ##
+  ## Can parse a single message for a line:
+  ##
+  ##   ```nim
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name] ]#
+  ##   ```
+  ##
+  ## Can parse multiple messages for a line when they are separated by ';':
+  ##
+  ##   ```nim
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error
+  ##                           ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error
+  ##                                       ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
+  ##   ```
+  ##
+  ##   ```nim
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name];
+  ##     tt.Error              ^ 'no_destroy' should be: 'nodestroy' [Name];
+  ##     tt.Error                          ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
+  ##   ```
+  result = i + len(inlineErrorMarker)
+  inc col, len(inlineErrorMarker)
+  let msgLine = line
+  var msgCol = -1
+  var msg = ""
+  var kind = ""
+
+  template parseKind =
+    while result < s.len and s[result] in IdentChars:
+      kind.add s[result]
+      inc result
+      inc col
+    if kind notin ["Hint", "Warning", "Error"]:
+      spec.parseErrors.addLine "expected inline message kind: Hint, Warning, Error"
+
+  template skipWhitespace =
+    while result < s.len and s[result] in Whitespace:
+      if s[result] == '\n':
+        col = 1
+        inc line
+      else:
+        inc col
+      inc result
+
+  template parseCaret =
+    if result < s.len and s[result] == '^':
+      msgCol = col
+      inc result
+      inc col
+      skipWhitespace()
+    else:
+      spec.parseErrors.addLine "expected column marker ('^') for inline message"
+
+  template isMsgDelimiter: bool =
+    s[result] == ';' and
+    (block:
+      let nextTokenIdx = result + 1 + parseutils.skipWhitespace(s, result + 1)
+      if s.len > nextTokenIdx + len(inlineErrorKindMarker) and
+         s[nextTokenIdx..(nextTokenIdx + len(inlineErrorKindMarker) - 1)] == inlineErrorKindMarker:
+        true
+      else:
+        false)
+
+  template trimTrailingMsgWhitespace =
+    while msg.len > 0 and msg[^1] in Whitespace:
+      setLen msg, msg.len - 1
+
+  template addInlineError =
+    doAssert msg[^1] notin Whitespace
+    if kind == "Error": spec.action = actionReject
+    spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: msgLine, col: msgCol)
+
+  parseKind()
+  skipWhitespace()
+  parseCaret()
+
+  while result < s.len-1:
+    if s[result] == '\n':
+      if result > 0 and s[result - 1] == '\r':
+        msg[^1] = '\n'
+      else:
+        msg.add '\n'
+      inc result
+      inc line
+      col = 1
+    elif isMsgDelimiter():
+      trimTrailingMsgWhitespace()
+      inc result
+      skipWhitespace()
+      addInlineError()
+      inc result, len(inlineErrorKindMarker)
+      inc col, 1 + len(inlineErrorKindMarker)
+      kind.setLen 0
+      msg.setLen 0
+      parseKind()
+      skipWhitespace()
+      parseCaret()
+    elif s[result] == ']' and s[result+1] == '#':
+      trimTrailingMsgWhitespace()
+      inc result, 2
+      inc col, 2
+      addInlineError()
+      break
+    else:
+      msg.add s[result]
+      inc result
+      inc col
+
+  if spec.inlineErrors.len > 0:
+    spec.unjoinable = true
+
+proc extractSpec(filename: string; spec: var TSpec): string =
+  const
+    tripleQuote = "\"\"\""
+    specStart = "discard " & tripleQuote
+  var s = readFile(filename)
+
+  var i = 0
+  var a = -1
+  var b = -1
+  var line = 1
+  var col = 1
+  while i < s.len:
+    if (i == 0 or s[i-1] != ' ') and s.continuesWith(specStart, i):
+      # `s[i-1] == '\n'` would not work because of `tests/stdlib/tbase64.nim` which contains BOM (https://en.wikipedia.org/wiki/Byte_order_mark)
+      const lineMax = 10
+      if a != -1:
+        raise newException(ValueError, "testament spec violation: duplicate `specStart` found: " & $(filename, a, b, line))
+      elif line > lineMax:
+        # not overly restrictive, but prevents mistaking some `specStart` as spec if deeep inside a test file
+        raise newException(ValueError, "testament spec violation: `specStart` should be before line $1, or be indented; info: $2" % [$lineMax, $(filename, a, b, line)])
+      i += specStart.len
+      a = i
+    elif a > -1 and b == -1 and s.continuesWith(tripleQuote, i):
+      b = i
+      i += tripleQuote.len
+    elif s[i] == '\n':
+      inc line
+      inc i
+      col = 1
+    elif s.continuesWith(inlineErrorMarker, i):
+      i = extractErrorMsg(s, i, line, col, spec)
+    else:
+      inc col
+      inc i
+
+  if a >= 0 and b > a:
+    result = s.substr(a, b-1).multiReplace({"'''": tripleQuote, "\\31": "\31"})
+  elif a >= 0:
+    raise newException(ValueError, "testament spec violation: `specStart` found but not trailing `tripleQuote`: $1" % $(filename, a, b, line))
   else:
-    #echo "warning: file does not contain spec: " & filename
     result = ""
 
-when not defined(nimhygiene):
-  {.pragma: inject.}
-
 proc parseTargets*(value: string): set[TTarget] =
   for v in value.normalize.splitWhitespace:
     case v
@@ -116,31 +303,38 @@ proc parseTargets*(value: string): set[TTarget] =
     of "cpp", "c++": result.incl(targetCpp)
     of "objc": result.incl(targetObjC)
     of "js": result.incl(targetJS)
-    else: echo "target ignored: " & v
-
-proc addLine*(self: var string; a: string) =
-  self.add a
-  self.add "\n"
-
-proc addLine*(self: var string; a,b: string) =
-  self.add a
-  self.add b
-  self.add "\n"
+    else: raise newException(ValueError, "invalid target: '$#'" % v)
 
 proc initSpec*(filename: string): TSpec =
   result.file = filename
 
+proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool =
+  if testamentData.testamentNumBatch != 0:
+    hash(filename) mod testamentData.testamentNumBatch == testamentData.testamentBatch
+  else:
+    true
+
 proc parseSpec*(filename: string): TSpec =
   result.file = filename
-  let specStr = extractSpec(filename)
+  result.filename = extractFilename(filename)
+  let specStr = extractSpec(filename, result)
   var ss = newStringStream(specStr)
   var p: CfgParser
   open(p, ss, filename, 1)
+  var flags: HashSet[string]
+  var nimoutFound = false
   while true:
     var e = next(p)
     case e.kind
     of cfgKeyValuePair:
-      case normalize(e.key)
+      let key = e.key.normalize
+      const whiteListMulti = ["disabled", "ccodecheck"]
+        ## list of flags that are correctly handled when passed multiple times
+        ## (instead of being overwritten)
+      if key notin whiteListMulti:
+        doAssert key notin flags, $(key, filename)
+      flags.incl key
+      case key
       of "action":
         case e.value.normalize
         of "compile":
@@ -163,16 +357,10 @@ proc parseSpec*(filename: string): TSpec =
         if result.msg.len == 0 and result.nimout.len == 0:
           result.parseErrors.addLine "errormsg or msg needs to be specified before column"
         discard parseInt(e.value, result.column)
-      of "tfile":
-        result.tfile = e.value
-      of "tline":
-        discard parseInt(e.value, result.tline)
-      of "tcolumn":
-        discard parseInt(e.value, result.tcolumn)
       of "output":
         if result.outputCheck != ocSubstr:
           result.outputCheck = ocEqual
-        result.output = strip(e.value)
+        result.output = e.value
       of "input":
         result.input = e.value
       of "outputsub":
@@ -180,71 +368,108 @@ proc parseSpec*(filename: string): TSpec =
         result.output = strip(e.value)
       of "sortoutput":
         try:
-          result.sortoutput  = parseCfgBool(e.value)
+          result.sortoutput = parseCfgBool(e.value)
         except:
           result.parseErrors.addLine getCurrentExceptionMsg()
       of "exitcode":
         discard parseInt(e.value, result.exitCode)
         result.action = actionRun
-      of "msg":
-        result.msg = e.value
-        if result.action != actionRun:
-          result.action = actionCompile
-      of "errormsg", "errmsg":
+      of "errormsg":
         result.msg = e.value
         result.action = actionReject
       of "nimout":
         result.nimout = e.value
+        nimoutFound = true
+      of "nimoutfull":
+        result.nimoutFull = parseCfgBool(e.value)
+      of "batchable":
+        result.unbatchable = not parseCfgBool(e.value)
       of "joinable":
         result.unjoinable = not parseCfgBool(e.value)
       of "valgrind":
         when defined(linux) and sizeof(int) == 8:
-          result.useValgrind = parseCfgBool(e.value)
+          result.useValgrind = if e.value.normalize == "leaks": leaking
+                               else: ValgrindSpec(parseCfgBool(e.value))
           result.unjoinable = true
-          if result.useValgrind:
+          if result.useValgrind != disabled:
             result.outputCheck = ocSubstr
         else:
           # Windows lacks valgrind. Silly OS.
           # Valgrind only supports OSX <= 17.x
-          result.useValgrind = false
+          result.useValgrind = disabled
       of "disabled":
-        case e.value.normalize
+        let value = e.value.normalize
+        case value
         of "y", "yes", "true", "1", "on": result.err = reDisabled
         of "n", "no", "false", "0", "off": discard
-        of "win", "windows":
+        # These values are defined in `compiler/options.isDefined`
+        of "win":
           when defined(windows): result.err = reDisabled
         of "linux":
           when defined(linux): result.err = reDisabled
         of "bsd":
           when defined(bsd): result.err = reDisabled
-        of "macosx":
-          when defined(macosx): result.err = reDisabled
-        of "unix":
-          when defined(unix): result.err = reDisabled
-        of "posix":
+        of "osx":
+          when defined(osx): result.err = reDisabled
+        of "unix", "posix":
           when defined(posix): result.err = reDisabled
-        of "travis":
+        of "freebsd":
+          when defined(freebsd): result.err = reDisabled
+        of "littleendian":
+          when defined(littleendian): result.err = reDisabled
+        of "bigendian":
+          when defined(bigendian): result.err = reDisabled
+        of "cpu8", "8bit":
+          when defined(cpu8): result.err = reDisabled
+        of "cpu16", "16bit":
+          when defined(cpu16): result.err = reDisabled
+        of "cpu32", "32bit":
+          when defined(cpu32): result.err = reDisabled
+        of "cpu64", "64bit":
+          when defined(cpu64): result.err = reDisabled
+        # These values are for CI environments
+        of "travis": # deprecated
           if isTravis: result.err = reDisabled
-        of "appveyor":
+        of "appveyor": # deprecated
           if isAppVeyor: result.err = reDisabled
         of "azure":
           if isAzure: result.err = reDisabled
-        of "32bit":
-          if sizeof(int) == 4:
-            result.err = reDisabled
-        of "freebsd":
-          when defined(freebsd): result.err = reDisabled
-        of "arm64":
-          when defined(arm64): result.err = reDisabled
         else:
-          result.parseErrors.addLine "cannot interpret as a bool: ", e.value
+          # Check whether the value exists as an OS or CPU that is
+          # defined in `compiler/platform`.
+          block checkHost:
+            for os in platform.OS:
+              # Check if the value exists as OS.
+              if value == os.name.normalize:
+                # The value exists; is it the same as the current host?
+                if value == hostOS.normalize:
+                  # The value exists and is the same as the current host,
+                  # so disable the test.
+                  result.err = reDisabled
+                # The value was defined, so there is no need to check further
+                # values or raise an error.
+                break checkHost
+            for cpu in platform.CPU:
+              # Check if the value exists as CPU.
+              if value == cpu.name.normalize:
+                # The value exists; is it the same as the current host?
+                if value == hostCPU.normalize:
+                  # The value exists and is the same as the current host,
+                  # so disable the test.
+                  result.err = reDisabled
+                # The value was defined, so there is no need to check further
+                # values or raise an error.
+                break checkHost
+            # The value doesn't exist as an OS, CPU, or any previous value
+            # defined in this case statement, so raise an error.
+            result.parseErrors.addLine "cannot interpret as a bool: ", e.value
       of "cmd":
         if e.value.startsWith("nim "):
           result.cmd = compilerPrefix & e.value[3..^1]
         else:
           result.cmd = e.value
       of "ccodecheck":
-        result.ccodeCheck = e.value
+        result.ccodeCheck.add e.value
       of "maxcodesize":
         discard parseInt(e.value, result.maxCodeSize)
       of "timeout":
@@ -252,19 +477,14 @@ proc parseSpec*(filename: string): TSpec =
           result.timeout = parseFloat(e.value)
         except ValueError:
           result.parseErrors.addLine "cannot interpret as a float: ", e.value
-      of "target", "targets":
-        for v in e.value.normalize.splitWhitespace:
-          case v
-          of "c":
-            result.targets.incl(targetC)
-          of "cpp", "c++":
-            result.targets.incl(targetCpp)
-          of "objc":
-            result.targets.incl(targetObjC)
-          of "js":
-            result.targets.incl(targetJS)
-          else:
-            result.parseErrors.addLine "cannot interpret as a target: ", e.value
+      of "targets", "target":
+        try:
+          result.targets.incl parseTargets(e.value)
+        except ValueError as e:
+          result.parseErrors.addLine e.msg
+      of "matrix":
+        for v in e.value.split(';'):
+          result.matrix.add(v.strip)
       else:
         result.parseErrors.addLine "invalid key for test spec: ", e.key
 
@@ -280,3 +500,22 @@ proc parseSpec*(filename: string): TSpec =
 
   if skips.anyIt(it in result.file):
     result.err = reDisabled
+
+  if nimoutFound and result.nimout.len == 0 and not result.nimoutFull:
+    result.parseErrors.addLine "empty `nimout` is vacuously true, use `nimoutFull:true` if intentional"
+
+  result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable
+  if not result.inCurrentBatch:
+    result.err = reDisabled
+
+  # Interpolate variables in msgs:
+  template varSub(msg: string): string =
+    try:
+      msg % ["/", $DirSep, "file", result.filename]
+    except ValueError:
+      result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unit-tests-output-message-variable-interpolation')"
+      msg
+  result.nimout = result.nimout.varSub
+  result.msg = result.msg.varSub
+  for inlineError in result.inlineErrors.mitems:
+    inlineError.msg = inlineError.msg.varSub
diff --git a/testament/testament.nim b/testament/testament.nim
index b19f13256..1e892e636 100644
--- a/testament/testament.nim
+++ b/testament/testament.nim
@@ -10,87 +10,114 @@
 ## This program verifies Nim against the testcases.
 
 import
-  strutils, pegs, os, osproc, streams, json,
-  backend, parseopt, specs, htmlgen, browsers, terminal,
-  algorithm, times, md5, sequtils, azure
+  std/[strutils, pegs, os, osproc, streams, json,
+    parseopt, browsers, terminal, exitprocs,
+    algorithm, times, intsets, macros]
 
-include compiler/nodejs
+import backend, specs, azure, htmlgen
+
+from std/sugar import dup
+import compiler/nodejs
+import lib/stdtest/testutils
+from lib/stdtest/specialpaths import splitTestFile
+from std/private/gitutils import diffStrings
+
+import ../dist/checksums/src/checksums/md5
+
+proc trimUnitSep(x: var string) =
+  let L = x.len
+  if L > 0 and x[^1] == '\31':
+    setLen x, L-1
 
 var useColors = true
 var backendLogging = true
 var simulate = false
+var optVerbose = false
+var useMegatest = true
+var valgrindEnabled = true
+
+proc verboseCmd(cmd: string) =
+  if optVerbose:
+    echo "executing: ", cmd
 
 const
+  failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs
   testsDir = "tests" & DirSep
   resultsFile = "testresults.html"
-  #jsonFile = "testresults.json" # not used
   Usage = """Usage:
   testament [options] command [arguments]
 
 Command:
   p|pat|pattern <glob>        run all the tests matching the given pattern
-  all                         run all tests
+  all                         run all tests in category folders
   c|cat|category <category>   run all the tests of a certain category
   r|run <test>                run single test file
   html                        generate $1 from the database
-  stats                       generate statistics about test cases
 Arguments:
   arguments are passed to the compiler
 Options:
-  --print                   also print results to the console
+  --print                   print results to the console
+  --verbose                 print commands (compiling and running tests)
   --simulate                see what tests would be run but don't run them (for debugging)
   --failing                 only show failing/ignored tests
-  --targets:"c c++ js objc" run tests for specified targets (default: all)
+  --targets:"c cpp js objc" run tests for specified targets (default: c)
   --nim:path                use a particular nim executable (default: $$PATH/nim)
   --directory:dir           Change to directory dir before reading the tests or doing anything else.
-  --colors:on|off           Turn messagescoloring on|off.
+  --colors:on|off           Turn messages coloring on|off.
   --backendLogging:on|off   Disable or enable backend logging. By default turned on.
   --megatest:on|off         Enable or disable megatest. Default is on.
+  --valgrind:on|off         Enable or disable valgrind support. Default is on.
   --skipFrom:file           Read tests to skip from `file` - one test per line, # comments ignored
 
 On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API
 provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN.
+
+Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables
+tests with remote networking (as in CI).
 """ % resultsFile
 
+proc isNimRepoTests(): bool =
+  # this logic could either be specific to cwd, or to some file derived from
+  # the input file, eg testament r /pathto/tests/foo/tmain.nim; we choose
+  # the former since it's simpler and also works with `testament all`.
+  let file = "testament"/"testament.nim.cfg"
+  result = file.fileExists
+
 type
   Category = distinct string
   TResults = object
-    total, passed, skipped: int
+    total, passed, failedButAllowed, skipped: int
+      ## xxx rename passed to passedOrAllowedFailure
     data: string
-
   TTest = object
     name: string
     cat: Category
     options: string
-    args: seq[string]
+    testArgs: seq[string]
     spec: TSpec
     startTime: float
+    debugInfo: string
 
 # ----------------------------------------------------------------------------
 
 let
   pegLineError =
     peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}"
-  pegLineTemplate =
-    peg"""
-      {[^(]*} '(' {\d+} ', ' {\d+} ') '
-      'template/generic instantiation' ( ' of `' [^`]+ '`' )? ' from here' .*
-    """
   pegOtherError = peg"'Error:' \s* {.*}"
   pegOfInterest = pegLineError / pegOtherError
 
 var gTargets = {low(TTarget)..high(TTarget)}
+var targetsSet = false
 
 proc isSuccess(input: string): bool =
   # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs
+  # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we
+  # could handle colors, there would be other issues such as handling other flags
+  # that may appear in user config (eg: `--filenames`).
+  # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config
+  # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory
   input.startsWith("Hint: ") and input.endsWith("[SuccessX]")
 
-proc normalizeMsg(s: string): string =
-  result = newStringOfCap(s.len+1)
-  for x in splitLines(s):
-    if result.len > 0: result.add '\L'
-    result.add x.strip
-
 proc getFileDir(filename: string): string =
   result = filename.splitFile().dir
   if not result.isAbsolute():
@@ -98,7 +125,7 @@ proc getFileDir(filename: string): string =
 
 proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: string = ""): tuple[
                 cmdLine: string,
-                output: TaintedString,
+                output: string,
                 exitCode: int] {.tags:
                 [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
 
@@ -106,8 +133,9 @@ proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: str
   for arg in args:
     result.cmdLine.add ' '
     result.cmdLine.add quoteShell(arg)
-  var p = startProcess(command, workingDir=workingDir, args=args,
-                       options={poStdErrToStdOut, poUsePath})
+  verboseCmd(result.cmdLine)
+  var p = startProcess(command, workingDir = workingDir, args = args,
+                       options = {poStdErrToStdOut, poUsePath})
   var outp = outputStream(p)
 
   # There is no way to provide input for the child process
@@ -117,12 +145,12 @@ proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: str
   instream.write(input)
   close instream
 
-  result.exitCode =  -1
-  var line = newStringOfCap(120).TaintedString
+  result.exitCode = -1
+  var line = newStringOfCap(120)
   while true:
     if outp.readLine(line):
-      result.output.string.add(line.string)
-      result.output.string.add("\n")
+      result.output.add line
+      result.output.add '\n'
     else:
       result.exitCode = peekExitCode(p)
       if result.exitCode != -1: break
@@ -133,54 +161,64 @@ proc nimcacheDir(filename, options: string, target: TTarget): string =
   let hashInput = options & $target
   result = "nimcache" / (filename & '_' & hashInput.getMD5)
 
-proc prepareTestArgs(cmdTemplate, filename, options, nimcache: string,
-                     target: TTarget, extraOptions=""): seq[string] =
-  let options = options & " " & quoteShell("--nimCache:" & nimcache) & extraOptions
-  result = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
+proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string,
+                     target: TTarget, extraOptions = ""): string =
+  var options = target.defaultOptions & ' ' & options
+  if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell)
+  options.add ' ' & extraOptions
+  # we avoid using `parseCmdLine` which is buggy, refs bug #14343
+  result = cmdTemplate % ["target", targetToCmd[target],
                       "options", options, "file", filename.quoteShell,
-                      "filedir", filename.getFileDir()])
+                      "filedir", filename.getFileDir(), "nim", compilerPrefix]
 
-proc callCompiler(cmdTemplate, filename, options, nimcache: string,
-                  target: TTarget,
-                  extraOptions=""): TSpec =
-  let c = prepareTestArgs(cmdTemplate, filename, options, nimcache, target,
+proc callNimCompiler(cmdTemplate, filename, options, nimcache: string,
+                     target: TTarget, extraOptions = ""): TSpec =
+  result.cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache, target,
                           extraOptions)
-  result.cmd = quoteShellCommand(c)
-  var p = startProcess(command=c[0], args=c[1 .. ^1],
-                       options={poStdErrToStdOut, poUsePath})
+  verboseCmd(result.cmd)
+  var p = startProcess(command = result.cmd,
+                       options = {poStdErrToStdOut, poUsePath, poEvalCommand})
   let outp = p.outputStream
-  var suc = ""
+  var foundSuccessMsg = false
+  var foundErrorMsg = false
   var err = ""
-  var tmpl = ""
   var x = newStringOfCap(120)
   result.nimout = ""
   while true:
-    if outp.readLine(x.TaintedString):
-      result.nimout.add(x & "\n")
+    if outp.readLine(x):
+      trimUnitSep x
+      result.nimout.add(x & '\n')
       if x =~ pegOfInterest:
-        # `err` should contain the last error/warning message
+        # `err` should contain the last error message
         err = x
-      elif x =~ pegLineTemplate and err == "":
-        # `tmpl` contains the last template expansion before the error
-        tmpl = x
+        foundErrorMsg = true
       elif x.isSuccess:
-        suc = x
+        foundSuccessMsg = true
     elif not running(p):
       break
-  close(p)
   result.msg = ""
   result.file = ""
   result.output = ""
   result.line = 0
   result.column = 0
-  result.tfile = ""
-  result.tline = 0
-  result.tcolumn = 0
+
   result.err = reNimcCrash
-  if tmpl =~ pegLineTemplate:
-    result.tfile = extractFilename(matches[0])
-    result.tline = parseInt(matches[1])
-    result.tcolumn = parseInt(matches[2])
+  result.exitCode = p.peekExitCode
+  close p
+  case result.exitCode
+  of 0:
+    if foundErrorMsg:
+      result.debugInfo.add " compiler exit code was 0 but some Error's were found."
+    else:
+      result.err = reSuccess
+  of 1:
+    if not foundErrorMsg:
+      result.debugInfo.add " compiler exit code was 1 but no Error's were found."
+    if foundSuccessMsg:
+      result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true."
+  else:
+    result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode
+
   if err =~ pegLineError:
     result.file = extractFilename(matches[0])
     result.line = parseInt(matches[1])
@@ -188,40 +226,15 @@ proc callCompiler(cmdTemplate, filename, options, nimcache: string,
     result.msg = matches[3]
   elif err =~ pegOtherError:
     result.msg = matches[0]
-  elif suc.isSuccess:
-    result.err = reSuccess
-
-proc callCCompiler(cmdTemplate, filename, options: string,
-                  target: TTarget): TSpec =
-  let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
-                       "options", options, "file", filename.quoteShell,
-                       "filedir", filename.getFileDir()])
-  var p = startProcess(command="gcc", args=c[5 .. ^1],
-                       options={poStdErrToStdOut, poUsePath})
-  let outp = p.outputStream
-  var x = newStringOfCap(120)
-  result.nimout = ""
-  result.msg = ""
-  result.file = ""
-  result.output = ""
-  result.line = -1
-  while true:
-    if outp.readLine(x.TaintedString):
-      result.nimout.add(x & "\n")
-    elif not running(p):
-      break
-  close(p)
-  if p.peekExitCode == 0:
-    result.err = reSuccess
+  trimUnitSep result.msg
 
 proc initResults: TResults =
   result.total = 0
   result.passed = 0
+  result.failedButAllowed = 0
   result.skipped = 0
   result.data = ""
 
-import macros
-
 macro ignoreStyleEcho(args: varargs[typed]): untyped =
   let typForegroundColor = bindSym"ForegroundColor".getType
   let typBackgroundColor = bindSym"BackgroundColor".getType
@@ -246,22 +259,34 @@ template maybeStyledEcho(args: varargs[untyped]): untyped =
 
 
 proc `$`(x: TResults): string =
-  result = ("Tests passed: $1 / $3 <br />\n" &
-            "Tests skipped: $2 / $3 <br />\n") %
-            [$x.passed, $x.skipped, $x.total]
+  result = """
+Tests passed or allowed to fail: $2 / $1 <br />
+Tests failed and allowed to fail: $3 / $1 <br />
+Tests skipped: $4 / $1 <br />
+""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped]
+
+proc testName(test: TTest, target: TTarget, extraOptions: string, allowFailure: bool): string =
+  var name = test.name.replace(DirSep, '/')
+  name.add ' ' & $target
+  if allowFailure:
+    name.add " (allowed to fail) "
+  if test.options.len > 0: name.add ' ' & test.options
+  if extraOptions.len > 0: name.add ' ' & extraOptions
+  name.strip()
 
 proc addResult(r: var TResults, test: TTest, target: TTarget,
-               expected, given: string, successOrig: TResultEnum) =
+               extraOptions, expected, given: string, successOrig: TResultEnum,
+               allowFailure = false, givenSpec: ptr TSpec = nil) =
+  # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need
+  # instead of having to pass individual fields, or abusing existing ones like expected vs given.
   # test.name is easier to find than test.name.extractFilename
   # A bit hacky but simple and works with tests/testament/tshould_not_work.nim
-  var name = test.name.replace(DirSep, '/')
-  name.add " " & $target & test.options
-
+  let name = testName(test, target, extraOptions, allowFailure)
   let duration = epochTime() - test.startTime
   let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout
                 else: successOrig
 
-  let durationStr = duration.formatFloat(ffDecimal, precision = 8).align(11)
+  let durationStr = duration.formatFloat(ffDecimal, precision = 2).align(5)
   if backendLogging:
     backend.writeTestResult(name = name,
                             category = test.cat.string,
@@ -271,16 +296,22 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
                             expected = expected,
                             given = given)
   r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
+  template dispNonSkipped(color, outcome) =
+    maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)"
+  template disp(msg) =
+    maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name
   if success == reSuccess:
-    maybeStyledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)"
+    dispNonSkipped(fgGreen, "PASS: ")
   elif success == reDisabled:
-    maybeStyledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name
-  elif success == reJoined:
-    maybeStyledEcho styleDim, fgYellow, "JOINED: ", styleBright, fgCyan, name
+    if test.spec.inCurrentBatch: disp("SKIP:")
+    else: disp("NOTINBATCH:")
+  elif success == reJoined: disp("JOINED:")
   else:
-    maybeStyledEcho styleBright, fgRed, "FAIL: ", fgCyan, name
+    dispNonSkipped(fgRed, failString)
     maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
     maybeStyledEcho styleBright, fgRed, "Failure: ", $success
+    if givenSpec != nil and givenSpec.debugInfo.len > 0:
+      echo "debugInfo: " & givenSpec.debugInfo
     if success in {reBuildFailed, reNimcCrash, reInstallFailed}:
       # expected is empty, no reason to print it.
       echo given
@@ -289,7 +320,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
       maybeStyledEcho styleBright, expected, "\n"
       maybeStyledEcho fgYellow, "Gotten:"
       maybeStyledEcho styleBright, given, "\n"
-
+      echo diffStrings(expected, given).output
 
   if backendLogging and (isAppVeyor or isAzure):
     let (outcome, msg) =
@@ -299,44 +330,66 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
       of reDisabled, reJoined:
         ("Skipped", "")
       of reBuildFailed, reNimcCrash, reInstallFailed:
-        ("Failed", "Failure: " & $success & "\n" & given)
+        ("Failed", "Failure: " & $success & '\n' & given)
       else:
         ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
     if isAzure:
       azure.addTestResult(name, test.cat.string, int(duration * 1000), msg, success)
     else:
-      var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options,
+      var p = startProcess("appveyor", args = ["AddTest", test.name.replace("\\", "/") & test.options,
                            "-Framework", "nim-testament", "-FileName",
                            test.cat.string,
                            "-Outcome", outcome, "-ErrorMessage", msg,
-                           "-Duration", $(duration*1000).int],
-                           options={poStdErrToStdOut, poUsePath, poParentStreams})
+                           "-Duration", $(duration * 1000).int],
+                           options = {poStdErrToStdOut, poUsePath, poParentStreams})
       discard waitForExit(p)
       close(p)
 
-proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) =
-  if strip(expected.msg) notin strip(given.msg):
-    r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer)
-  elif expected.nimout.len > 0 and expected.nimout.normalizeMsg notin given.nimout.normalizeMsg:
-    r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer)
-  elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and
+proc toString(inlineError: InlineError, filename: string): string =
+  result.add "$file($line, $col) $kind: $msg" % [
+    "file", filename,
+    "line", $inlineError.line,
+    "col", $inlineError.col,
+    "kind", $inlineError.kind,
+    "msg", $inlineError.msg
+  ]
+
+proc inlineErrorsMsgs(expected: TSpec): string =
+  for inlineError in expected.inlineErrors.items:
+    result.addLine inlineError.toString(expected.filename)
+
+proc checkForInlineErrors(expected, given: TSpec): bool =
+  for inlineError in expected.inlineErrors:
+    if inlineError.toString(expected.filename) notin given.nimout:
+      return false
+  true
+
+proc nimoutCheck(expected, given: TSpec): bool =
+  result = true
+  if expected.nimoutFull:
+    if expected.nimout != given.nimout:
+      result = false
+  elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout):
+    result = false
+
+proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest,
+             target: TTarget, extraOptions: string) =
+  if not checkForInlineErrors(expected, given) or
+    (not expected.nimoutFull and not nimoutCheck(expected, given)):
+      r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer)
+  elif strip(expected.msg) notin strip(given.msg):
+    r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer)
+  elif not nimoutCheck(expected, given):
+    r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer)
+  elif extractFilename(expected.file) != extractFilename(given.file) and
       "internal error:" notin expected.msg:
-    r.addResult(test, target, expected.file, given.file, reFilesDiffer)
+    r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer)
   elif expected.line != given.line and expected.line != 0 or
        expected.column != given.column and expected.column != 0:
-    r.addResult(test, target, $expected.line & ':' & $expected.column,
-                      $given.line & ':' & $given.column,
-                      reLinesDiffer)
-  elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and
-      "internal error:" notin expected.msg:
-    r.addResult(test, target, expected.tfile, given.tfile, reFilesDiffer)
-  elif expected.tline != given.tline and expected.tline != 0 or
-       expected.tcolumn != given.tcolumn and expected.tcolumn != 0:
-    r.addResult(test, target, $expected.tline & ':' & $expected.tcolumn,
-                      $given.tline & ':' & $given.tcolumn,
-                      reLinesDiffer)
+    r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column,
+                      $given.line & ':' & $given.column, reLinesDiffer)
   else:
-    r.addResult(test, target, expected.msg, given.msg, reSuccess)
+    r.addResult(test, target, extraOptions, expected.msg, given.msg, reSuccess)
     inc(r.passed)
 
 proc generatedFile(test: TTest, target: TTarget): string =
@@ -354,10 +407,9 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st
                   given: var TSpec) =
   try:
     let genFile = generatedFile(test, target)
-    let contents = readFile(genFile).string
-    let check = spec.ccodeCheck
-    if check.len > 0:
-      if check[0] == '\\':
+    let contents = readFile(genFile)
+    for check in spec.ccodeCheck:
+      if check.len > 0 and check[0] == '\\':
         # little hack to get 'match' support:
         if not contents.match(check.peg):
           given.err = reCodegenFailure
@@ -375,88 +427,78 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st
     given.err = reCodeNotFound
     echo getCurrentExceptionMsg()
 
-proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) =
-  let giv = given.nimout.strip
-  var currentPos = 0
-  # Only check that nimout contains all expected lines in that order.
-  # There may be more output in nimout. It is ignored here.
-  for line in expectedNimout.strip.splitLines:
-    currentPos = giv.find(line.strip, currentPos)
-    if currentPos < 0:
-      given.err = reMsgsDiffer
-      break
-
-proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
-                         expected: TSpec; r: var TResults) =
+proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string,
+                         given: var TSpec, expected: TSpec; r: var TResults) =
   var expectedmsg: string = ""
   var givenmsg: string = ""
   if given.err == reSuccess:
     if expected.needsCodegenCheck:
       codegenCheck(test, target, expected, expectedmsg, given)
       givenmsg = given.msg
-    if expected.nimout.len > 0:
-      expectedmsg = expected.nimout
+    if not nimoutCheck(expected, given) or
+       not checkForInlineErrors(expected, given):
+      given.err = reMsgsDiffer
+      expectedmsg = expected.nimout & inlineErrorsMsgs(expected)
       givenmsg = given.nimout.strip
-      nimoutCheck(test, expectedmsg, given)
   else:
-    givenmsg = "$ " & given.cmd & "\n" & given.nimout
+    givenmsg = "$ " & given.cmd & '\n' & given.nimout
   if given.err == reSuccess: inc(r.passed)
-  r.addResult(test, target, expectedmsg, givenmsg, given.err)
+  r.addResult(test, target, extraOptions, expectedmsg, givenmsg, given.err)
 
 proc getTestSpecTarget(): TTarget =
-  if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true":
+  if getEnv("NIM_COMPILE_TO_CPP", "false") == "true":
     result = targetCpp
   else:
     result = targetC
 
-proc checkDisabled(r: var TResults, test: TTest): bool =
-  if test.spec.err in {reDisabled, reJoined}:
-    # targetC is a lie, but parameter is required
-    r.addResult(test, targetC, "", "", test.spec.err)
-    inc(r.skipped)
-    inc(r.total)
-    result = false
-  else:
-    result = true
-
 var count = 0
 
-proc testSpecHelper(r: var TResults, test: TTest, expected: TSpec, target: TTarget, nimcache: string) =
+proc equalModuloLastNewline(a, b: string): bool =
+  # allow lazy output spec that omits last newline, but really those should be fixed instead
+  result = a == b or b.endsWith("\n") and a == b[0 ..< ^1]
+
+proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
+                    target: TTarget, extraOptions: string, nimcache: string) =
+  test.startTime = epochTime()
+  if testName(test, target, extraOptions, false) in skips:
+    test.spec.err = reDisabled
+
+  if test.spec.err in {reDisabled, reJoined}:
+    r.addResult(test, target, extraOptions, "", "", test.spec.err)
+    inc(r.skipped)
+    return
+  var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
   case expected.action
   of actionCompile:
-    var given = callCompiler(expected.getCmd, test.name, test.options, nimcache, target,
-          extraOptions = " --stdout --hint[Path]:off --hint[Processing]:off")
-    compilerOutputTests(test, target, given, expected, r)
+    compilerOutputTests(test, target, extraOptions, given, expected, r)
   of actionRun:
-    var given = callCompiler(expected.getCmd, test.name, test.options, nimcache, target)
     if given.err != reSuccess:
-      r.addResult(test, target, "", "$ " & given.cmd & "\n" & given.nimout, given.err)
+      r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr)
     else:
       let isJsTarget = target == targetJS
       var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt)
-      if not existsFile(exeFile):
-        r.addResult(test, target, expected.output,
+      if not fileExists(exeFile):
+        r.addResult(test, target, extraOptions, expected.output,
                     "executable not found: " & exeFile, reExeNotFound)
       else:
         let nodejs = if isJsTarget: findNodeJs() else: ""
         if isJsTarget and nodejs == "":
-          r.addResult(test, target, expected.output, "nodejs binary not in PATH",
+          r.addResult(test, target, extraOptions, expected.output, "nodejs binary not in PATH",
                       reExeNotFound)
         else:
           var exeCmd: string
-          var args = test.args
+          var args = test.testArgs
           if isJsTarget:
             exeCmd = nodejs
-            args = concat(@[exeFile], args)
+            # see D20210217T215950
+            args = @["--unhandled-rejections=strict", exeFile] & args
           else:
-            if defined(posix) and not exeFile.contains('/'):
-              # "security" in Posix is actually just a euphemism
-              # for "unproductive arbitrary shit"
-              exeCmd = "./" & exeFile
-            else:
-              exeCmd = exeFile
-            if expected.useValgrind:
-              args = @["--error-exitcode=1"] & exeCmd & args
+            exeCmd = exeFile.dup(normalizeExe)
+            if valgrindEnabled and expected.useValgrind != disabled:
+              var valgrindOptions = @["--error-exitcode=1"]
+              if expected.useValgrind != leaking:
+                valgrindOptions.add "--leak-check=yes"
+              args = valgrindOptions & exeCmd & args
               exeCmd = "valgrind"
           var (_, buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input)
           # Treat all failure codes from nodejs as 1. Older versions of nodejs used
@@ -464,89 +506,86 @@ proc testSpecHelper(r: var TResults, test: TTest, expected: TSpec, target: TTarg
           if exitCode != 0: exitCode = 1
           let bufB =
             if expected.sortoutput:
-              var x = splitLines(strip(buf.string))
+              var buf2 = buf
+              buf2.stripLineEnd
+              var x = splitLines(buf2)
               sort(x, system.cmp)
-              join(x, "\n")
+              join(x, "\n") & '\n'
             else:
-              strip(buf.string)
+              buf
           if exitCode != expected.exitCode:
-            r.addResult(test, target, "exitcode: " & $expected.exitCode,
+            given.err = reExitcodesDiffer
+            r.addResult(test, target, extraOptions, "exitcode: " & $expected.exitCode,
                               "exitcode: " & $exitCode & "\n\nOutput:\n" &
                               bufB, reExitcodesDiffer)
-          elif (expected.outputCheck == ocEqual and expected.output != bufB) or
+          elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or
               (expected.outputCheck == ocSubstr and expected.output notin bufB):
             given.err = reOutputsDiffer
-            r.addResult(test, target, expected.output, bufB, reOutputsDiffer)
-          else:
-            compilerOutputTests(test, target, given, expected, r)
+            r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer)
+          compilerOutputTests(test, target, extraOptions, given, expected, r)
   of actionReject:
-    var given = callCompiler(expected.getCmd, test.name, test.options,
-                              nimcache, target)
-    cmpMsgs(r, expected, given, test, target)
+    # Make sure its the compiler rejecting and not the system (e.g. segfault)
+    cmpMsgs(r, expected, given, test, target, extraOptions)
+    if given.exitCode != QuitFailure:
+      r.addResult(test, target, extraOptions, "exitcode: " & $QuitFailure,
+                        "exitcode: " & $given.exitCode & "\n\nOutput:\n" &
+                        given.nimout, reExitcodesDiffer)
 
 
-proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
-  var expected = test.spec
-  if expected.parseErrors.len > 0:
-    # targetC is a lie, but a parameter is required
-    r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec)
-    inc(r.total)
-    return
-  if not checkDisabled(r, test): return
 
-  expected.targets.incl targets
-  # still no target specified at all
-  if expected.targets == {}:
-    expected.targets = {getTestSpecTarget()}
+proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget =
+  result = defaultTarget
+  var p = parseopt.initOptParser(extraOptions)
+
+  while true:
+    parseopt.next(p)
+    case p.kind
+    of cmdEnd: break
+    of cmdLongOption, cmdShortOption:
+      if p.key == "b" or p.key == "backend":
+        result = parseEnum[TTarget](p.val.normalize)
+        # chooses the last one
+    else:
+      discard
+
+proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) =
   for target in expected.targets:
     inc(r.total)
     if target notin gTargets:
-      r.addResult(test, target, "", "", reDisabled)
+      r.addResult(test, target, extraOptions, "", "", reDisabled)
       inc(r.skipped)
     elif simulate:
       inc count
       echo "testSpec count: ", count, " expected: ", expected
     else:
       let nimcache = nimcacheDir(test.name, test.options, target)
-      testSpecHelper(r, test, expected, target, nimcache)
+      var testClone = test
+      let target = changeTarget(extraOptions, target)
+      testSpecHelper(r, testClone, expected, target, extraOptions, nimcache)
 
-proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) =
-  if not checkDisabled(r, test): return
-  for target in test.spec.targets:
+proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
+  var expected = test.spec
+  if expected.parseErrors.len > 0:
+    # targetC is a lie, but a parameter is required
+    r.addResult(test, targetC, "", "", expected.parseErrors, reInvalidSpec)
     inc(r.total)
-    testSpecHelper(r, test, test.spec, target, nimcache)
-
-proc testC(r: var TResults, test: TTest, action: TTestAction) =
-  # runs C code. Doesn't support any specs, just goes by exit code.
-  if not checkDisabled(r, test): return
-
-  let tname = test.name.addFileExt(".c")
-  inc(r.total)
-  maybeStyledEcho "Processing ", fgCyan, extractFilename(tname)
-  var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC)
-  if given.err != reSuccess:
-    r.addResult(test, targetC, "", given.msg, given.err)
-  elif action == actionRun:
-    let exeFile = changeFileExt(test.name, ExeExt)
-    var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath})
-    if exitCode != 0: given.err = reExitcodesDiffer
-  if given.err == reSuccess: inc(r.passed)
-
-proc testExec(r: var TResults, test: TTest) =
-  # runs executable or script, just goes by exit code
-  if not checkDisabled(r, test): return
+    return
 
-  inc(r.total)
-  let (outp, errC) = execCmdEx(test.options.strip())
-  var given: TSpec
-  if errC == 0:
-    given.err = reSuccess
+  expected.targets.incl targets
+  # still no target specified at all
+  if expected.targets == {}:
+    expected.targets = {getTestSpecTarget()}
+  if test.spec.matrix.len > 0:
+    for m in test.spec.matrix:
+      targetHelper(r, test, expected, m)
   else:
-    given.err = reExitcodesDiffer
-    given.msg = outp.string
+    targetHelper(r, test, expected, "")
 
-  if given.err == reSuccess: inc(r.passed)
-  r.addResult(test, targetC, "", given.msg, given.err)
+proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} =
+  for target in test.spec.targets:
+    inc(r.total)
+    var testClone = test
+    testSpecHelper(r, testClone, test.spec, target, "", nimcache)
 
 proc makeTest(test, options: string, cat: Category): TTest =
   result.cat = cat
@@ -555,19 +594,17 @@ proc makeTest(test, options: string, cat: Category): TTest =
   result.spec = parseSpec(addFileExt(test, ".nim"))
   result.startTime = epochTime()
 
-proc makeRawTest(test, options: string, cat: Category): TTest =
+proc makeRawTest(test, options: string, cat: Category): TTest {.used.} =
   result.cat = cat
   result.name = test
   result.options = options
   result.spec = initSpec(addFileExt(test, ".nim"))
-  result.startTime = epochTime()
   result.spec.action = actionCompile
   result.spec.targets = {getTestSpecTarget()}
+  result.startTime = epochTime()
 
 # TODO: fix these files
 const disabledFilesDefault = @[
-  "LockFreeHash.nim",
-  "sharedstrings.nim",
   "tableimpl.nim",
   "setimpl.nim",
   "hashcommon.nim",
@@ -602,13 +639,10 @@ proc loadSkipFrom(name: string): seq[string] =
   # used by `nlvm` (at least)
   for line in lines(name):
     let sline = line.strip()
-    if sline.len > 0 and not sline.startsWith("#"):
+    if sline.len > 0 and not sline.startsWith('#'):
       result.add sline
 
 proc main() =
-  os.putEnv "NIMTEST_COLOR", "never"
-  os.putEnv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
-
   azure.init()
   backend.open()
   var optPrintResults = false
@@ -616,42 +650,60 @@ proc main() =
   var targetsStr = ""
   var isMainProcess = true
   var skipFrom = ""
-  var useMegatest = true
 
   var p = initOptParser()
   p.next()
   while p.kind in {cmdLongOption, cmdShortOption}:
-    case p.key.string.normalize
-    of "print", "verbose": optPrintResults = true
+    case p.key.normalize
+    of "print": optPrintResults = true
+    of "verbose": optVerbose = true
     of "failing": optFailing = true
-    of "pedantic": discard "now always enabled"
+    of "pedantic": discard # deadcode refs https://github.com/nim-lang/Nim/issues/16731
     of "targets":
-      targetsStr = p.val.string
+      targetsStr = p.val
       gTargets = parseTargets(targetsStr)
+      targetsSet = true
     of "nim":
-      compilerPrefix = addFileExt(p.val.string, ExeExt)
+      compilerPrefix = addFileExt(p.val.absolutePath, ExeExt)
     of "directory":
-      setCurrentDir(p.val.string)
+      setCurrentDir(p.val)
     of "colors":
-      case p.val.string:
+      case p.val:
       of "on":
         useColors = true
       of "off":
         useColors = false
       else:
         quit Usage
+    of "batch":
+      testamentData0.batchArg = p.val
+      if p.val != "_" and p.val.len > 0 and p.val[0] in {'0'..'9'}:
+        let s = p.val.split("_")
+        doAssert s.len == 2, $(p.val, s)
+        testamentData0.testamentBatch = s[0].parseInt
+        testamentData0.testamentNumBatch = s[1].parseInt
+        doAssert testamentData0.testamentNumBatch > 0
+        doAssert testamentData0.testamentBatch >= 0 and testamentData0.testamentBatch < testamentData0.testamentNumBatch
     of "simulate":
       simulate = true
     of "megatest":
-      case p.val.string:
+      case p.val:
       of "on":
         useMegatest = true
       of "off":
         useMegatest = false
       else:
         quit Usage
+    of "valgrind":
+      case p.val:
+      of "on":
+        valgrindEnabled = true
+      of "off":
+        valgrindEnabled = false
+      else:
+        quit Usage
     of "backendlogging":
-      case p.val.string:
+      case p.val:
       of "on":
         backendLogging = true
       of "off":
@@ -659,36 +711,39 @@ proc main() =
       else:
         quit Usage
     of "skipfrom":
-      skipFrom = p.val.string
+      skipFrom = p.val
     else:
       quit Usage
     p.next()
   if p.kind != cmdArgument:
     quit Usage
-  var action = p.key.string.normalize
+  var action = p.key.normalize
   p.next()
   var r = initResults()
   case action
   of "all":
-    #processCategory(r, Category"megatest", p.cmdLineRest.string, testsDir, runJoinableTests = false)
+    #processCategory(r, Category"megatest", p.cmdLineRest, testsDir, runJoinableTests = false)
 
-    var myself = quoteShell(findExe("testament" / "testament"))
+    var myself = quoteShell(getAppFilename())
     if targetsStr.len > 0:
       myself &= " " & quoteShell("--targets:" & targetsStr)
 
     myself &= " " & quoteShell("--nim:" & compilerPrefix)
+    if testamentData0.batchArg.len > 0:
+      myself &= " --batch:" & testamentData0.batchArg
 
     if skipFrom.len > 0:
       myself &= " " & quoteShell("--skipFrom:" & skipFrom)
 
     var cats: seq[string]
-    let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: ""
+    let rest = if p.cmdLineRest.len > 0: " " & p.cmdLineRest else: ""
     for kind, dir in walkDir(testsDir):
       assert testsDir.startsWith(testsDir)
       let cat = dir[testsDir.len .. ^1]
       if kind == pcDir and cat notin ["testdata", "nimcache"]:
         cats.add cat
-    cats.add AdditionalCategories
+    if isNimRepoTests():
+      cats.add AdditionalCategories
     if useMegatest: cats.add MegaTestCat
 
     var cmds: seq[string]
@@ -697,21 +752,20 @@ proc main() =
       cmds.add(myself & runtype & quoteShell(cat) & rest)
 
     proc progressStatus(idx: int) =
-      echo "progress[all]: i: " & $idx & " / " & $cats.len & " cat: " & cats[idx]
+      echo "progress[all]: $1/$2 starting: cat: $3" % [$idx, $cats.len, cats[idx]]
 
     if simulate:
       skips = loadSkipFrom(skipFrom)
       for i, cati in cats:
         progressStatus(i)
-        processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, runJoinableTests = false)
+        processCategory(r, Category(cati), p.cmdLineRest, testsDir, runJoinableTests = false)
     else:
-      addQuitProc azure.finalize
+      addExitProc azure.finalize
       quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus)
   of "c", "cat", "category":
     skips = loadSkipFrom(skipFrom)
     var cat = Category(p.key)
-    p.next
-    processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true)
+    processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = true)
   of "pcat":
     skips = loadSkipFrom(skipFrom)
     # 'pcat' is used for running a category in parallel. Currently the only
@@ -720,19 +774,15 @@ proc main() =
     isMainProcess = false
     var cat = Category(p.key)
     p.next
-    processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = false)
+    processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = false)
   of "p", "pat", "pattern":
     skips = loadSkipFrom(skipFrom)
     let pattern = p.key
     p.next
-    processPattern(r, pattern, p.cmdLineRest.string, simulate)
+    processPattern(r, pattern, p.cmdLineRest, simulate)
   of "r", "run":
-    # at least one directory is required in the path, to use as a category name
-    let pathParts = split(p.key.string, {DirSep, AltSep})
-    # "stdlib/nre/captures.nim" -> "stdlib" + "nre/captures.nim"
-    let cat = Category(pathParts[0])
-    let subPath = joinPath(pathParts[1..^1])
-    processSingleTest(r, cat, p.cmdLineRest.string, subPath)
+    let (cat, path) = splitTestFile(p.key)
+    processSingleTest(r, cat.Category, p.cmdLineRest, path, gTargets, targetsSet)
   of "html":
     generateHtml(resultsFile, optFailing)
   else:
diff --git a/testament/testament.nim.cfg b/testament/testament.nim.cfg
index 69f044554..c97284129 100644
--- a/testament/testament.nim.cfg
+++ b/testament/testament.nim.cfg
@@ -1,2 +1,6 @@
+# don't move this file without updating the logic in `isNimRepoTests`
+
 path = "$nim" # For compiler/nodejs
 -d:ssl # For azure
+# my SSL doesn't have this feature and I don't care:
+-d:nimDisableCertificateValidation
diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim
index a8d216a5b..477da1e23 100644
--- a/testament/tests/shouldfail/tccodecheck.nim
+++ b/testament/tests/shouldfail/tccodecheck.nim
@@ -1,5 +1,5 @@
 discard """
-ccodecheck: "baz"
+  ccodecheck: "baz"
 """
 
 proc foo(): void {.exportc: "bar".}=
diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim
index 89482e673..809ddec74 100644
--- a/testament/tests/shouldfail/tcolumn.nim
+++ b/testament/tests/shouldfail/tcolumn.nim
@@ -1,7 +1,7 @@
 discard """
-errormsg: "undeclared identifier: 'undeclared'"
-line: 8
-column: 7
+  errormsg: "undeclared identifier: 'undeclared'"
+  line: 9
+  column: 7
 """
 
 # test should fail because the line directive is wrong
diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim
index dbbdf5021..6c130d107 100644
--- a/testament/tests/shouldfail/terrormsg.nim
+++ b/testament/tests/shouldfail/terrormsg.nim
@@ -1,7 +1,7 @@
 discard """
-errormsg: "wrong error message"
-line: 8
-column: 6
+  errormsg: "wrong error message"
+  line: 9
+  column: 6
 """
 
 # test should fail because the line directive is wrong
diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim
index 1b38b4f2e..605f046db 100644
--- a/testament/tests/shouldfail/texitcode1.nim
+++ b/testament/tests/shouldfail/texitcode1.nim
@@ -1,3 +1,3 @@
 discard """
-exitcode: 1
+  exitcode: 1
 """
diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim
index a6c2ad359..b40a4f44f 100644
--- a/testament/tests/shouldfail/tfile.nim
+++ b/testament/tests/shouldfail/tfile.nim
@@ -1,6 +1,6 @@
 discard """
-errmsg: "undeclared identifier: 'undefined'"
-file: "notthisfile.nim"
+  errormsg: "undeclared identifier: 'undefined'"
+  file: "notthisfile.nim"
 """
 
 echo undefined
diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim
index f7a09875c..fe782eb03 100644
--- a/testament/tests/shouldfail/tline.nim
+++ b/testament/tests/shouldfail/tline.nim
@@ -1,7 +1,7 @@
 discard """
-errormsg: "undeclared identifier: 'undeclared'"
-line: 9
-column: 6
+  errormsg: "undeclared identifier: 'undeclared'"
+  line: 10
+  column: 6
 """
 
 # test should fail because the line directive is wrong
diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim
index 9879e4181..92022ee97 100644
--- a/testament/tests/shouldfail/tmaxcodesize.nim
+++ b/testament/tests/shouldfail/tmaxcodesize.nim
@@ -1,5 +1,5 @@
 discard """
-maxcodesize: 1
+  maxcodesize: 1
 """
 
 echo "Hello World"
diff --git a/testament/tests/shouldfail/tmsg.nim b/testament/tests/shouldfail/tmsg.nim
deleted file mode 100644
index 4ad17fa95..000000000
--- a/testament/tests/shouldfail/tmsg.nim
+++ /dev/null
@@ -1,6 +0,0 @@
-discard """
-msg: "Hello World"
-"""
-
-static:
-  echo "something else"
diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim
index c0e332053..0a65bfb70 100644
--- a/testament/tests/shouldfail/tnimout.nim
+++ b/testament/tests/shouldfail/tnimout.nim
@@ -1,6 +1,6 @@
 discard """
-nimout: "Hello World!"
-action: compile
+  nimout: "Hello World!"
+  action: compile
 """
 
 static:
diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim
new file mode 100644
index 000000000..4fc93f6d2
--- /dev/null
+++ b/testament/tests/shouldfail/tnimoutfull.nim
@@ -0,0 +1,14 @@
+discard """
+  nimout: '''
+msg1
+msg2
+'''
+  action: compile
+  nimoutFull: true
+"""
+
+# should fail because `msg3` is not in nimout and `nimoutFill: true` was given
+static:
+  echo "msg1"
+  echo "msg2"
+  echo "msg3"
diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim
index ac0bc7a46..eaf9e8652 100644
--- a/testament/tests/shouldfail/toutput.nim
+++ b/testament/tests/shouldfail/toutput.nim
@@ -1,7 +1,7 @@
 discard """
-output: '''
-done
-'''
+  output: '''
+  done
+  '''
 """
 
 echo "broken"
diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim
index 7cc51ee8d..47324ecee 100644
--- a/testament/tests/shouldfail/toutputsub.nim
+++ b/testament/tests/shouldfail/toutputsub.nim
@@ -1,5 +1,5 @@
 discard """
-outputsub: "something else"
+  outputsub: "something else"
 """
 
 echo "Hello World!"
diff --git a/testament/tests/shouldfail/treject.nim b/testament/tests/shouldfail/treject.nim
new file mode 100644
index 000000000..1e7258f70
--- /dev/null
+++ b/testament/tests/shouldfail/treject.nim
@@ -0,0 +1,7 @@
+discard """
+  action: "reject"
+"""
+
+# Because we set action="reject", we expect this line not to compile. But the
+# line does compile, therefore the test fails.
+assert true
diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim
index 4ce9ce26d..69dfbc0a0 100644
--- a/testament/tests/shouldfail/tsortoutput.nim
+++ b/testament/tests/shouldfail/tsortoutput.nim
@@ -1,6 +1,6 @@
 discard """
-sortoutput: true
-output: '''
+  sortoutput: true
+  output: '''
 2
 1
 '''
diff --git a/testament/tests/shouldfail/tvalgrind.nim b/testament/tests/shouldfail/tvalgrind.nim
new file mode 100644
index 000000000..d551ff12e
--- /dev/null
+++ b/testament/tests/shouldfail/tvalgrind.nim
@@ -0,0 +1,17 @@
+discard """
+  valgrind: true
+  cmd: "nim $target --gc:arc -d:useMalloc $options $file"
+"""
+
+# this is the same check used by testament/specs.nim whether or not valgrind
+# tests are supported
+when defined(linux) and sizeof(int) == 8:
+  # discarding this allocation will cause valgrind to fail (which is what we
+  # want), but valgrind only runs on 64-bit Linux machines...
+  discard alloc(1)
+else:
+  # ...so on all other OS/architectures, simulate any non-zero exit code to
+  # mimic how valgrind would have failed on this test. We cannot use things like
+  # `disabled: "freebsd"` in the Testament configs above or else the tests will
+  # be SKIP-ed rather than FAIL-ed
+  quit(1) # choose 1 to match valgrind's `--error-exit=1`, but could be anything