summary refs log tree commit diff stats
path: root/testament
diff options
context:
space:
mode:
Diffstat (limited to 'testament')
-rw-r--r--testament/azure.nim147
-rw-r--r--testament/backend.nim70
-rw-r--r--testament/caasdriver.nim195
-rw-r--r--testament/categories.nim774
-rw-r--r--testament/htmlgen.nim149
-rw-r--r--testament/important_packages.nim193
-rw-r--r--testament/lib/readme.md4
-rw-r--r--testament/lib/stdtest/netutils.nim13
-rw-r--r--testament/lib/stdtest/specialpaths.nim55
-rw-r--r--testament/lib/stdtest/testutils.nim126
-rw-r--r--testament/lib/stdtest/unittest_light.nim37
-rw-r--r--testament/specs.nim521
-rw-r--r--testament/testament.nim806
-rw-r--r--testament/testament.nim.cfg6
-rw-r--r--testament/testamenthtml.nimf297
-rw-r--r--testament/tests/shouldfail/tccodecheck.nim8
-rw-r--r--testament/tests/shouldfail/tcolumn.nim8
-rw-r--r--testament/tests/shouldfail/terrormsg.nim8
-rw-r--r--testament/tests/shouldfail/texitcode1.nim3
-rw-r--r--testament/tests/shouldfail/tfile.nim6
-rw-r--r--testament/tests/shouldfail/tline.nim8
-rw-r--r--testament/tests/shouldfail/tmaxcodesize.nim5
-rw-r--r--testament/tests/shouldfail/tnimout.nim7
-rw-r--r--testament/tests/shouldfail/tnimoutfull.nim14
-rw-r--r--testament/tests/shouldfail/toutput.nim7
-rw-r--r--testament/tests/shouldfail/toutputsub.nim5
-rw-r--r--testament/tests/shouldfail/treject.nim7
-rw-r--r--testament/tests/shouldfail/tsortoutput.nim11
-rw-r--r--testament/tests/shouldfail/ttimeout.nim7
-rw-r--r--testament/tests/shouldfail/tvalgrind.nim17
30 files changed, 3514 insertions, 0 deletions
diff --git a/testament/azure.nim b/testament/azure.nim
new file mode 100644
index 000000000..af65d6a1c
--- /dev/null
+++ b/testament/azure.nim
@@ -0,0 +1,147 @@
+#
+#
+#              The Nim Tester
+#        (c) Copyright 2019 Leorize
+#
+#    Look at license.txt for more info.
+#    All rights reserved.
+
+import base64, json, httpclient, os, strutils, uri
+import specs
+
+const
+  RunIdEnv = "TESTAMENT_AZURE_RUN_ID"
+  CacheSize = 8 # How many results should be cached before uploading to
+                # Azure Pipelines. This prevents throttling that might arise.
+
+proc getAzureEnv(env: string): string =
+  # Conversion rule at:
+  # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#set-variables-in-pipeline
+  env.toUpperAscii().replace('.', '_').getEnv
+
+template getRun(): string =
+  ## Get the test run attached to this instance
+  getEnv(RunIdEnv)
+
+template setRun(id: string) =
+  ## Attach a test run to this instance and its future children
+  putEnv(RunIdEnv, id)
+
+template delRun() =
+  ## Unattach the test run associtated with this instance and its future children
+  delEnv(RunIdEnv)
+
+template warning(args: varargs[untyped]) =
+  ## Add a warning to the current task
+  stderr.writeLine "##vso[task.logissue type=warning;]", args
+
+let
+  ownRun = not existsEnv RunIdEnv
+    ## Whether the test run is owned by this instance
+  accessToken = getAzureEnv("System.AccessToken")
+    ## Access token to Azure Pipelines
+
+var
+  active = false ## Whether the backend should be activated
+  requestBase: Uri ## Base URI for all API requests
+  requestHeaders: HttpHeaders ## Headers required for all API requests
+  results: JsonNode ## A cache for test results before uploading
+
+proc request(api: string, httpMethod: HttpMethod, body = ""): Response {.inline.} =
+  let client = newHttpClient(timeout = 3000)
+  defer: close client
+  result = client.request($(requestBase / api), httpMethod, body, requestHeaders)
+  if result.code != Http200:
+    raise newException(CatchableError, "Request failed")
+
+proc init*() =
+  ## Initialize the Azure Pipelines backend.
+  ##
+  ## If an access token is provided and no test run is associated with the
+  ## current instance, this proc will create a test run named after the current
+  ## Azure Pipelines' job name, then associate it to the current testament
+  ## instance and its future children. Should this fail, the backend will be
+  ## disabled.
+  if isAzure and accessToken.len > 0:
+    active = true
+    requestBase = parseUri(getAzureEnv("System.TeamFoundationCollectionUri")) /
+      getAzureEnv("System.TeamProjectId") / "_apis" ? {"api-version": "5.0"}
+    requestHeaders = newHttpHeaders {
+      "Accept": "application/json",
+      "Authorization": "Basic " & encode(':' & accessToken),
+      "Content-Type": "application/json"
+    }
+    results = newJArray()
+    if ownRun:
+      try:
+        let resp = request(
+          "test/runs",
+          HttpPost,
+          $ %* {
+            "automated": true,
+            "build": { "id": getAzureEnv("Build.BuildId") },
+            "buildPlatform": hostCPU,
+            "controller": "nim-testament",
+            "name": getAzureEnv("Agent.JobName")
+          }
+        )
+        setRun $resp.body.parseJson["id"].getInt
+      except:
+        warning "Couldn't create test run for Azure Pipelines integration"
+        # Set run id to empty to prevent child processes from trying to request
+        # for yet another test run id, which wouldn't be shared with other
+        # instances.
+        setRun ""
+        active = false
+    elif getRun().len == 0:
+      # Disable integration if there aren't any valid test run id
+      active = false
+
+proc uploadAndClear() =
+  ## Upload test results from cache to Azure Pipelines. Then clear the cache
+  ## after.
+  if results.len > 0:
+    try:
+      discard request("test/runs/" & getRun() & "/results", HttpPost, $results)
+    except:
+      for i in results:
+        warning "Couldn't log test result to Azure Pipelines: ",
+          i["automatedTestName"], ", outcome: ", i["outcome"]
+    results = newJArray()
+
+proc finalize*() {.noconv.} =
+  ## Finalize the Azure Pipelines backend.
+  ##
+  ## If a test run has been associated and is owned by this instance, it will
+  ## be marked as complete.
+  if active:
+    if ownRun:
+      uploadAndClear()
+      try:
+        discard request("test/runs/" & getRun(), HttpPatch,
+                        $ %* {"state": "Completed"})
+      except:
+        warning "Couldn't update test run ", getRun(), " on Azure Pipelines"
+      delRun()
+
+proc addTestResult*(name, category: string; durationInMs: int; errorMsg: string;
+                    outcome: TResultEnum) =
+  if not active:
+    return
+
+  let outcome = case outcome
+                of reSuccess: "Passed"
+                of reDisabled, reJoined: "NotExecuted"
+                else: "Failed"
+
+  results.add(%* {
+      "automatedTestName": name,
+      "automatedTestStorage": category,
+      "durationInMs": durationInMs,
+      "errorMessage": errorMsg,
+      "outcome": outcome,
+      "testCaseTitle": name
+  })
+
+  if results.len > CacheSize:
+    uploadAndClear()
diff --git a/testament/backend.nim b/testament/backend.nim
new file mode 100644
index 000000000..1770c6657
--- /dev/null
+++ b/testament/backend.nim
@@ -0,0 +1,70 @@
+#
+#
+#              The Nim Tester
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    Look at license.txt for more info.
+#    All rights reserved.
+
+import strutils, os, osproc, json
+
+type
+  MachineId* = distinct string
+  CommitId = distinct string
+
+proc `$`*(id: MachineId): string {.borrow.}
+
+var
+  thisMachine: MachineId
+  thisCommit: CommitId
+  thisBranch: string
+
+proc getMachine*(): MachineId =
+  var name = execProcess("hostname").strip
+  if name.len == 0:
+    name = when defined(posix): getEnv("HOSTNAME")
+           else: getEnv("COMPUTERNAME")
+  if name.len == 0:
+    quit "cannot determine the machine name"
+
+  result = MachineId(name)
+
+proc getCommit(): CommitId =
+  const commLen = "commit ".len
+  let hash = execProcess("git log -n 1").strip[commLen..commLen+10]
+  thisBranch = execProcess("git symbolic-ref --short HEAD").strip
+  if hash.len == 0 or thisBranch.len == 0: quit "cannot determine git HEAD"
+  result = CommitId(hash)
+
+var
+  results: File
+  currentCategory: string
+  entries: int
+
+proc writeTestResult*(name, category, target, action, result, expected, given: string) =
+  createDir("testresults")
+  if currentCategory != category:
+    if currentCategory.len > 0:
+      results.writeLine("]")
+      close(results)
+    currentCategory = category
+    results = open("testresults" / category.addFileExt"json", fmWrite)
+    results.writeLine("[")
+    entries = 0
+
+  let jentry = %*{"name": name, "category": category, "target": target,
+    "action": action, "result": result, "expected": expected, "given": given,
+    "machine": thisMachine.string, "commit": thisCommit.string, "branch": thisBranch}
+  if entries > 0:
+    results.writeLine(",")
+  results.write($jentry)
+  inc entries
+
+proc open*() =
+  thisMachine = getMachine()
+  thisCommit = getCommit()
+
+proc close*() =
+  if currentCategory.len > 0:
+    results.writeLine("]")
+    close(results)
diff --git a/testament/caasdriver.nim b/testament/caasdriver.nim
new file mode 100644
index 000000000..01e402e07
--- /dev/null
+++ b/testament/caasdriver.nim
@@ -0,0 +1,195 @@
+import osproc, streams, os, strutils, re
+{.experimental.}
+
+## Compiler as a service tester.
+##
+## Please read docs/idetools.txt for information about this.
+
+
+type
+  TRunMode = enum
+    ProcRun, CaasRun, SymbolProcRun
+
+  NimSession* = object
+    nim: Process # Holds the open process for CaasRun sessions, nil otherwise.
+    mode: TRunMode # Stores the type of run mode the session was started with.
+    lastOutput: string # Preserves the last output, needed for ProcRun mode.
+    filename: string # Appended to each command starting with '>'. Also a var.
+    modname: string # Like filename but without extension.
+    nimcache: string # Input script based name for the nimcache dir.
+
+const
+  modes = [CaasRun, ProcRun, SymbolProcRun]
+  filenameReplaceVar = "$TESTNIM"
+  moduleReplaceVar = "$MODULE"
+  silentReplaceVar = "$SILENT"
+  silentReplaceText = "--verbosity:0 --hints:off"
+
+var
+  TesterDir = getAppDir() / ".."
+  NimBin = TesterDir / "../bin/nim"
+
+proc replaceVars(session: var NimSession, text: string): string =
+  result = text.replace(filenameReplaceVar, session.filename)
+  result = result.replace(moduleReplaceVar, session.modname)
+  result = result.replace(silentReplaceVar, silentReplaceText)
+
+proc startNimSession(project, script: string, mode: TRunMode):
+                        NimSession =
+  let (dir, name, ext) = project.splitFile
+  result.mode = mode
+  result.lastOutput = ""
+  result.filename = name & ext
+  result.modname = name
+
+  let (nimcacheDir, nimcacheName, nimcacheExt) = script.splitFile
+  result.nimcache = "SymbolProcRun." & nimcacheName
+
+  if mode == SymbolProcRun:
+    removeDir(nimcacheDir / result.nimcache)
+  else:
+    removeDir(nimcacheDir / "nimcache")
+
+  if mode == CaasRun:
+    result.nim = startProcess(NimBin, workingDir = dir,
+      args = ["serve", "--server.type:stdin", name])
+
+proc doCaasCommand(session: var NimSession, command: string): string =
+  assert session.mode == CaasRun
+  session.nim.inputStream.write(session.replaceVars(command) & "\n")
+  session.nim.inputStream.flush
+
+  result = ""
+
+  while true:
+    var line = ""
+    if session.nim.outputStream.readLine(line):
+      if line.string == "": break
+      result.add(line.string & "\n")
+    else:
+      result = "FAILED TO EXECUTE: " & command & "\n" & result
+      break
+
+proc doProcCommand(session: var NimSession, command: string): string =
+  try:
+    assert session.mode == ProcRun or session.mode == SymbolProcRun
+  except:
+    result = "FAILED TO EXECUTE: " & command & "\n" & result
+  var
+    process = startProcess(NimBin, args = session.replaceVars(command).split)
+    stream = outputStream(process)
+    line = ""
+
+  result = ""
+  while stream.readLine(line):
+    if result.len > 0: result &= "\n"
+    result &= line.string
+
+  process.close()
+
+proc doCommand(session: var NimSession, command: string) =
+  if session.mode == CaasRun:
+    if not session.nim.running:
+      session.lastOutput = "FAILED TO EXECUTE: " & command & "\n" &
+          "Exit code " & $session.nim.peekExitCode
+      return
+    session.lastOutput = doCaasCommand(session,
+                                       command & " " & session.filename)
+  else:
+    var command = command
+    # For symbol runs we prepend the necessary parameters to avoid clobbering
+    # the normal nimcache.
+    if session.mode == SymbolProcRun:
+      command = "--symbolFiles:on --nimcache:" & session.nimcache &
+                " " & command
+    session.lastOutput = doProcCommand(session,
+                                       command & " " & session.filename)
+
+proc destroy(session: var NimSession) {.destructor.} =
+  if session.mode == CaasRun:
+    session.nim.close
+
+proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): bool =
+  result = true
+
+  var f = open(script)
+  var project = ""
+
+  if f.readLine(project):
+    var
+      s = startNimSession(script.parentDir / project.string, script, mode)
+      tline = ""
+      ln = 1
+
+    while f.readLine(tline):
+      var line = tline.string
+      inc ln
+
+      # Filter lines by run mode, removing the prefix if the mode is current.
+      for testMode in modes:
+        if line.startsWith($testMode):
+          if testMode != mode:
+            line = ""
+          else:
+            line = line[len($testMode)..len(line) - 1].strip
+          break
+
+      if line.strip.len == 0: continue
+
+      if line.startsWith("#"):
+        output.writeLine line
+        continue
+      elif line.startsWith(">"):
+        s.doCommand(line.substr(1).strip)
+        output.writeLine line, "\n", if verbose: s.lastOutput else: ""
+      else:
+        var expectMatch = true
+        var pattern = s.replaceVars(line)
+        if line.startsWith("!"):
+          pattern = pattern.substr(1).strip
+          expectMatch = false
+
+        let actualMatch =
+          s.lastOutput.find(re(pattern, flags = {reStudy})) != -1
+
+        if expectMatch == actualMatch:
+          output.writeLine "SUCCESS ", line
+        else:
+          output.writeLine "FAILURE ", line
+          result = false
+
+iterator caasTestsRunner*(filter = "", verbose = false): tuple[test,
+                                              output: string, status: bool,
+                                              mode: TRunMode] =
+  for scenario in os.walkFiles(TesterDir / "caas/*.txt"):
+    if filter.len > 0 and find(scenario, filter) == -1: continue
+    for mode in modes:
+      var outStream = newStringStream()
+      let r = doScenario(scenario, outStream, mode, verbose)
+      yield (scenario, outStream.data, r, mode)
+
+when isMainModule:
+  var
+    filter = ""
+    failures = 0
+    verbose = false
+
+  for i in 0..paramCount() - 1:
+    let param = paramStr(i + 1)
+    case param
+    of "verbose": verbose = true
+    else: filter = param
+
+  if verbose and len(filter) > 0:
+    echo "Running only test cases matching filter '$1'" % [filter]
+
+  for test, output, result, mode in caasTestsRunner(filter, verbose):
+    if not result or verbose:
+      echo "Mode ", $mode, " (", if result: "succeeded)" else: "failed)"
+      echo test
+      echo output
+      echo "---------\n"
+    if not result:
+      failures += 1
+
+  quit(failures)
diff --git a/testament/categories.nim b/testament/categories.nim
new file mode 100644
index 000000000..843bef3f9
--- /dev/null
+++ b/testament/categories.nim
@@ -0,0 +1,774 @@
+#
+#
+#            Nim Tester
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Include for the tester that contains test suites that test special features
+## of the compiler.
+
+# included from testament.nim
+
+import important_packages
+import std/[strformat, strutils]
+from std/sequtils import filterIt
+
+const
+  specialCategories = [
+    "assert",
+    "async",
+    "debugger",
+    "dll",
+    "examples",
+    "gc",
+    "io",
+    "js",
+    "ic",
+    "lib",
+    "manyloc",
+    "nimble-packages",
+    "niminaction",
+    "threads",
+    "untestable", # see trunner_special
+    "testdata",
+    "nimcache",
+    "coroutines",
+    "osproc",
+    "shouldfail",
+    "destructor"
+  ]
+
+proc isTestFile*(file: string): bool =
+  let (_, name, ext) = splitFile(file)
+  result = ext == ".nim" and name.startsWith("t")
+
+# --------------------- DLL generation tests ----------------------------------
+
+proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = false) =
+  const rpath = when defined(macosx):
+      " --passL:-rpath --passL:@loader_path"
+    else:
+      ""
+
+  var test1 = makeTest("lib/nimrtl.nim", options & " --outdir:tests/dll", cat)
+  test1.spec.action = actionCompile
+  testSpec c, test1
+  var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat)
+  test2.spec.action = actionCompile
+  testSpec c, test2
+
+  var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat)
+  test3.spec.action = actionCompile
+  testSpec c, test3
+  var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat)
+  test4.spec.action = actionCompile
+  testSpec c, test4
+
+  # windows looks in the dir of the exe (yay!):
+  when not defined(windows):
+    # posix relies on crappy LD_LIBRARY_PATH (ugh!):
+    const libpathenv = when defined(haiku): "LIBRARY_PATH"
+                       else: "LD_LIBRARY_PATH"
+    var libpath = getEnv(libpathenv)
+    # Temporarily add the lib directory to LD_LIBRARY_PATH:
+    putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: ""))
+    defer: putEnv(libpathenv, libpath)
+
+  testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat)
+  testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat)
+  testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat)
+
+  if "boehm" notin options:
+    # hcr tests
+    
+    var basicHcrTest = makeTest("tests/dll/nimhcr_basic.nim", options & " --threads:off --forceBuild --hotCodeReloading:on " & rpath, cat)
+    # test segfaults for now but compiles:
+    if isOrc: basicHcrTest.spec.action = actionCompile
+    testSpec r, basicHcrTest
+
+    # force build required - see the comments in the .nim file for more details
+    var hcri = makeTest("tests/dll/nimhcr_integration.nim",
+                                   options & " --threads:off --forceBuild --hotCodeReloading:on" & rpath, cat)
+    let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget())
+    let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name,
+                                hcri.options, nimcache, getTestSpecTarget())
+    hcri.testArgs = cmd.parseCmdLine
+    testSpec r, hcri
+
+proc dllTests(r: var TResults, cat: Category, options: string) =
+  # dummy compile result:
+  var c = initResults()
+
+  runBasicDLLTest c, r, cat, options & " --mm:refc"
+  runBasicDLLTest c, r, cat, options & " -d:release --mm:refc"
+  runBasicDLLTest c, r, cat, options, isOrc = true
+  runBasicDLLTest c, r, cat, options & " -d:release", isOrc = true
+  when not defined(windows):
+    # still cannot find a recent Windows version of boehm.dll:
+    runBasicDLLTest c, r, cat, options & " --gc:boehm"
+    runBasicDLLTest c, r, cat, options & " -d:release --gc:boehm"
+
+# ------------------------------ GC tests -------------------------------------
+
+proc gcTests(r: var TResults, cat: Category, options: string) =
+  template testWithoutMs(filename: untyped) =
+    testSpec r, makeTest("tests/gc" / filename, options & "--mm:refc", cat)
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " -d:release -d:useRealtimeGC --mm:refc", cat)
+    when filename != "gctest":
+      testSpec r, makeTest("tests/gc" / filename, options &
+                    " --gc:orc", cat)
+      testSpec r, makeTest("tests/gc" / filename, options &
+                    " --gc:orc -d:release", cat)
+
+  template testWithoutBoehm(filename: untyped) =
+    testWithoutMs filename
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " --gc:markAndSweep", cat)
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " -d:release --gc:markAndSweep", cat)
+
+  template test(filename: untyped) =
+    testWithoutBoehm filename
+    when not defined(windows) and not defined(android):
+      # AR: cannot find any boehm.dll on the net, right now, so disabled
+      # for windows:
+      testSpec r, makeTest("tests/gc" / filename, options &
+                    " --gc:boehm", cat)
+      testSpec r, makeTest("tests/gc" / filename, options &
+                    " -d:release --gc:boehm", cat)
+
+  testWithoutBoehm "foreign_thr"
+  test "gcemscripten"
+  test "growobjcrash"
+  test "gcbench"
+  test "gcleak"
+  test "gcleak2"
+  testWithoutBoehm "gctest"
+  test "gcleak3"
+  test "gcleak4"
+  # Disabled because it works and takes too long to run:
+  #test "gcleak5"
+  testWithoutBoehm "weakrefs"
+  test "cycleleak"
+  testWithoutBoehm "closureleak"
+  testWithoutMs "refarrayleak"
+
+  testWithoutBoehm "tlists"
+  testWithoutBoehm "thavlak"
+
+  test "stackrefleak"
+  test "cyclecollector"
+  testWithoutBoehm "trace_globals"
+
+# ------------------------- threading tests -----------------------------------
+
+proc threadTests(r: var TResults, cat: Category, options: string) =
+  template test(filename: untyped) =
+    testSpec r, makeTest(filename, options, cat)
+    testSpec r, makeTest(filename, options & " -d:release", cat)
+    testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat)
+  for t in os.walkFiles("tests/threads/t*.nim"):
+    test(t)
+
+# ------------------------- IO tests ------------------------------------------
+
+proc ioTests(r: var TResults, cat: Category, options: string) =
+  # We need readall_echo to be compiled for this test to run.
+  # dummy compile result:
+  var c = initResults()
+  testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat)
+  #        ^- why is this not appended to r? Should this be discarded?
+  # EDIT: this should be replaced by something like in D20210524T180826,
+  # likewise in similar instances where `testSpec c` is used, or more generally
+  # when a test depends on another test, as it makes tests non-independent,
+  # creating complications for batching and megatest logic.
+  testSpec r, makeTest("tests/system/tio", options, cat)
+
+# ------------------------- async tests ---------------------------------------
+proc asyncTests(r: var TResults, cat: Category, options: string) =
+  template test(filename: untyped) =
+    testSpec r, makeTest(filename, options, cat)
+  for t in os.walkFiles("tests/async/t*.nim"):
+    test(t)
+
+# ------------------------- debugger tests ------------------------------------
+
+proc debuggerTests(r: var TResults, cat: Category, options: string) =
+  if fileExists("tools/nimgrep.nim"):
+    var t = makeTest("tools/nimgrep", options & " --debugger:on", cat)
+    t.spec.action = actionCompile
+    # force target to C because of MacOS 10.15 SDK headers bug
+    # https://github.com/nim-lang/Nim/pull/15612#issuecomment-712471879
+    t.spec.targets = {targetC}
+    testSpec r, t
+
+# ------------------------- JS tests ------------------------------------------
+
+proc jsTests(r: var TResults, cat: Category, options: string) =
+  template test(filename: untyped) =
+    testSpec r, makeTest(filename, options, cat), {targetJS}
+    testSpec r, makeTest(filename, options & " -d:release", cat), {targetJS}
+
+  for t in os.walkFiles("tests/js/t*.nim"):
+    test(t)
+  for testfile in ["exception/texceptions", "exception/texcpt1",
+                   "exception/texcsub", "exception/tfinally",
+                   "exception/tfinally2", "exception/tfinally3",
+                   "collections/tactiontable", "method/tmultimjs",
+                   "varres/tvarres0", "varres/tvarres3", "varres/tvarres4",
+                   "varres/tvartup", "int/tints", "int/tunsignedinc",
+                   "async/tjsandnativeasync"]:
+    test "tests/" & testfile & ".nim"
+
+  for testfile in ["strutils", "json", "random", "times", "logging"]:
+    test "lib/pure/" & testfile & ".nim"
+
+# ------------------------- nim in action -----------
+
+proc testNimInAction(r: var TResults, cat: Category, options: string) =
+  template test(filename: untyped) =
+    testSpec r, makeTest(filename, options, cat)
+
+  template testJS(filename: untyped) =
+    testSpec r, makeTest(filename, options, cat), {targetJS}
+
+  template testCPP(filename: untyped) =
+    testSpec r, makeTest(filename, options, cat), {targetCpp}
+
+  let tests = [
+    "niminaction/Chapter1/various1",
+    "niminaction/Chapter2/various2",
+    "niminaction/Chapter2/resultaccept",
+    "niminaction/Chapter2/resultreject",
+    "niminaction/Chapter2/explicit_discard",
+    "niminaction/Chapter2/no_def_eq",
+    "niminaction/Chapter2/no_iterator",
+    "niminaction/Chapter2/no_seq_type",
+    "niminaction/Chapter3/ChatApp/src/server",
+    "niminaction/Chapter3/ChatApp/src/client",
+    "niminaction/Chapter3/various3",
+    "niminaction/Chapter6/WikipediaStats/concurrency_regex",
+    "niminaction/Chapter6/WikipediaStats/concurrency",
+    "niminaction/Chapter6/WikipediaStats/naive",
+    "niminaction/Chapter6/WikipediaStats/parallel_counts",
+    "niminaction/Chapter6/WikipediaStats/race_condition",
+    "niminaction/Chapter6/WikipediaStats/sequential_counts",
+    "niminaction/Chapter6/WikipediaStats/unguarded_access",
+    "niminaction/Chapter7/Tweeter/src/tweeter",
+    "niminaction/Chapter7/Tweeter/src/createDatabase",
+    "niminaction/Chapter7/Tweeter/tests/database_test",
+    "niminaction/Chapter8/sdl/sdl_test"
+    ]
+
+  when false:
+    # Verify that the files have not been modified. Death shall fall upon
+    # whoever edits these hashes without dom96's permission, j/k. But please only
+    # edit when making a conscious breaking change, also please try to make your
+    # commit message clear and notify me so I can easily compile an errata later.
+    # ---------------------------------------------------------
+    # Hash-checks are disabled for Nim 1.1 and beyond
+    # since we needed to fix the deprecated unary '<' operator.
+    const refHashes = @[
+      "51afdfa84b3ca3d810809d6c4e5037ba",
+      "30f07e4cd5eaec981f67868d4e91cfcf",
+      "d14e7c032de36d219c9548066a97e846",
+      "b335635562ff26ec0301bdd86356ac0c",
+      "6c4add749fbf50860e2f523f548e6b0e",
+      "76de5833a7cc46f96b006ce51179aeb1",
+      "705eff79844e219b47366bd431658961",
+      "a1e87b881c5eb161553d119be8b52f64",
+      "2d706a6ec68d2973ec7e733e6d5dce50",
+      "c11a013db35e798f44077bc0763cc86d",
+      "3e32e2c5e9a24bd13375e1cd0467079c",
+      "a5452722b2841f0c1db030cf17708955",
+      "dc6c45eb59f8814aaaf7aabdb8962294",
+      "69d208d281a2e7bffd3eaf4bab2309b1",
+      "ec05666cfb60211bedc5e81d4c1caf3d",
+      "da520038c153f4054cb8cc5faa617714",
+      "59906c8cd819cae67476baa90a36b8c1",
+      "9a8fe78c588d08018843b64b57409a02",
+      "8b5d28e985c0542163927d253a3e4fc9",
+      "783299b98179cc725f9c46b5e3b5381f",
+      "1a2b3fba1187c68d6a9bfa66854f3318",
+      "391ff57b38d9ea6f3eeb3fe69ab539d3"
+    ]
+    for i, test in tests:
+      let filename = testsDir / test.addFileExt("nim")
+      let testHash = getMD5(readFile(filename).string)
+      doAssert testHash == refHashes[i], "Nim in Action test " & filename &
+          " was changed: " & $(i: i, testHash: testHash, refHash: refHashes[i])
+
+  # Run the tests.
+  for testfile in tests:
+    test "tests/" & testfile & ".nim"
+  let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim"
+  testJS jsFile
+  let cppFile = "tests/niminaction/Chapter8/sfml/sfml_test.nim"
+  testCPP cppFile
+
+# ------------------------- manyloc -------------------------------------------
+
+proc findMainFile(dir: string): string =
+  # finds the file belonging to ".nim.cfg"; if there is no such file
+  # it returns the some ".nim" file if there is only one:
+  const cfgExt = ".nim.cfg"
+  result = ""
+  var nimFiles = 0
+  for kind, file in os.walkDir(dir):
+    if kind == pcFile:
+      if file.endsWith(cfgExt): return file[0..^(cfgExt.len+1)] & ".nim"
+      elif file.endsWith(".nim"):
+        if result.len == 0: result = file
+        inc nimFiles
+  if nimFiles != 1: result.setLen(0)
+
+proc manyLoc(r: var TResults, cat: Category, options: string) =
+  for kind, dir in os.walkDir("tests/manyloc"):
+    if kind == pcDir:
+      when defined(windows):
+        if dir.endsWith"nake": continue
+      if dir.endsWith"named_argument_bug": continue
+      let mainfile = findMainFile(dir)
+      if mainfile != "":
+        var test = makeTest(mainfile, options, cat)
+        test.spec.action = actionCompile
+        testSpec r, test
+
+proc compileExample(r: var TResults, pattern, options: string, cat: Category) =
+  for test in os.walkFiles(pattern):
+    var test = makeTest(test, options, cat)
+    test.spec.action = actionCompile
+    testSpec r, test
+
+proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
+  var files: seq[string]
+
+  proc isValid(file: string): bool =
+    for dir in parentDirs(file, inclusive = false):
+      if dir.lastPathPart in ["includes", "nimcache"]:
+        # e.g.: lib/pure/includes/osenv.nim gives: Error: This is an include file for os.nim!
+        return false
+    let name = extractFilename(file)
+    if name.splitFile.ext != ".nim": return false
+    for namei in disabledFiles:
+      # because of `LockFreeHash.nim` which has case
+      if namei.cmpPaths(name) == 0: return false
+    return true
+
+  for testFile in os.walkDirRec(pattern):
+    if isValid(testFile):
+      files.add testFile
+
+  files.sort # reproducible order
+  for testFile in files:
+    let contents = readFile(testFile)
+    var testObj = makeTest(testFile, options, cat)
+    #[
+    todo:
+    this logic is fragile:
+    false positives (if appears in a comment), or false negatives, e.g.
+    `when defined(osx) and isMainModule`.
+    Instead of fixing this, see https://github.com/nim-lang/Nim/issues/10045
+    for a much better way.
+    ]#
+    if "when isMainModule" notin contents:
+      testObj.spec.action = actionCompile
+    testSpec r, testObj
+
+# ----------------------------- nimble ----------------------------------------
+proc listPackagesAll(): seq[NimblePackage] =
+  var nimbleDir = getEnv("NIMBLE_DIR")
+  if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
+  let packageIndex = nimbleDir / "packages_official.json"
+  let packageList = parseFile(packageIndex)
+  proc findPackage(name: string): JsonNode =
+    for a in packageList:
+      if a["name"].str == name: return a
+  for pkg in important_packages.packages.items:
+    var pkg = pkg
+    if pkg.url.len == 0:
+      let pkg2 = findPackage(pkg.name)
+      if pkg2 == nil:
+        raise newException(ValueError, "Cannot find package '$#'." % pkg.name)
+      pkg.url = pkg2["url"].str
+    result.add pkg
+
+proc listPackages(packageFilter: string): seq[NimblePackage] =
+  let pkgs = listPackagesAll()
+  if packageFilter.len != 0:
+    # xxx document `packageFilter`, seems like a bad API,
+    # at least should be a regex; a substring match makes no sense.
+    result = pkgs.filterIt(packageFilter in it.name)
+  else:
+    if testamentData0.batchArg == "allowed_failures":
+      result = pkgs.filterIt(it.allowFailure)
+    elif testamentData0.testamentNumBatch == 0:
+      result = pkgs
+    else:
+      let pkgs2 = pkgs.filterIt(not it.allowFailure)
+      for i in 0..<pkgs2.len:
+        if i mod testamentData0.testamentNumBatch == testamentData0.testamentBatch:
+          result.add pkgs2[i]
+
+proc makeSupTest(test, options: string, cat: Category, debugInfo = ""): TTest =
+  result.cat = cat
+  result.name = test
+  result.options = options
+  result.debugInfo = debugInfo
+  result.startTime = epochTime()
+
+import std/private/gitutils
+
+proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) =
+  let nimbleExe = findExe("nimble")
+  doAssert nimbleExe != "", "Cannot run nimble tests: Nimble binary not found."
+  doAssert execCmd("$# update" % nimbleExe) == 0, "Cannot run nimble tests: Nimble update failed."
+  let packageFileTest = makeSupTest("PackageFileParsed", "", cat)
+  let packagesDir = "pkgstemp"
+  createDir(packagesDir)
+  var errors = 0
+  try:
+    let pkgs = listPackages(packageFilter)
+    for i, pkg in pkgs:
+      inc r.total
+      var test = makeSupTest(pkg.name, "", cat, "[$#/$#] " % [$i, $pkgs.len])
+      let buildPath = packagesDir / pkg.name
+      template tryCommand(cmd: string, workingDir2 = buildPath, reFailed = reInstallFailed, maxRetries = 1): string =
+        var outp: string
+        let ok = retryCall(maxRetry = maxRetries, backoffDuration = 10.0):
+          var status: int
+          (outp, status) = execCmdEx(cmd, workingDir = workingDir2)
+          status == QuitSuccess
+        if not ok:
+          if pkg.allowFailure:
+            inc r.passed
+            inc r.failedButAllowed
+          addResult(r, test, targetC, "", "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure)
+          continue
+        outp
+
+      if not dirExists(buildPath):
+        discard tryCommand("git clone $# $#" % [pkg.url.quoteShell, buildPath.quoteShell], workingDir2 = ".", maxRetries = 3)
+        if not pkg.useHead:
+          discard tryCommand("git fetch --tags", maxRetries = 3)
+          let describeOutput = tryCommand("git describe --tags --abbrev=0")
+          discard tryCommand("git checkout $#" % [describeOutput.strip.quoteShell])
+        discard tryCommand("nimble install --depsOnly -y", maxRetries = 3)
+      let cmds = pkg.cmd.split(';')
+      for i in 0 ..< cmds.len - 1:
+        discard tryCommand(cmds[i], maxRetries = 3)
+      discard tryCommand(cmds[^1], reFailed = reBuildFailed)
+      inc r.passed
+      r.addResult(test, targetC, "", "", "", reSuccess, allowFailure = pkg.allowFailure)
+
+    errors = r.total - r.passed
+    if errors == 0:
+      r.addResult(packageFileTest, targetC, "", "", "", reSuccess)
+    else:
+      r.addResult(packageFileTest, targetC, "", "", "", reBuildFailed)
+
+  except JsonParsingError:
+    errors = 1
+    r.addResult(packageFileTest, targetC, "", "", "Invalid package file", reBuildFailed)
+    raise
+  except ValueError:
+    errors = 1
+    r.addResult(packageFileTest, targetC, "", "", "Unknown package", reBuildFailed)
+    raise # bug #18805
+  finally:
+    if errors == 0: removeDir(packagesDir)
+
+# ---------------- IC tests ---------------------------------------------
+
+proc icTests(r: var TResults; testsDir: string, cat: Category, options: string;
+             isNavigatorTest: bool) =
+  const
+    tooltests = ["compiler/nim.nim"]
+    writeOnly = " --incremental:writeonly "
+    readOnly = " --incremental:readonly "
+    incrementalOn = " --incremental:on -d:nimIcIntegrityChecks "
+    navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint:Conf:off --warnings:off "
+
+  template test(x: untyped) =
+    testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache)
+
+  template editedTest(x: untyped) =
+    var test = makeTest(file, x & options, cat)
+    if isNavigatorTest:
+      test.spec.action = actionCompile
+    test.spec.targets = {getTestSpecTarget()}
+    testSpecWithNimcache(r, test, nimcache)
+
+  template checkTest() =
+    var test = makeRawTest(file, options, cat)
+    test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:on $options " & file
+    testSpecWithNimcache(r, test, nimcache)
+
+  if not isNavigatorTest:
+    for file in tooltests:
+      let nimcache = nimcacheDir(file, options, getTestSpecTarget())
+      removeDir(nimcache)
+
+      let oldPassed = r.passed
+      checkTest()
+
+      if r.passed == oldPassed+1:
+        checkTest()
+        if r.passed == oldPassed+2:
+          checkTest()
+
+  const tempExt = "_temp.nim"
+  for it in walkDirRec(testsDir):
+  # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test
+    if isTestFile(it) and not it.endsWith(tempExt):
+      let nimcache = nimcacheDir(it, options, getTestSpecTarget())
+      removeDir(nimcache)
+
+      let content = readFile(it)
+      for fragment in content.split("#!EDIT!#"):
+        let file = it.replace(".nim", tempExt)
+        writeFile(file, fragment)
+        let oldPassed = r.passed
+        editedTest(if isNavigatorTest: navTestConfig else: incrementalOn)
+        if r.passed != oldPassed+1: break
+
+# ----------------------------------------------------------------------------
+
+const AdditionalCategories = ["debugger", "examples", "lib", "ic", "navigator"]
+const MegaTestCat = "megatest"
+
+proc `&.?`(a, b: string): string =
+  # candidate for the stdlib?
+  result = if b.startsWith(a): b else: a & b
+
+proc processSingleTest(r: var TResults, cat: Category, options, test: string, targets: set[TTarget], targetsSet: bool) =
+  var targets = targets
+  if not targetsSet:
+    let target = if cat.string.normalize == "js": targetJS else: targetC
+    targets = {target}
+  doAssert fileExists(test), test & " test does not exist"
+  testSpec r, makeTest(test, options, cat), targets
+
+proc isJoinableSpec(spec: TSpec): bool =
+  # xxx simplify implementation using a whitelist of fields that are allowed to be
+  # set to non-default values (use `fieldPairs`), to avoid issues like bug #16576.
+  result = useMegatest and not spec.sortoutput and
+    spec.action == actionRun and
+    not fileExists(spec.file.changeFileExt("cfg")) and
+    not fileExists(spec.file.changeFileExt("nims")) and
+    not fileExists(parentDir(spec.file) / "nim.cfg") and
+    not fileExists(parentDir(spec.file) / "config.nims") and
+    spec.cmd.len == 0 and
+    spec.err != reDisabled and
+    not spec.unjoinable and
+    spec.exitCode == 0 and
+    spec.input.len == 0 and
+    spec.nimout.len == 0 and
+    spec.nimoutFull == false and
+      # so that tests can have `nimoutFull: true` with `nimout.len == 0` with
+      # the meaning that they expect empty output.
+    spec.matrix.len == 0 and
+    spec.outputCheck != ocSubstr and
+    spec.ccodeCheck.len == 0 and
+    (spec.targets == {} or spec.targets == {targetC})
+  if result:
+    if spec.file.readFile.contains "when isMainModule":
+      result = false
+
+proc quoted(a: string): string =
+  # todo: consider moving to system.nim
+  result.addQuoted(a)
+
+proc runJoinedTest(r: var TResults, cat: Category, testsDir: string, options: string) =
+  ## returns a list of tests that have problems
+  #[
+  xxx create a reusable megatest API after abstracting out testament specific code,
+  refs https://github.com/timotheecour/Nim/issues/655
+  and https://github.com/nim-lang/gtk2/pull/28; it's useful in other contexts.
+  ]#
+  var specs: seq[TSpec] = @[]
+  for kind, dir in walkDir(testsDir):
+    assert dir.startsWith(testsDir)
+    let cat = dir[testsDir.len .. ^1]
+    if kind == pcDir and cat notin specialCategories:
+      for file in walkDirRec(testsDir / cat):
+        if isTestFile(file):
+          var spec: TSpec
+          try:
+            spec = parseSpec(file)
+          except ValueError:
+            # e.g. for `tests/navigator/tincludefile.nim` which have multiple
+            # specs; this will be handled elsewhere
+            echo "parseSpec raised ValueError for: '$1', assuming this will be handled outside of megatest" % file
+            continue
+          if isJoinableSpec(spec):
+            specs.add spec
+
+  proc cmp(a: TSpec, b: TSpec): auto = cmp(a.file, b.file)
+  sort(specs, cmp = cmp) # reproducible order
+  echo "joinable specs: ", specs.len
+
+  if simulate:
+    var s = "runJoinedTest: "
+    for a in specs: s.add a.file & " "
+    echo s
+    return
+
+  var megatest: string
+  # xxx (minor) put outputExceptedFile, outputGottenFile, megatestFile under here or `buildDir`
+  var outDir = nimcacheDir(testsDir / "megatest", "", targetC)
+  template toMarker(file, i): string =
+    "megatest:processing: [$1] $2" % [$i, file]
+  for i, runSpec in specs:
+    let file = runSpec.file
+    let file2 = outDir / ("megatest_a_$1.nim" % $i)
+    # `include` didn't work with `trecmod2.nim`, so using `import`
+    let code = "echo $1\nstatic: echo \"CT:\", $1\n" % [toMarker(file, i).quoted]
+    createDir(file2.parentDir)
+    writeFile(file2, code)
+    megatest.add "import $1\nimport $2 as megatest_b_$3\n" % [file2.quoted, file.quoted, $i]
+
+  let megatestFile = testsDir / "megatest.nim" # so it uses testsDir / "config.nims"
+  writeFile(megatestFile, megatest)
+
+  let root = getCurrentDir()
+
+  var args = @["c", "--nimCache:" & outDir, "-d:testing", "-d:nimMegatest", "--listCmd",
+              "--path:" & root]
+  args.add options.parseCmdLine
+  args.add megatestFile
+  var (cmdLine, buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, input = "")
+  if exitCode != 0:
+    echo "$ " & cmdLine & "\n" & buf
+    quit(failString & "megatest compilation failed")
+
+  (buf, exitCode) = execCmdEx(megatestFile.changeFileExt(ExeExt).dup normalizeExe)
+  if exitCode != 0:
+    echo buf
+    quit(failString & "megatest execution failed")
+
+  const outputExceptedFile = "outputExpected.txt"
+  const outputGottenFile = "outputGotten.txt"
+  writeFile(outputGottenFile, buf)
+  var outputExpected = ""
+  for i, runSpec in specs:
+    outputExpected.add toMarker(runSpec.file, i) & "\n"
+    if runSpec.output.len > 0:
+      outputExpected.add runSpec.output
+      if not runSpec.output.endsWith "\n":
+        outputExpected.add '\n'
+
+  if buf != outputExpected:
+    writeFile(outputExceptedFile, outputExpected)
+    echo diffFiles(outputGottenFile, outputExceptedFile).output
+    echo failString & "megatest output different, see $1 vs $2" % [outputGottenFile, outputExceptedFile]
+    # outputGottenFile, outputExceptedFile not removed on purpose for debugging.
+    quit 1
+  else:
+    echo "megatest output OK"
+
+
+# ---------------------------------------------------------------------------
+
+proc processCategory(r: var TResults, cat: Category,
+                     options, testsDir: string,
+                     runJoinableTests: bool) =
+  let cat2 = cat.string.normalize
+  var handled = false
+  if isNimRepoTests():
+    handled = true
+    case cat2
+    of "js":
+      # only run the JS tests on Windows or Linux because Travis is bad
+      # and other OSes like Haiku might lack nodejs:
+      if not defined(linux) and isTravis:
+        discard
+      else:
+        jsTests(r, cat, options)
+    of "dll":
+      dllTests(r, cat, options & " -d:nimDebugDlOpen")
+    of "gc":
+      gcTests(r, cat, options)
+    of "debugger":
+      debuggerTests(r, cat, options)
+    of "manyloc":
+      manyLoc r, cat, options
+    of "threads":
+      threadTests r, cat, options & " --threads:on"
+    of "io":
+      ioTests r, cat, options
+    of "async":
+      asyncTests r, cat, options
+    of "lib":
+      testStdlib(r, "lib/pure/", options, cat)
+      testStdlib(r, "lib/packages/docutils/", options, cat)
+    of "examples":
+      compileExample(r, "examples/*.nim", options, cat)
+      compileExample(r, "examples/gtk/*.nim", options, cat)
+      compileExample(r, "examples/talk/*.nim", options, cat)
+    of "nimble-packages":
+      testNimblePackages(r, cat, options)
+    of "niminaction":
+      testNimInAction(r, cat, options)
+    of "ic":
+      icTests(r, testsDir / cat2, cat, options, isNavigatorTest=false)
+    of "navigator":
+      icTests(r, testsDir / cat2, cat, options, isNavigatorTest=true)
+    of "untestable":
+      # These require special treatment e.g. because they depend on a third party
+      # dependency; see `trunner_special` which runs some of those.
+      discard
+    else:
+      handled = false
+  if not handled:
+    case cat2
+    of "megatest":
+      runJoinedTest(r, cat, testsDir, options)
+      if isNimRepoTests():
+        runJoinedTest(r, cat, testsDir, options & " --mm:refc")
+    else:
+      var testsRun = 0
+      var files: seq[string]
+      for file in walkDirRec(testsDir &.? cat.string):
+        if isTestFile(file): files.add file
+      files.sort # give reproducible order
+      for i, name in files:
+        var test = makeTest(name, options, cat)
+        if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories:
+          discard "run the test"
+        else:
+          test.spec.err = reJoined
+        testSpec r, test
+        inc testsRun
+      if testsRun == 0:
+        const whiteListedDirs = ["deps", "htmldocs", "pkgs"]
+          # `pkgs` because bug #16556 creates `pkgs` dirs and this can affect some users
+          # that try an old version of choosenim.
+        doAssert cat.string in whiteListedDirs,
+          "Invalid category specified: '$#' not in whilelist: $#" % [cat.string, $whiteListedDirs]
+
+proc processPattern(r: var TResults, pattern, options: string; simulate: bool) =
+  var testsRun = 0
+  if dirExists(pattern):
+    for k, name in walkDir(pattern):
+      if k in {pcFile, pcLinkToFile} and name.endsWith(".nim"):
+        if simulate:
+          echo "Detected test: ", name
+        else:
+          var test = makeTest(name, options, Category"pattern")
+          testSpec r, test
+        inc testsRun
+  else:
+    for name in walkPattern(pattern):
+      if simulate:
+        echo "Detected test: ", name
+      else:
+        var test = makeTest(name, options, Category"pattern")
+        testSpec r, test
+      inc testsRun
+  if testsRun == 0:
+    echo "no tests were found for pattern: ", pattern
diff --git a/testament/htmlgen.nim b/testament/htmlgen.nim
new file mode 100644
index 000000000..174f36d0b
--- /dev/null
+++ b/testament/htmlgen.nim
@@ -0,0 +1,149 @@
+#
+#
+#            Nim Tester
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## HTML generator for the tester.
+
+import strutils, json, os, times
+
+import "testamenthtml.nimf"
+
+proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) =
+  let
+    trId = htmlQuote(testResultRow["category"].str & "_" & testResultRow["name"].str).
+        multiReplace({".": "_", " ": "_", ":": "_", "/": "_"})
+    name = testResultRow["name"].str.htmlQuote()
+    category = testResultRow["category"].str.htmlQuote()
+    target = testResultRow["target"].str.htmlQuote()
+    action = testResultRow["action"].str.htmlQuote()
+    result = htmlQuote testResultRow["result"].str
+    expected = testResultRow["expected"].getStr
+    gotten = testResultRow["given"].getStr
+    timestamp = "unknown"
+  var
+    panelCtxClass, textCtxClass, bgCtxClass: string
+    resultSign, resultDescription: string
+  case result
+  of "reSuccess":
+    panelCtxClass = "success"
+    textCtxClass = "success"
+    bgCtxClass = "success"
+    resultSign = "ok"
+    resultDescription = "PASS"
+  of "reDisabled", "reJoined":
+    panelCtxClass = "info"
+    textCtxClass = "info"
+    bgCtxClass = "info"
+    resultSign = "question"
+    resultDescription = if result != "reJoined": "SKIP" else: "JOINED"
+  else:
+    panelCtxClass = "danger"
+    textCtxClass = "danger"
+    bgCtxClass = "danger"
+    resultSign = "exclamation"
+    resultDescription = "FAIL"
+
+  outfile.generateHtmlTestresultPanelBegin(
+    trId, name, target, category, action, resultDescription,
+    timestamp, result, resultSign, panelCtxClass, textCtxClass, bgCtxClass
+  )
+  if expected.isEmptyOrWhitespace() and gotten.isEmptyOrWhitespace():
+    outfile.generateHtmlTestresultOutputNone()
+  else:
+    outfile.generateHtmlTestresultOutputDetails(
+      expected.strip().htmlQuote,
+      gotten.strip().htmlQuote
+    )
+  outfile.generateHtmlTestresultPanelEnd()
+
+type
+  AllTests = object
+    data: JsonNode
+    totalCount, successCount, ignoredCount, failedCount: int
+    successPercentage, ignoredPercentage, failedPercentage: BiggestFloat
+
+proc allTestResults(onlyFailing = false): AllTests =
+  result.data = newJArray()
+  for file in os.walkFiles("testresults/*.json"):
+    let data = parseFile(file)
+    if data.kind != JArray:
+      echo "[ERROR] ignoring json file that is not an array: ", file
+    else:
+      for elem in data:
+        let state = elem["result"].str
+        inc result.totalCount
+        if state.contains("reSuccess"): inc result.successCount
+        elif state.contains("reDisabled") or state.contains("reJoined"):
+          inc result.ignoredCount
+        if not onlyFailing or not(state.contains("reSuccess")):
+          result.data.add elem
+  result.successPercentage = 100 *
+    (result.successCount.toBiggestFloat / result.totalCount.toBiggestFloat)
+  result.ignoredPercentage = 100 *
+    (result.ignoredCount.toBiggestFloat / result.totalCount.toBiggestFloat)
+  result.failedCount = result.totalCount -
+    result.successCount - result.ignoredCount
+  result.failedPercentage = 100 *
+    (result.failedCount.toBiggestFloat / result.totalCount.toBiggestFloat)
+
+proc generateTestResultsPanelGroupPartial(outfile: File, allResults: JsonNode) =
+  for testresultRow in allResults:
+    generateTestResultPanelPartial(outfile, testresultRow)
+
+proc generateAllTestsContent(outfile: File, allResults: AllTests,
+  onlyFailing = false) =
+  if allResults.data.len < 1: return # Nothing to do if there is no data.
+  # Only results from one test run means that test run environment info is the
+  # same for all tests
+  let
+    firstRow = allResults.data[0]
+    commit = htmlQuote firstRow["commit"].str
+    branch = htmlQuote firstRow["branch"].str
+    machine = htmlQuote firstRow["machine"].str
+
+  outfile.generateHtmlAllTestsBegin(
+    machine, commit, branch,
+    allResults.totalCount,
+    allResults.successCount,
+    formatBiggestFloat(allResults.successPercentage, ffDecimal, 2) & "%",
+    allResults.ignoredCount,
+    formatBiggestFloat(allResults.ignoredPercentage, ffDecimal, 2) & "%",
+    allResults.failedCount,
+    formatBiggestFloat(allResults.failedPercentage, ffDecimal, 2) & "%",
+    onlyFailing
+  )
+  generateTestResultsPanelGroupPartial(outfile, allResults.data)
+  outfile.generateHtmlAllTestsEnd()
+
+proc generateHtml*(filename: string, onlyFailing: bool) =
+  let
+    currentTime = getTime().local()
+    timestring = htmlQuote format(currentTime, "yyyy-MM-dd HH:mm:ss 'UTC'zzz")
+  var outfile = open(filename, fmWrite)
+
+  outfile.generateHtmlBegin()
+
+  generateAllTestsContent(outfile, allTestResults(onlyFailing), onlyFailing)
+
+  outfile.generateHtmlEnd(timestring)
+
+  outfile.flushFile()
+  close(outfile)
+
+proc dumpJsonTestResults*(prettyPrint, onlyFailing: bool) =
+  var
+    outfile = stdout
+    jsonString: string
+
+  let results = allTestResults(onlyFailing)
+  if prettyPrint:
+    jsonString = results.data.pretty()
+  else:
+    jsonString = $ results.data
+
+  outfile.writeLine(jsonString)
diff --git a/testament/important_packages.nim b/testament/important_packages.nim
new file mode 100644
index 000000000..efec04b3c
--- /dev/null
+++ b/testament/important_packages.nim
@@ -0,0 +1,193 @@
+##[
+## note 1
+`useHead` should ideally be used as the default but lots of packages (e.g. `chronos`)
+don't have release tags (or have really old ones compared to HEAD), making it
+impossible to test them reliably here.
+
+packages listed here should ideally have regularly updated release tags, so that:
+* we're testing recent versions of the package
+* the version that's tested is stable enough even if HEAD may occasionally break
+
+## note 2: D20210308T165435:here
+nimble packages should be testable as follows:
+git clone $url $dir && cd $dir
+NIMBLE_DIR=$TMP_NIMBLE_DIR XDG_CONFIG_HOME= nimble install --depsOnly -y
+NIMBLE_DIR=$TMP_NIMBLE_DIR XDG_CONFIG_HOME= nimble test
+
+if this fails (e.g. nimcrypto), it could be because a package lacks a `tests/nim.cfg` with `--path:..`,
+so the above commands would've worked by accident with `nimble install` but not with `nimble install --depsOnly`.
+When this is the case, a workaround is to test this package here by adding `--path:$srcDir` on the test `cmd`.
+]##
+
+type NimblePackage* = object
+  name*, cmd*, url*: string
+  useHead*: bool
+  allowFailure*: bool
+    ## When true, we still run the test but the test is allowed to fail.
+    ## This is useful for packages that currently fail but that we still want to
+    ## run in CI, e.g. so that we can monitor when they start working again and
+    ## are reminded about those failures without making CI fail for unrelated PRs.
+
+var packages*: seq[NimblePackage]
+
+proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true, allowFailure = false) =
+  packages.add NimblePackage(name: name, cmd: cmd, url: url, useHead: useHead, allowFailure: allowFailure)
+
+pkg "alea"
+pkg "argparse"
+pkg "arraymancer", "nim c tests/tests_cpu.nim"
+pkg "ast_pattern_matching", "nim c -r tests/test1.nim"
+pkg "asyncftpclient", "nimble compileExample"
+pkg "asyncthreadpool", "nimble test --mm:refc"
+pkg "awk"
+pkg "bigints"
+pkg "binaryheap", "nim c -r binaryheap.nim"
+pkg "BipBuffer"
+pkg "blscurve", allowFailure = true
+pkg "bncurve"
+pkg "brainfuck", "nim c -d:release -r tests/compile.nim"
+pkg "bump", "nim c --mm:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump", allowFailure = true
+pkg "c2nim", "nim c testsuite/tester.nim"
+pkg "cascade"
+pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true
+pkg "checksums"
+pkg "chroma"
+pkg "chronicles", "nim c -o:chr -r chronicles.nim"
+pkg "chronos", "nim c -r -d:release tests/testall"
+pkg "cligen", "nim c --path:. -r cligen.nim"
+pkg "combparser", "nimble test --mm:orc"
+pkg "compactdict"
+pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehension"
+pkg "constantine", "nimble make_lib"
+pkg "cowstrings"
+pkg "criterion", allowFailure = true # needs testing binary
+pkg "datamancer"
+pkg "dashing", "nim c tests/functional.nim"
+pkg "delaunay"
+pkg "dnsclient", allowFailure = true # super fragile
+pkg "docopt"
+pkg "dotenv"
+# when defined(linux): pkg "drchaos"
+pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl"
+pkg "elvis"
+pkg "faststreams"
+pkg "fidget"
+pkg "fragments", "nim c -r fragments/dsl.nim", allowFailure = true # pending https://github.com/nim-lang/packages/issues/2115 
+pkg "fusion"
+pkg "gara"
+pkg "glob"
+pkg "ggplotnim", "nim c -d:noCairo -r tests/tests.nim"
+pkg "gittyup", "nimble test", "https://github.com/disruptek/gittyup", allowFailure = true
+pkg "gnuplot", "nim c gnuplot.nim"
+# pkg "gram", "nim c -r --mm:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram"
+  # pending https://github.com/nim-lang/Nim/issues/16509
+pkg "hts", "nim c -o:htss src/hts.nim"
+pkg "httpauth"
+pkg "httputils"
+pkg "illwill", "nimble examples"
+pkg "inim"
+pkg "itertools", "nim doc src/itertools.nim"
+pkg "iterutils"
+pkg "json_rpc"
+pkg "json_serialization"
+pkg "jstin"
+pkg "karax", "nim c -r tests/tester.nim"
+pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree"
+pkg "loopfusion"
+pkg "lockfreequeues"
+pkg "macroutils"
+pkg "manu"
+pkg "markdown"
+pkg "measuremancer", "nimble testDeps; nimble -y test"
+pkg "memo"
+pkg "msgpack4nim", "nim c -r tests/test_spec.nim"
+pkg "nake", "nim c nakefile.nim"
+pkg "neo", "nim c -d:blas=openblas --mm:refc tests/all.nim"
+pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true, allowFailure = true
+  # inactive, tests not adapted to #23096
+pkg "netty"
+pkg "nico", allowFailure = true
+pkg "nicy", "nim c -r src/nicy.nim"
+pkg "nigui", "nim c -o:niguii -r src/nigui.nim"
+pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround needed, see D20210308T165435
+pkg "NimData", "nim c -o:nimdataa src/nimdata.nim"
+pkg "nimes", "nim c src/nimes.nim"
+pkg "nimfp", "nim c -o:nfp -r src/fp.nim"
+pkg "nimgame2", "nim c --mm:refc nimgame2/nimgame.nim"
+pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim"
+pkg "nimib"
+pkg "nimlsp"
+pkg "nimly", "nim c -r tests/test_readme_example.nim"
+pkg "nimongo", "nimble test_ci", allowFailure = true
+pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true
+pkg "nimPNG", useHead = true
+pkg "nimpy", "nim c -r tests/nimfrompy.nim"
+pkg "nimquery"
+pkg "nimsl"
+pkg "nimsvg"
+pkg "nimterop", "nimble minitest", url = "https://github.com/nim-lang/nimterop"
+pkg "nimwc", "nim c nimwc.nim"
+pkg "nimx", "nim c test/main.nim", allowFailure = true
+pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter"
+pkg "norm", "testament r tests/common/tmodel.nim"
+pkg "normalize"
+pkg "npeg", "nimble testarc"
+pkg "numericalnim", "nimble nimCI"
+pkg "optionsutils"
+pkg "ormin", "nim c -o:orminn ormin.nim"
+pkg "parsetoml"
+pkg "patty"
+pkg "pixie"
+pkg "plotly", "nim c examples/all.nim"
+pkg "pnm"
+pkg "polypbren"
+pkg "presto"
+pkg "prologue", "nimble tcompile"
+# remove fork after https://github.com/PMunch/combparser/pull/7 is merged:
+pkg "protobuf", "nimble install -y https://github.com/metagn/combparser@#HEAD; nim c -o:protobuff -r src/protobuf.nim"
+pkg "rbtree"
+pkg "react", "nimble example"
+pkg "regex", "nim c src/regex"
+pkg "results", "nim c -r results.nim"
+pkg "RollingHash", "nim c -r tests/test_cyclichash.nim"
+pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim"
+pkg "sdl1", "nim c -r src/sdl.nim"
+pkg "sdl2_nim", "nim c -r sdl2/sdl.nim"
+pkg "serialization"
+pkg "sigv4", "nim c --mm:arc -r sigv4.nim", "https://github.com/disruptek/sigv4"
+pkg "sim"
+pkg "smtp", "nimble compileExample"
+pkg "snip", "nimble test", "https://github.com/genotrance/snip"
+pkg "ssostrings"
+pkg "stew"
+pkg "stint", "nim c stint.nim"
+pkg "strslice"
+pkg "strunicode", "nim c -r --mm:refc src/strunicode.nim"
+pkg "supersnappy"
+pkg "synthesis"
+pkg "taskpools"
+pkg "telebot", "nim c -o:tbot -r src/telebot.nim"
+pkg "tempdir"
+pkg "templates"
+pkg "tensordsl", "nim c -r --mm:refc tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git"
+pkg "terminaltables", "nim c src/terminaltables.nim"
+pkg "termstyle", "nim c -r termstyle.nim"
+pkg "testutils"
+pkg "timeit"
+pkg "timezones"
+pkg "tiny_sqlite"
+pkg "unicodedb", "nim c -d:release -r tests/tests.nim"
+pkg "unicodeplus", "nim c -d:release -r tests/tests.nim"
+pkg "union", "nim c -r tests/treadme.nim", url = "https://github.com/alaviss/union"
+pkg "unittest2"
+pkg "unpack"
+pkg "weave", "nimble install -y cligen@#HEAD; nimble test_gc_arc", useHead = true
+pkg "websock"
+pkg "websocket", "nim c websocket.nim"
+# pkg "winim", allowFailure = true
+pkg "with"
+pkg "ws", allowFailure = true
+pkg "yaml"
+pkg "zero_functional", "nim c -r test.nim"
+pkg "zippy"
+pkg "zxcvbn"
diff --git a/testament/lib/readme.md b/testament/lib/readme.md
new file mode 100644
index 000000000..20e866338
--- /dev/null
+++ b/testament/lib/readme.md
@@ -0,0 +1,4 @@
+This directory contains helper files used by several tests, to avoid
+code duplication, akin to a std extension tailored for testament.
+
+Some of these could later migrate to stdlib.
diff --git a/testament/lib/stdtest/netutils.nim b/testament/lib/stdtest/netutils.nim
new file mode 100644
index 000000000..5115390e0
--- /dev/null
+++ b/testament/lib/stdtest/netutils.nim
@@ -0,0 +1,13 @@
+import std/[nativesockets, asyncdispatch, os]
+
+proc bindAvailablePort*(handle: SocketHandle, port = Port(0)): Port =
+  ## See also `asynchttpserver.getPort`.
+  block:
+    var name: Sockaddr_in
+    name.sin_family = typeof(name.sin_family)(toInt(AF_INET))
+    name.sin_port = htons(uint16(port))
+    name.sin_addr.s_addr = htonl(INADDR_ANY)
+    if bindAddr(handle, cast[ptr SockAddr](addr(name)),
+                sizeof(name).Socklen) < 0'i32:
+      raiseOSError(osLastError(), $port)
+  result = getLocalAddr(handle, AF_INET)[1]
diff --git a/testament/lib/stdtest/specialpaths.nim b/testament/lib/stdtest/specialpaths.nim
new file mode 100644
index 000000000..e214d113d
--- /dev/null
+++ b/testament/lib/stdtest/specialpaths.nim
@@ -0,0 +1,55 @@
+#[
+todo: move findNimStdLibCompileTime, findNimStdLib here
+xxx: factor pending https://github.com/timotheecour/Nim/issues/616
+
+## note: $lib vs $nim
+note: these can resolve to 3 different paths if running via `nim c --lib:lib foo`,
+eg if compiler was installed via nimble (or is in nim path), and nim is external
+(ie not in `$lib/../bin/` dir)
+
+import "$lib/../compiler/nimpaths" # <- most robust if you want to favor --lib:lib
+import "$nim/compiler/nimpaths"
+import compiler/nimpaths
+]#
+
+import os
+when defined(nimPreviewSlimSystem):
+  import std/assertions
+
+# Note: all the const paths defined here are known at compile time and valid
+# so long Nim repo isn't relocated after compilation.
+# This means the binaries they produce will embed hardcoded paths, which
+# isn't appropriate for some applications that need to be relocatable.
+
+const
+  sourcePath = currentSourcePath()
+    # robust way to derive other paths here
+    # We don't depend on PATH so this is robust to having multiple nim binaries
+  nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir ## root of Nim repo
+  testsFname* = "tests"
+  stdlibDir* = nimRootDir / "lib"
+  systemPath* = stdlibDir / "system.nim"
+  testsDir* = nimRootDir / testsFname
+  buildDir* = nimRootDir / "build"
+    ## refs #10268: all testament generated files should go here to avoid
+    ## polluting .gitignore
+
+proc splitTestFile*(file: string): tuple[cat: string, path: string] =
+  ## At least one directory is required in the path, to use as a category name
+  runnableExamples:
+    doAssert splitTestFile("tests/fakedir/tfakename.nim") == ("fakedir", "tests/fakedir/tfakename.nim".unixToNativePath)
+  for p in file.parentDirs(inclusive = false):
+    let parent = p.parentDir
+    if parent.lastPathPart == testsFname:
+      result.cat = p.lastPathPart
+      let dir = getCurrentDir()
+      if file.isRelativeTo(dir):
+        result.path = file.relativePath(dir)
+      else:
+        result.path = file
+      return result
+  raiseAssert "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'"
+
+static:
+  # sanity check
+  doAssert fileExists(systemPath)
diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim
new file mode 100644
index 000000000..a490b17c8
--- /dev/null
+++ b/testament/lib/stdtest/testutils.nim
@@ -0,0 +1,126 @@
+import std/private/miscdollars
+when defined(nimscript):
+  import std/os # xxx investigate why needed
+else:
+  from std/os import getEnv
+import std/[macros, genasts]
+
+template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) =
+  ## API to deal with flaky or failing tests. This avoids disabling entire tests
+  ## altogether so that at least the parts that are working are kept being
+  ## tested. This also avoids making CI fail periodically for tests known to
+  ## be flaky. Finally, for known failures, passing `notifySuccess = true` will
+  ## log that the test succeeded, which may indicate that a bug was fixed
+  ## "by accident" and should be looked into.
+  const info = instantiationInfo(-1, true)
+  const expr = astToStr(cond)
+  if cond and not notifySuccess:
+    discard # silent success
+  else:
+    var msg2 = ""
+    toLocation(msg2, info.filename, info.line, info.column)
+    if cond:
+      # a flaky test is failing, we still report it but we don't fail CI
+      msg2.add " FLAKY_SUCCESS "
+    else:
+      # a previously failing test is now passing, a pre-existing bug might've been
+      # fixed by accidend
+      msg2.add " FLAKY_FAILURE "
+    msg2.add $expr & " " & msg
+    echo msg2
+
+when not defined(js) and not defined(nimscript):
+  import std/strutils
+
+  proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool =
+    ## Returns true if each stripped line in `lhs` appears in rhs, using a greedy matching.
+    # xxx improve error reporting by showing the last matched pair
+    iterator splitLinesClosure(): string {.closure.} =
+      for line in splitLines(rhs.strip):
+        yield line
+    template isMatch(lhsi, rhsi): bool =
+      if allowPrefixMatch:
+        startsWith(rhsi, lhsi)
+      else:
+        lhsi == rhsi
+
+    var rhsIter = splitLinesClosure
+    var currentLine = strip(rhsIter())
+
+    for line in lhs.strip.splitLines:
+      let line = line.strip
+      if line.len != 0:
+        while not isMatch(line, currentLine):
+          currentLine = strip(rhsIter())
+          if rhsIter.finished:
+            return false
+
+      if rhsIter.finished:
+        return false
+    return true
+
+template enableRemoteNetworking*: bool =
+  ## Allows contolling whether to run some test at a statement-level granularity.
+  ## Using environment variables simplifies propagating this all the way across
+  ## process calls, e.g. `testament all` calls itself, which in turns invokes
+  ## a `nim` invocation (possibly via additional intermediate processes).
+  getEnv("NIM_TESTAMENT_REMOTE_NETWORKING") == "1"
+
+template disableSSLTesting*: bool =
+  ## TODO: workaround for GitHub Action gcc 14 matrix; remove this
+  ## matrix and the flag after Azure agent supports ubuntu 24.04
+  getEnv("NIM_TESTAMENT_DISABLE_SSL") == "1"
+
+template whenRuntimeJs*(bodyIf, bodyElse) =
+  ##[
+  Behaves as `when defined(js) and not nimvm` (which isn't legal yet).
+  pending improvements to `nimvm`, this sugar helps; use as follows:
+
+  whenRuntimeJs:
+    doAssert defined(js)
+    when nimvm: doAssert false
+    else: discard
+  do:
+    discard
+  ]##
+  when nimvm: bodyElse
+  else:
+    when defined(js): bodyIf
+    else: bodyElse
+
+template whenVMorJs*(bodyIf, bodyElse) =
+  ## Behaves as: `when defined(js) or nimvm`
+  when nimvm: bodyIf
+  else:
+    when defined(js): bodyIf
+    else: bodyElse
+
+template accept*(a) =
+  doAssert compiles(a)
+
+template reject*(a) =
+  doAssert not compiles(a)
+
+template disableVm*(body) =
+  when nimvm: discard
+  else: body
+
+macro assertAll*(body) =
+  ## works in VM, unlike `check`, `require`
+  runnableExamples:
+    assertAll:
+      1+1 == 2
+      var a = @[1, 2] # statements work
+      a.len == 2
+  # remove this once these support VM, pending #10129 (closed but not yet fixed)
+  result = newStmtList()
+  for a in body:
+    result.add genAst(a, a2 = a.repr, info = lineInfo(a)) do:
+      # D20210421T014713:here
+      # xxx pending https://github.com/nim-lang/Nim/issues/12030,
+      # `typeof` should introduce its own scope, so that this
+      # is sufficient: `typeof(a)` instead of `typeof(block: a)`
+      when typeof(block: a) is void: a
+      else:
+        if not a:
+          raise newException(AssertionDefect, info & " " & a2)
diff --git a/testament/lib/stdtest/unittest_light.nim b/testament/lib/stdtest/unittest_light.nim
new file mode 100644
index 000000000..4ab1d7543
--- /dev/null
+++ b/testament/lib/stdtest/unittest_light.nim
@@ -0,0 +1,37 @@
+import std/assertions
+
+
+proc mismatch*[T](lhs: T, rhs: T): string =
+  ## Simplified version of `unittest.require` that satisfies a common use case,
+  ## while avoiding pulling too many dependencies. On failure, diagnostic
+  ## information is provided that in particular makes it easy to spot
+  ## whitespace mismatches and where the mismatch is.
+  proc replaceInvisible(s: string): string =
+    for a in s:
+      case a
+      of '\n': result.add "\\n\n"
+      else: result.add a
+
+  proc quoted(s: string): string = result.addQuoted s
+
+  result.add '\n'
+  result.add "lhs:{" & replaceInvisible(
+      $lhs) & "}\nrhs:{" & replaceInvisible($rhs) & "}\n"
+  when compiles(lhs.len):
+    if lhs.len != rhs.len:
+      result.add "lhs.len: " & $lhs.len & " rhs.len: " & $rhs.len & '\n'
+    when compiles(lhs[0]):
+      var i = 0
+      while i < lhs.len and i < rhs.len:
+        if lhs[i] != rhs[i]: break
+        i.inc
+      result.add "first mismatch index: " & $i & '\n'
+      if i < lhs.len and i < rhs.len:
+        result.add "lhs[i]: {" & quoted($lhs[i]) & "}\nrhs[i]: {" & quoted(
+            $rhs[i]) & "}\n"
+      result.add "lhs[0..<i]:{" & replaceInvisible($lhs[
+          0..<i]) & '}'
+
+proc assertEquals*[T](lhs: T, rhs: T, msg = "") =
+  if lhs != rhs:
+    doAssert false, mismatch(lhs, rhs) & '\n' & msg
diff --git a/testament/specs.nim b/testament/specs.nim
new file mode 100644
index 000000000..c3040c1d8
--- /dev/null
+++ b/testament/specs.nim
@@ -0,0 +1,521 @@
+#
+#
+#            Nim Tester
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import sequtils, parseutils, strutils, os, streams, parsecfg,
+  tables, hashes, sets
+import compiler/platform
+
+type TestamentData* = ref object
+  # better to group globals under 1 object; could group the other ones here too
+  batchArg*: string
+  testamentNumBatch*: int
+  testamentBatch*: int
+
+let testamentData0* = TestamentData()
+
+var compilerPrefix* = findExe("nim")
+
+let isTravis* = existsEnv("TRAVIS")
+let isAppVeyor* = existsEnv("APPVEYOR")
+let isAzure* = existsEnv("TF_BUILD")
+
+var skips*: seq[string]
+
+type
+  TTestAction* = enum
+    actionRun = "run"
+    actionCompile = "compile"
+    actionReject = "reject"
+
+  TOutputCheck* = enum
+    ocIgnore = "ignore"
+    ocEqual = "equal"
+    ocSubstr = "substr"
+
+  TResultEnum* = enum
+    reNimcCrash,       # nim compiler seems to have crashed
+    reMsgsDiffer,      # error messages differ
+    reFilesDiffer,     # expected and given filenames differ
+    reLinesDiffer,     # expected and given line numbers differ
+    reOutputsDiffer,
+    reExitcodesDiffer, # exit codes of program or of valgrind differ
+    reTimeout,
+    reInvalidPeg,
+    reCodegenFailure,
+    reCodeNotFound,
+    reExeNotFound,
+    reInstallFailed    # package installation failed
+    reBuildFailed      # package building failed
+    reDisabled,        # test is disabled
+    reJoined,          # test is disabled because it was joined into the megatest
+    reSuccess          # test was successful
+    reInvalidSpec      # test had problems to parse the spec
+
+  TTarget* = enum
+    targetC = "c"
+    targetCpp = "cpp"
+    targetObjC = "objc"
+    targetJS = "js"
+
+  InlineError* = object
+    kind*: string
+    msg*: string
+    line*, col*: int
+
+  ValgrindSpec* = enum
+    disabled, enabled, leaking
+
+  TSpec* = object
+    # xxx make sure `isJoinableSpec` takes into account each field here.
+    action*: TTestAction
+    file*, cmd*: string
+    filename*: string ## Test filename (without path).
+    input*: string
+    outputCheck*: TOutputCheck
+    sortoutput*: bool
+    output*: string
+    line*, column*: int
+    exitCode*: int
+    msg*: string
+    ccodeCheck*: seq[string]
+    maxCodeSize*: int
+    err*: TResultEnum
+    inCurrentBatch*: bool
+    targets*: set[TTarget]
+    matrix*: seq[string]
+    nimout*: string
+    nimoutFull*: bool # whether nimout is all compiler output or a subset
+    parseErrors*: string            # when the spec definition is invalid, this is not empty.
+    unjoinable*: bool
+    unbatchable*: bool
+      # whether this test can be batchable via `NIM_TESTAMENT_BATCH`; only very
+      # few tests are not batchable; the ones that are not could be turned batchable
+      # by making the dependencies explicit
+    useValgrind*: ValgrindSpec
+    timeout*: float # in seconds, fractions possible,
+                      # but don't rely on much precision
+    inlineErrors*: seq[InlineError] # line information to error message
+    debugInfo*: string # debug info to give more context
+
+proc getCmd*(s: TSpec): string =
+  if s.cmd.len == 0:
+    result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:build/deps/pkgs2 $options $file"
+  else:
+    result = s.cmd
+
+const
+  targetToExt*: array[TTarget, string] = ["nim.c", "nim.cpp", "nim.m", "js"]
+  targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"]
+
+proc defaultOptions*(a: TTarget): string =
+  case a
+  of targetJS: "-d:nodejs"
+    # once we start testing for `nim js -d:nimbrowser` (eg selenium or similar),
+    # we can adapt this logic; or a given js test can override with `-u:nodejs`.
+  else: ""
+
+when not declared(parseCfgBool):
+  # candidate for the stdlib:
+  proc parseCfgBool(s: string): bool =
+    case normalize(s)
+    of "y", "yes", "true", "1", "on": result = true
+    of "n", "no", "false", "0", "off": result = false
+    else: raise newException(ValueError, "cannot interpret as a bool: " & s)
+
+proc addLine*(self: var string; pieces: varargs[string]) =
+  for piece in pieces:
+    self.add piece
+  self.add "\n"
+
+
+const
+  inlineErrorKindMarker = "tt."
+  inlineErrorMarker = "#[" & inlineErrorKindMarker
+
+proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int =
+  ## Extract inline error messages.
+  ##
+  ## Can parse a single message for a line:
+  ##
+  ##   ```nim
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name] ]#
+  ##   ```
+  ##
+  ## Can parse multiple messages for a line when they are separated by ';':
+  ##
+  ##   ```nim
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error
+  ##                           ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error
+  ##                                       ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
+  ##   ```
+  ##
+  ##   ```nim
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name];
+  ##     tt.Error              ^ 'no_destroy' should be: 'nodestroy' [Name];
+  ##     tt.Error                          ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
+  ##   ```
+  result = i + len(inlineErrorMarker)
+  inc col, len(inlineErrorMarker)
+  let msgLine = line
+  var msgCol = -1
+  var msg = ""
+  var kind = ""
+
+  template parseKind =
+    while result < s.len and s[result] in IdentChars:
+      kind.add s[result]
+      inc result
+      inc col
+    if kind notin ["Hint", "Warning", "Error"]:
+      spec.parseErrors.addLine "expected inline message kind: Hint, Warning, Error"
+
+  template skipWhitespace =
+    while result < s.len and s[result] in Whitespace:
+      if s[result] == '\n':
+        col = 1
+        inc line
+      else:
+        inc col
+      inc result
+
+  template parseCaret =
+    if result < s.len and s[result] == '^':
+      msgCol = col
+      inc result
+      inc col
+      skipWhitespace()
+    else:
+      spec.parseErrors.addLine "expected column marker ('^') for inline message"
+
+  template isMsgDelimiter: bool =
+    s[result] == ';' and
+    (block:
+      let nextTokenIdx = result + 1 + parseutils.skipWhitespace(s, result + 1)
+      if s.len > nextTokenIdx + len(inlineErrorKindMarker) and
+         s[nextTokenIdx..(nextTokenIdx + len(inlineErrorKindMarker) - 1)] == inlineErrorKindMarker:
+        true
+      else:
+        false)
+
+  template trimTrailingMsgWhitespace =
+    while msg.len > 0 and msg[^1] in Whitespace:
+      setLen msg, msg.len - 1
+
+  template addInlineError =
+    doAssert msg[^1] notin Whitespace
+    if kind == "Error": spec.action = actionReject
+    spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: msgLine, col: msgCol)
+
+  parseKind()
+  skipWhitespace()
+  parseCaret()
+
+  while result < s.len-1:
+    if s[result] == '\n':
+      if result > 0 and s[result - 1] == '\r':
+        msg[^1] = '\n'
+      else:
+        msg.add '\n'
+      inc result
+      inc line
+      col = 1
+    elif isMsgDelimiter():
+      trimTrailingMsgWhitespace()
+      inc result
+      skipWhitespace()
+      addInlineError()
+      inc result, len(inlineErrorKindMarker)
+      inc col, 1 + len(inlineErrorKindMarker)
+      kind.setLen 0
+      msg.setLen 0
+      parseKind()
+      skipWhitespace()
+      parseCaret()
+    elif s[result] == ']' and s[result+1] == '#':
+      trimTrailingMsgWhitespace()
+      inc result, 2
+      inc col, 2
+      addInlineError()
+      break
+    else:
+      msg.add s[result]
+      inc result
+      inc col
+
+  if spec.inlineErrors.len > 0:
+    spec.unjoinable = true
+
+proc extractSpec(filename: string; spec: var TSpec): string =
+  const
+    tripleQuote = "\"\"\""
+    specStart = "discard " & tripleQuote
+  var s = readFile(filename)
+
+  var i = 0
+  var a = -1
+  var b = -1
+  var line = 1
+  var col = 1
+  while i < s.len:
+    if (i == 0 or s[i-1] != ' ') and s.continuesWith(specStart, i):
+      # `s[i-1] == '\n'` would not work because of `tests/stdlib/tbase64.nim` which contains BOM (https://en.wikipedia.org/wiki/Byte_order_mark)
+      const lineMax = 10
+      if a != -1:
+        raise newException(ValueError, "testament spec violation: duplicate `specStart` found: " & $(filename, a, b, line))
+      elif line > lineMax:
+        # not overly restrictive, but prevents mistaking some `specStart` as spec if deeep inside a test file
+        raise newException(ValueError, "testament spec violation: `specStart` should be before line $1, or be indented; info: $2" % [$lineMax, $(filename, a, b, line)])
+      i += specStart.len
+      a = i
+    elif a > -1 and b == -1 and s.continuesWith(tripleQuote, i):
+      b = i
+      i += tripleQuote.len
+    elif s[i] == '\n':
+      inc line
+      inc i
+      col = 1
+    elif s.continuesWith(inlineErrorMarker, i):
+      i = extractErrorMsg(s, i, line, col, spec)
+    else:
+      inc col
+      inc i
+
+  if a >= 0 and b > a:
+    result = s.substr(a, b-1).multiReplace({"'''": tripleQuote, "\\31": "\31"})
+  elif a >= 0:
+    raise newException(ValueError, "testament spec violation: `specStart` found but not trailing `tripleQuote`: $1" % $(filename, a, b, line))
+  else:
+    result = ""
+
+proc parseTargets*(value: string): set[TTarget] =
+  for v in value.normalize.splitWhitespace:
+    case v
+    of "c": result.incl(targetC)
+    of "cpp", "c++": result.incl(targetCpp)
+    of "objc": result.incl(targetObjC)
+    of "js": result.incl(targetJS)
+    else: raise newException(ValueError, "invalid target: '$#'" % v)
+
+proc initSpec*(filename: string): TSpec =
+  result.file = filename
+
+proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool =
+  if testamentData.testamentNumBatch != 0:
+    hash(filename) mod testamentData.testamentNumBatch == testamentData.testamentBatch
+  else:
+    true
+
+proc parseSpec*(filename: string): TSpec =
+  result.file = filename
+  result.filename = extractFilename(filename)
+  let specStr = extractSpec(filename, result)
+  var ss = newStringStream(specStr)
+  var p: CfgParser
+  open(p, ss, filename, 1)
+  var flags: HashSet[string]
+  var nimoutFound = false
+  while true:
+    var e = next(p)
+    case e.kind
+    of cfgKeyValuePair:
+      let key = e.key.normalize
+      const whiteListMulti = ["disabled", "ccodecheck"]
+        ## list of flags that are correctly handled when passed multiple times
+        ## (instead of being overwritten)
+      if key notin whiteListMulti:
+        doAssert key notin flags, $(key, filename)
+      flags.incl key
+      case key
+      of "action":
+        case e.value.normalize
+        of "compile":
+          result.action = actionCompile
+        of "run":
+          result.action = actionRun
+        of "reject":
+          result.action = actionReject
+        else:
+          result.parseErrors.addLine "cannot interpret as action: ", e.value
+      of "file":
+        if result.msg.len == 0 and result.nimout.len == 0:
+          result.parseErrors.addLine "errormsg or msg needs to be specified before file"
+        result.file = e.value
+      of "line":
+        if result.msg.len == 0 and result.nimout.len == 0:
+          result.parseErrors.addLine "errormsg, msg or nimout needs to be specified before line"
+        discard parseInt(e.value, result.line)
+      of "column":
+        if result.msg.len == 0 and result.nimout.len == 0:
+          result.parseErrors.addLine "errormsg or msg needs to be specified before column"
+        discard parseInt(e.value, result.column)
+      of "output":
+        if result.outputCheck != ocSubstr:
+          result.outputCheck = ocEqual
+        result.output = e.value
+      of "input":
+        result.input = e.value
+      of "outputsub":
+        result.outputCheck = ocSubstr
+        result.output = strip(e.value)
+      of "sortoutput":
+        try:
+          result.sortoutput = parseCfgBool(e.value)
+        except:
+          result.parseErrors.addLine getCurrentExceptionMsg()
+      of "exitcode":
+        discard parseInt(e.value, result.exitCode)
+        result.action = actionRun
+      of "errormsg":
+        result.msg = e.value
+        result.action = actionReject
+      of "nimout":
+        result.nimout = e.value
+        nimoutFound = true
+      of "nimoutfull":
+        result.nimoutFull = parseCfgBool(e.value)
+      of "batchable":
+        result.unbatchable = not parseCfgBool(e.value)
+      of "joinable":
+        result.unjoinable = not parseCfgBool(e.value)
+      of "valgrind":
+        when defined(linux) and sizeof(int) == 8:
+          result.useValgrind = if e.value.normalize == "leaks": leaking
+                               else: ValgrindSpec(parseCfgBool(e.value))
+          result.unjoinable = true
+          if result.useValgrind != disabled:
+            result.outputCheck = ocSubstr
+        else:
+          # Windows lacks valgrind. Silly OS.
+          # Valgrind only supports OSX <= 17.x
+          result.useValgrind = disabled
+      of "disabled":
+        let value = e.value.normalize
+        case value
+        of "y", "yes", "true", "1", "on": result.err = reDisabled
+        of "n", "no", "false", "0", "off": discard
+        # These values are defined in `compiler/options.isDefined`
+        of "win":
+          when defined(windows): result.err = reDisabled
+        of "linux":
+          when defined(linux): result.err = reDisabled
+        of "bsd":
+          when defined(bsd): result.err = reDisabled
+        of "osx":
+          when defined(osx): result.err = reDisabled
+        of "unix", "posix":
+          when defined(posix): result.err = reDisabled
+        of "freebsd":
+          when defined(freebsd): result.err = reDisabled
+        of "littleendian":
+          when defined(littleendian): result.err = reDisabled
+        of "bigendian":
+          when defined(bigendian): result.err = reDisabled
+        of "cpu8", "8bit":
+          when defined(cpu8): result.err = reDisabled
+        of "cpu16", "16bit":
+          when defined(cpu16): result.err = reDisabled
+        of "cpu32", "32bit":
+          when defined(cpu32): result.err = reDisabled
+        of "cpu64", "64bit":
+          when defined(cpu64): result.err = reDisabled
+        # These values are for CI environments
+        of "travis": # deprecated
+          if isTravis: result.err = reDisabled
+        of "appveyor": # deprecated
+          if isAppVeyor: result.err = reDisabled
+        of "azure":
+          if isAzure: result.err = reDisabled
+        else:
+          # Check whether the value exists as an OS or CPU that is
+          # defined in `compiler/platform`.
+          block checkHost:
+            for os in platform.OS:
+              # Check if the value exists as OS.
+              if value == os.name.normalize:
+                # The value exists; is it the same as the current host?
+                if value == hostOS.normalize:
+                  # The value exists and is the same as the current host,
+                  # so disable the test.
+                  result.err = reDisabled
+                # The value was defined, so there is no need to check further
+                # values or raise an error.
+                break checkHost
+            for cpu in platform.CPU:
+              # Check if the value exists as CPU.
+              if value == cpu.name.normalize:
+                # The value exists; is it the same as the current host?
+                if value == hostCPU.normalize:
+                  # The value exists and is the same as the current host,
+                  # so disable the test.
+                  result.err = reDisabled
+                # The value was defined, so there is no need to check further
+                # values or raise an error.
+                break checkHost
+            # The value doesn't exist as an OS, CPU, or any previous value
+            # defined in this case statement, so raise an error.
+            result.parseErrors.addLine "cannot interpret as a bool: ", e.value
+      of "cmd":
+        if e.value.startsWith("nim "):
+          result.cmd = compilerPrefix & e.value[3..^1]
+        else:
+          result.cmd = e.value
+      of "ccodecheck":
+        result.ccodeCheck.add e.value
+      of "maxcodesize":
+        discard parseInt(e.value, result.maxCodeSize)
+      of "timeout":
+        try:
+          result.timeout = parseFloat(e.value)
+        except ValueError:
+          result.parseErrors.addLine "cannot interpret as a float: ", e.value
+      of "targets", "target":
+        try:
+          result.targets.incl parseTargets(e.value)
+        except ValueError as e:
+          result.parseErrors.addLine e.msg
+      of "matrix":
+        for v in e.value.split(';'):
+          result.matrix.add(v.strip)
+      else:
+        result.parseErrors.addLine "invalid key for test spec: ", e.key
+
+    of cfgSectionStart:
+      result.parseErrors.addLine "section ignored: ", e.section
+    of cfgOption:
+      result.parseErrors.addLine "command ignored: ", e.key & ": " & e.value
+    of cfgError:
+      result.parseErrors.addLine e.msg
+    of cfgEof:
+      break
+  close(p)
+
+  if skips.anyIt(it in result.file):
+    result.err = reDisabled
+
+  if nimoutFound and result.nimout.len == 0 and not result.nimoutFull:
+    result.parseErrors.addLine "empty `nimout` is vacuously true, use `nimoutFull:true` if intentional"
+
+  result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable
+  if not result.inCurrentBatch:
+    result.err = reDisabled
+
+  # Interpolate variables in msgs:
+  template varSub(msg: string): string =
+    try:
+      msg % ["/", $DirSep, "file", result.filename]
+    except ValueError:
+      result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unit-tests-output-message-variable-interpolation')"
+      msg
+  result.nimout = result.nimout.varSub
+  result.msg = result.msg.varSub
+  for inlineError in result.inlineErrors.mitems:
+    inlineError.msg = inlineError.msg.varSub
diff --git a/testament/testament.nim b/testament/testament.nim
new file mode 100644
index 000000000..1e892e636
--- /dev/null
+++ b/testament/testament.nim
@@ -0,0 +1,806 @@
+#
+#
+#            Nim Testament
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This program verifies Nim against the testcases.
+
+import
+  std/[strutils, pegs, os, osproc, streams, json,
+    parseopt, browsers, terminal, exitprocs,
+    algorithm, times, intsets, macros]
+
+import backend, specs, azure, htmlgen
+
+from std/sugar import dup
+import compiler/nodejs
+import lib/stdtest/testutils
+from lib/stdtest/specialpaths import splitTestFile
+from std/private/gitutils import diffStrings
+
+import ../dist/checksums/src/checksums/md5
+
+proc trimUnitSep(x: var string) =
+  let L = x.len
+  if L > 0 and x[^1] == '\31':
+    setLen x, L-1
+
+var useColors = true
+var backendLogging = true
+var simulate = false
+var optVerbose = false
+var useMegatest = true
+var valgrindEnabled = true
+
+proc verboseCmd(cmd: string) =
+  if optVerbose:
+    echo "executing: ", cmd
+
+const
+  failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs
+  testsDir = "tests" & DirSep
+  resultsFile = "testresults.html"
+  Usage = """Usage:
+  testament [options] command [arguments]
+
+Command:
+  p|pat|pattern <glob>        run all the tests matching the given pattern
+  all                         run all tests in category folders
+  c|cat|category <category>   run all the tests of a certain category
+  r|run <test>                run single test file
+  html                        generate $1 from the database
+Arguments:
+  arguments are passed to the compiler
+Options:
+  --print                   print results to the console
+  --verbose                 print commands (compiling and running tests)
+  --simulate                see what tests would be run but don't run them (for debugging)
+  --failing                 only show failing/ignored tests
+  --targets:"c cpp js objc" run tests for specified targets (default: c)
+  --nim:path                use a particular nim executable (default: $$PATH/nim)
+  --directory:dir           Change to directory dir before reading the tests or doing anything else.
+  --colors:on|off           Turn messages coloring on|off.
+  --backendLogging:on|off   Disable or enable backend logging. By default turned on.
+  --megatest:on|off         Enable or disable megatest. Default is on.
+  --valgrind:on|off         Enable or disable valgrind support. Default is on.
+  --skipFrom:file           Read tests to skip from `file` - one test per line, # comments ignored
+
+On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API
+provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN.
+
+Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables
+tests with remote networking (as in CI).
+""" % resultsFile
+
+proc isNimRepoTests(): bool =
+  # this logic could either be specific to cwd, or to some file derived from
+  # the input file, eg testament r /pathto/tests/foo/tmain.nim; we choose
+  # the former since it's simpler and also works with `testament all`.
+  let file = "testament"/"testament.nim.cfg"
+  result = file.fileExists
+
+type
+  Category = distinct string
+  TResults = object
+    total, passed, failedButAllowed, skipped: int
+      ## xxx rename passed to passedOrAllowedFailure
+    data: string
+  TTest = object
+    name: string
+    cat: Category
+    options: string
+    testArgs: seq[string]
+    spec: TSpec
+    startTime: float
+    debugInfo: string
+
+# ----------------------------------------------------------------------------
+
+let
+  pegLineError =
+    peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}"
+  pegOtherError = peg"'Error:' \s* {.*}"
+  pegOfInterest = pegLineError / pegOtherError
+
+var gTargets = {low(TTarget)..high(TTarget)}
+var targetsSet = false
+
+proc isSuccess(input: string): bool =
+  # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs
+  # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we
+  # could handle colors, there would be other issues such as handling other flags
+  # that may appear in user config (eg: `--filenames`).
+  # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config
+  # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory
+  input.startsWith("Hint: ") and input.endsWith("[SuccessX]")
+
+proc getFileDir(filename: string): string =
+  result = filename.splitFile().dir
+  if not result.isAbsolute():
+    result = getCurrentDir() / result
+
+proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: string = ""): tuple[
+                cmdLine: string,
+                output: string,
+                exitCode: int] {.tags:
+                [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
+
+  result.cmdLine.add quoteShell(command)
+  for arg in args:
+    result.cmdLine.add ' '
+    result.cmdLine.add quoteShell(arg)
+  verboseCmd(result.cmdLine)
+  var p = startProcess(command, workingDir = workingDir, args = args,
+                       options = {poStdErrToStdOut, poUsePath})
+  var outp = outputStream(p)
+
+  # There is no way to provide input for the child process
+  # anymore. Closing it will create EOF on stdin instead of eternal
+  # blocking.
+  let instream = inputStream(p)
+  instream.write(input)
+  close instream
+
+  result.exitCode = -1
+  var line = newStringOfCap(120)
+  while true:
+    if outp.readLine(line):
+      result.output.add line
+      result.output.add '\n'
+    else:
+      result.exitCode = peekExitCode(p)
+      if result.exitCode != -1: break
+  close(p)
+
+proc nimcacheDir(filename, options: string, target: TTarget): string =
+  ## Give each test a private nimcache dir so they don't clobber each other's.
+  let hashInput = options & $target
+  result = "nimcache" / (filename & '_' & hashInput.getMD5)
+
+proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string,
+                     target: TTarget, extraOptions = ""): string =
+  var options = target.defaultOptions & ' ' & options
+  if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell)
+  options.add ' ' & extraOptions
+  # we avoid using `parseCmdLine` which is buggy, refs bug #14343
+  result = cmdTemplate % ["target", targetToCmd[target],
+                      "options", options, "file", filename.quoteShell,
+                      "filedir", filename.getFileDir(), "nim", compilerPrefix]
+
+proc callNimCompiler(cmdTemplate, filename, options, nimcache: string,
+                     target: TTarget, extraOptions = ""): TSpec =
+  result.cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache, target,
+                          extraOptions)
+  verboseCmd(result.cmd)
+  var p = startProcess(command = result.cmd,
+                       options = {poStdErrToStdOut, poUsePath, poEvalCommand})
+  let outp = p.outputStream
+  var foundSuccessMsg = false
+  var foundErrorMsg = false
+  var err = ""
+  var x = newStringOfCap(120)
+  result.nimout = ""
+  while true:
+    if outp.readLine(x):
+      trimUnitSep x
+      result.nimout.add(x & '\n')
+      if x =~ pegOfInterest:
+        # `err` should contain the last error message
+        err = x
+        foundErrorMsg = true
+      elif x.isSuccess:
+        foundSuccessMsg = true
+    elif not running(p):
+      break
+  result.msg = ""
+  result.file = ""
+  result.output = ""
+  result.line = 0
+  result.column = 0
+
+  result.err = reNimcCrash
+  result.exitCode = p.peekExitCode
+  close p
+  case result.exitCode
+  of 0:
+    if foundErrorMsg:
+      result.debugInfo.add " compiler exit code was 0 but some Error's were found."
+    else:
+      result.err = reSuccess
+  of 1:
+    if not foundErrorMsg:
+      result.debugInfo.add " compiler exit code was 1 but no Error's were found."
+    if foundSuccessMsg:
+      result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true."
+  else:
+    result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode
+
+  if err =~ pegLineError:
+    result.file = extractFilename(matches[0])
+    result.line = parseInt(matches[1])
+    result.column = parseInt(matches[2])
+    result.msg = matches[3]
+  elif err =~ pegOtherError:
+    result.msg = matches[0]
+  trimUnitSep result.msg
+
+proc initResults: TResults =
+  result.total = 0
+  result.passed = 0
+  result.failedButAllowed = 0
+  result.skipped = 0
+  result.data = ""
+
+macro ignoreStyleEcho(args: varargs[typed]): untyped =
+  let typForegroundColor = bindSym"ForegroundColor".getType
+  let typBackgroundColor = bindSym"BackgroundColor".getType
+  let typStyle = bindSym"Style".getType
+  let typTerminalCmd = bindSym"TerminalCmd".getType
+  result = newCall(bindSym"echo")
+  for arg in children(args):
+    if arg.kind == nnkNilLit: continue
+    let typ = arg.getType
+    if typ.kind != nnkEnumTy or
+       typ != typForegroundColor and
+       typ != typBackgroundColor and
+       typ != typStyle and
+       typ != typTerminalCmd:
+      result.add(arg)
+
+template maybeStyledEcho(args: varargs[untyped]): untyped =
+  if useColors:
+    styledEcho(args)
+  else:
+    ignoreStyleEcho(args)
+
+
+proc `$`(x: TResults): string =
+  result = """
+Tests passed or allowed to fail: $2 / $1 <br />
+Tests failed and allowed to fail: $3 / $1 <br />
+Tests skipped: $4 / $1 <br />
+""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped]
+
+proc testName(test: TTest, target: TTarget, extraOptions: string, allowFailure: bool): string =
+  var name = test.name.replace(DirSep, '/')
+  name.add ' ' & $target
+  if allowFailure:
+    name.add " (allowed to fail) "
+  if test.options.len > 0: name.add ' ' & test.options
+  if extraOptions.len > 0: name.add ' ' & extraOptions
+  name.strip()
+
+proc addResult(r: var TResults, test: TTest, target: TTarget,
+               extraOptions, expected, given: string, successOrig: TResultEnum,
+               allowFailure = false, givenSpec: ptr TSpec = nil) =
+  # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need
+  # instead of having to pass individual fields, or abusing existing ones like expected vs given.
+  # test.name is easier to find than test.name.extractFilename
+  # A bit hacky but simple and works with tests/testament/tshould_not_work.nim
+  let name = testName(test, target, extraOptions, allowFailure)
+  let duration = epochTime() - test.startTime
+  let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout
+                else: successOrig
+
+  let durationStr = duration.formatFloat(ffDecimal, precision = 2).align(5)
+  if backendLogging:
+    backend.writeTestResult(name = name,
+                            category = test.cat.string,
+                            target = $target,
+                            action = $test.spec.action,
+                            result = $success,
+                            expected = expected,
+                            given = given)
+  r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
+  template dispNonSkipped(color, outcome) =
+    maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)"
+  template disp(msg) =
+    maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name
+  if success == reSuccess:
+    dispNonSkipped(fgGreen, "PASS: ")
+  elif success == reDisabled:
+    if test.spec.inCurrentBatch: disp("SKIP:")
+    else: disp("NOTINBATCH:")
+  elif success == reJoined: disp("JOINED:")
+  else:
+    dispNonSkipped(fgRed, failString)
+    maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
+    maybeStyledEcho styleBright, fgRed, "Failure: ", $success
+    if givenSpec != nil and givenSpec.debugInfo.len > 0:
+      echo "debugInfo: " & givenSpec.debugInfo
+    if success in {reBuildFailed, reNimcCrash, reInstallFailed}:
+      # expected is empty, no reason to print it.
+      echo given
+    else:
+      maybeStyledEcho fgYellow, "Expected:"
+      maybeStyledEcho styleBright, expected, "\n"
+      maybeStyledEcho fgYellow, "Gotten:"
+      maybeStyledEcho styleBright, given, "\n"
+      echo diffStrings(expected, given).output
+
+  if backendLogging and (isAppVeyor or isAzure):
+    let (outcome, msg) =
+      case success
+      of reSuccess:
+        ("Passed", "")
+      of reDisabled, reJoined:
+        ("Skipped", "")
+      of reBuildFailed, reNimcCrash, reInstallFailed:
+        ("Failed", "Failure: " & $success & '\n' & given)
+      else:
+        ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
+    if isAzure:
+      azure.addTestResult(name, test.cat.string, int(duration * 1000), msg, success)
+    else:
+      var p = startProcess("appveyor", args = ["AddTest", test.name.replace("\\", "/") & test.options,
+                           "-Framework", "nim-testament", "-FileName",
+                           test.cat.string,
+                           "-Outcome", outcome, "-ErrorMessage", msg,
+                           "-Duration", $(duration * 1000).int],
+                           options = {poStdErrToStdOut, poUsePath, poParentStreams})
+      discard waitForExit(p)
+      close(p)
+
+proc toString(inlineError: InlineError, filename: string): string =
+  result.add "$file($line, $col) $kind: $msg" % [
+    "file", filename,
+    "line", $inlineError.line,
+    "col", $inlineError.col,
+    "kind", $inlineError.kind,
+    "msg", $inlineError.msg
+  ]
+
+proc inlineErrorsMsgs(expected: TSpec): string =
+  for inlineError in expected.inlineErrors.items:
+    result.addLine inlineError.toString(expected.filename)
+
+proc checkForInlineErrors(expected, given: TSpec): bool =
+  for inlineError in expected.inlineErrors:
+    if inlineError.toString(expected.filename) notin given.nimout:
+      return false
+  true
+
+proc nimoutCheck(expected, given: TSpec): bool =
+  result = true
+  if expected.nimoutFull:
+    if expected.nimout != given.nimout:
+      result = false
+  elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout):
+    result = false
+
+proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest,
+             target: TTarget, extraOptions: string) =
+  if not checkForInlineErrors(expected, given) or
+    (not expected.nimoutFull and not nimoutCheck(expected, given)):
+      r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer)
+  elif strip(expected.msg) notin strip(given.msg):
+    r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer)
+  elif not nimoutCheck(expected, given):
+    r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer)
+  elif extractFilename(expected.file) != extractFilename(given.file) and
+      "internal error:" notin expected.msg:
+    r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer)
+  elif expected.line != given.line and expected.line != 0 or
+       expected.column != given.column and expected.column != 0:
+    r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column,
+                      $given.line & ':' & $given.column, reLinesDiffer)
+  else:
+    r.addResult(test, target, extraOptions, expected.msg, given.msg, reSuccess)
+    inc(r.passed)
+
+proc generatedFile(test: TTest, target: TTarget): string =
+  if target == targetJS:
+    result = test.name.changeFileExt("js")
+  else:
+    let (_, name, _) = test.name.splitFile
+    let ext = targetToExt[target]
+    result = nimcacheDir(test.name, test.options, target) / "@m" & name.changeFileExt(ext)
+
+proc needsCodegenCheck(spec: TSpec): bool =
+  result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0
+
+proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var string,
+                  given: var TSpec) =
+  try:
+    let genFile = generatedFile(test, target)
+    let contents = readFile(genFile)
+    for check in spec.ccodeCheck:
+      if check.len > 0 and check[0] == '\\':
+        # little hack to get 'match' support:
+        if not contents.match(check.peg):
+          given.err = reCodegenFailure
+      elif contents.find(check.peg) < 0:
+        given.err = reCodegenFailure
+      expectedMsg = check
+    if spec.maxCodeSize > 0 and contents.len > spec.maxCodeSize:
+      given.err = reCodegenFailure
+      given.msg = "generated code size: " & $contents.len
+      expectedMsg = "max allowed size: " & $spec.maxCodeSize
+  except ValueError:
+    given.err = reInvalidPeg
+    echo getCurrentExceptionMsg()
+  except IOError:
+    given.err = reCodeNotFound
+    echo getCurrentExceptionMsg()
+
+proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string,
+                         given: var TSpec, expected: TSpec; r: var TResults) =
+  var expectedmsg: string = ""
+  var givenmsg: string = ""
+  if given.err == reSuccess:
+    if expected.needsCodegenCheck:
+      codegenCheck(test, target, expected, expectedmsg, given)
+      givenmsg = given.msg
+    if not nimoutCheck(expected, given) or
+       not checkForInlineErrors(expected, given):
+      given.err = reMsgsDiffer
+      expectedmsg = expected.nimout & inlineErrorsMsgs(expected)
+      givenmsg = given.nimout.strip
+  else:
+    givenmsg = "$ " & given.cmd & '\n' & given.nimout
+  if given.err == reSuccess: inc(r.passed)
+  r.addResult(test, target, extraOptions, expectedmsg, givenmsg, given.err)
+
+proc getTestSpecTarget(): TTarget =
+  if getEnv("NIM_COMPILE_TO_CPP", "false") == "true":
+    result = targetCpp
+  else:
+    result = targetC
+
+var count = 0
+
+proc equalModuloLastNewline(a, b: string): bool =
+  # allow lazy output spec that omits last newline, but really those should be fixed instead
+  result = a == b or b.endsWith("\n") and a == b[0 ..< ^1]
+
+proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
+                    target: TTarget, extraOptions: string, nimcache: string) =
+  test.startTime = epochTime()
+  if testName(test, target, extraOptions, false) in skips:
+    test.spec.err = reDisabled
+
+  if test.spec.err in {reDisabled, reJoined}:
+    r.addResult(test, target, extraOptions, "", "", test.spec.err)
+    inc(r.skipped)
+    return
+  var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
+  case expected.action
+  of actionCompile:
+    compilerOutputTests(test, target, extraOptions, given, expected, r)
+  of actionRun:
+    if given.err != reSuccess:
+      r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr)
+    else:
+      let isJsTarget = target == targetJS
+      var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt)
+      if not fileExists(exeFile):
+        r.addResult(test, target, extraOptions, expected.output,
+                    "executable not found: " & exeFile, reExeNotFound)
+      else:
+        let nodejs = if isJsTarget: findNodeJs() else: ""
+        if isJsTarget and nodejs == "":
+          r.addResult(test, target, extraOptions, expected.output, "nodejs binary not in PATH",
+                      reExeNotFound)
+        else:
+          var exeCmd: string
+          var args = test.testArgs
+          if isJsTarget:
+            exeCmd = nodejs
+            # see D20210217T215950
+            args = @["--unhandled-rejections=strict", exeFile] & args
+          else:
+            exeCmd = exeFile.dup(normalizeExe)
+            if valgrindEnabled and expected.useValgrind != disabled:
+              var valgrindOptions = @["--error-exitcode=1"]
+              if expected.useValgrind != leaking:
+                valgrindOptions.add "--leak-check=yes"
+              args = valgrindOptions & exeCmd & args
+              exeCmd = "valgrind"
+          var (_, buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input)
+          # Treat all failure codes from nodejs as 1. Older versions of nodejs used
+          # to return other codes, but for us it is sufficient to know that it's not 0.
+          if exitCode != 0: exitCode = 1
+          let bufB =
+            if expected.sortoutput:
+              var buf2 = buf
+              buf2.stripLineEnd
+              var x = splitLines(buf2)
+              sort(x, system.cmp)
+              join(x, "\n") & '\n'
+            else:
+              buf
+          if exitCode != expected.exitCode:
+            given.err = reExitcodesDiffer
+            r.addResult(test, target, extraOptions, "exitcode: " & $expected.exitCode,
+                              "exitcode: " & $exitCode & "\n\nOutput:\n" &
+                              bufB, reExitcodesDiffer)
+          elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or
+              (expected.outputCheck == ocSubstr and expected.output notin bufB):
+            given.err = reOutputsDiffer
+            r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer)
+          compilerOutputTests(test, target, extraOptions, given, expected, r)
+  of actionReject:
+    # Make sure its the compiler rejecting and not the system (e.g. segfault)
+    cmpMsgs(r, expected, given, test, target, extraOptions)
+    if given.exitCode != QuitFailure:
+      r.addResult(test, target, extraOptions, "exitcode: " & $QuitFailure,
+                        "exitcode: " & $given.exitCode & "\n\nOutput:\n" &
+                        given.nimout, reExitcodesDiffer)
+
+
+
+proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget =
+  result = defaultTarget
+  var p = parseopt.initOptParser(extraOptions)
+
+  while true:
+    parseopt.next(p)
+    case p.kind
+    of cmdEnd: break
+    of cmdLongOption, cmdShortOption:
+      if p.key == "b" or p.key == "backend":
+        result = parseEnum[TTarget](p.val.normalize)
+        # chooses the last one
+    else:
+      discard
+
+proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) =
+  for target in expected.targets:
+    inc(r.total)
+    if target notin gTargets:
+      r.addResult(test, target, extraOptions, "", "", reDisabled)
+      inc(r.skipped)
+    elif simulate:
+      inc count
+      echo "testSpec count: ", count, " expected: ", expected
+    else:
+      let nimcache = nimcacheDir(test.name, test.options, target)
+      var testClone = test
+      let target = changeTarget(extraOptions, target)
+      testSpecHelper(r, testClone, expected, target, extraOptions, nimcache)
+
+proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
+  var expected = test.spec
+  if expected.parseErrors.len > 0:
+    # targetC is a lie, but a parameter is required
+    r.addResult(test, targetC, "", "", expected.parseErrors, reInvalidSpec)
+    inc(r.total)
+    return
+
+  expected.targets.incl targets
+  # still no target specified at all
+  if expected.targets == {}:
+    expected.targets = {getTestSpecTarget()}
+  if test.spec.matrix.len > 0:
+    for m in test.spec.matrix:
+      targetHelper(r, test, expected, m)
+  else:
+    targetHelper(r, test, expected, "")
+
+proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} =
+  for target in test.spec.targets:
+    inc(r.total)
+    var testClone = test
+    testSpecHelper(r, testClone, test.spec, target, "", nimcache)
+
+proc makeTest(test, options: string, cat: Category): TTest =
+  result.cat = cat
+  result.name = test
+  result.options = options
+  result.spec = parseSpec(addFileExt(test, ".nim"))
+  result.startTime = epochTime()
+
+proc makeRawTest(test, options: string, cat: Category): TTest {.used.} =
+  result.cat = cat
+  result.name = test
+  result.options = options
+  result.spec = initSpec(addFileExt(test, ".nim"))
+  result.spec.action = actionCompile
+  result.spec.targets = {getTestSpecTarget()}
+  result.startTime = epochTime()
+
+# TODO: fix these files
+const disabledFilesDefault = @[
+  "tableimpl.nim",
+  "setimpl.nim",
+  "hashcommon.nim",
+
+  # Requires compiling with '--threads:on`
+  "sharedlist.nim",
+  "sharedtables.nim",
+
+  # Error: undeclared identifier: 'hasThreadSupport'
+  "ioselectors_epoll.nim",
+  "ioselectors_kqueue.nim",
+  "ioselectors_poll.nim",
+
+  # Error: undeclared identifier: 'Timeval'
+  "ioselectors_select.nim",
+]
+
+when defined(windows):
+  const
+    # array of modules disabled from compilation test of stdlib.
+    disabledFiles = disabledFilesDefault & @["coro.nim"]
+else:
+  const
+    # array of modules disabled from compilation test of stdlib.
+    disabledFiles = disabledFilesDefault
+
+include categories
+
+proc loadSkipFrom(name: string): seq[string] =
+  if name.len == 0: return
+  # One skip per line, comments start with #
+  # used by `nlvm` (at least)
+  for line in lines(name):
+    let sline = line.strip()
+    if sline.len > 0 and not sline.startsWith('#'):
+      result.add sline
+
+proc main() =
+  azure.init()
+  backend.open()
+  var optPrintResults = false
+  var optFailing = false
+  var targetsStr = ""
+  var isMainProcess = true
+  var skipFrom = ""
+
+  var p = initOptParser()
+  p.next()
+  while p.kind in {cmdLongOption, cmdShortOption}:
+    case p.key.normalize
+    of "print": optPrintResults = true
+    of "verbose": optVerbose = true
+    of "failing": optFailing = true
+    of "pedantic": discard # deadcode refs https://github.com/nim-lang/Nim/issues/16731
+    of "targets":
+      targetsStr = p.val
+      gTargets = parseTargets(targetsStr)
+      targetsSet = true
+    of "nim":
+      compilerPrefix = addFileExt(p.val.absolutePath, ExeExt)
+    of "directory":
+      setCurrentDir(p.val)
+    of "colors":
+      case p.val:
+      of "on":
+        useColors = true
+      of "off":
+        useColors = false
+      else:
+        quit Usage
+    of "batch":
+      testamentData0.batchArg = p.val
+      if p.val != "_" and p.val.len > 0 and p.val[0] in {'0'..'9'}:
+        let s = p.val.split("_")
+        doAssert s.len == 2, $(p.val, s)
+        testamentData0.testamentBatch = s[0].parseInt
+        testamentData0.testamentNumBatch = s[1].parseInt
+        doAssert testamentData0.testamentNumBatch > 0
+        doAssert testamentData0.testamentBatch >= 0 and testamentData0.testamentBatch < testamentData0.testamentNumBatch
+    of "simulate":
+      simulate = true
+    of "megatest":
+      case p.val:
+      of "on":
+        useMegatest = true
+      of "off":
+        useMegatest = false
+      else:
+        quit Usage
+    of "valgrind":
+      case p.val:
+      of "on":
+        valgrindEnabled = true
+      of "off":
+        valgrindEnabled = false
+      else:
+        quit Usage
+    of "backendlogging":
+      case p.val:
+      of "on":
+        backendLogging = true
+      of "off":
+        backendLogging = false
+      else:
+        quit Usage
+    of "skipfrom":
+      skipFrom = p.val
+    else:
+      quit Usage
+    p.next()
+  if p.kind != cmdArgument:
+    quit Usage
+  var action = p.key.normalize
+  p.next()
+  var r = initResults()
+  case action
+  of "all":
+    #processCategory(r, Category"megatest", p.cmdLineRest, testsDir, runJoinableTests = false)
+
+    var myself = quoteShell(getAppFilename())
+    if targetsStr.len > 0:
+      myself &= " " & quoteShell("--targets:" & targetsStr)
+
+    myself &= " " & quoteShell("--nim:" & compilerPrefix)
+    if testamentData0.batchArg.len > 0:
+      myself &= " --batch:" & testamentData0.batchArg
+
+    if skipFrom.len > 0:
+      myself &= " " & quoteShell("--skipFrom:" & skipFrom)
+
+    var cats: seq[string]
+    let rest = if p.cmdLineRest.len > 0: " " & p.cmdLineRest else: ""
+    for kind, dir in walkDir(testsDir):
+      assert testsDir.startsWith(testsDir)
+      let cat = dir[testsDir.len .. ^1]
+      if kind == pcDir and cat notin ["testdata", "nimcache"]:
+        cats.add cat
+    if isNimRepoTests():
+      cats.add AdditionalCategories
+    if useMegatest: cats.add MegaTestCat
+
+    var cmds: seq[string]
+    for cat in cats:
+      let runtype = if useMegatest: " pcat " else: " cat "
+      cmds.add(myself & runtype & quoteShell(cat) & rest)
+
+    proc progressStatus(idx: int) =
+      echo "progress[all]: $1/$2 starting: cat: $3" % [$idx, $cats.len, cats[idx]]
+
+    if simulate:
+      skips = loadSkipFrom(skipFrom)
+      for i, cati in cats:
+        progressStatus(i)
+        processCategory(r, Category(cati), p.cmdLineRest, testsDir, runJoinableTests = false)
+    else:
+      addExitProc azure.finalize
+      quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus)
+  of "c", "cat", "category":
+    skips = loadSkipFrom(skipFrom)
+    var cat = Category(p.key)
+    processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = true)
+  of "pcat":
+    skips = loadSkipFrom(skipFrom)
+    # 'pcat' is used for running a category in parallel. Currently the only
+    # difference is that we don't want to run joinable tests here as they
+    # are covered by the 'megatest' category.
+    isMainProcess = false
+    var cat = Category(p.key)
+    p.next
+    processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = false)
+  of "p", "pat", "pattern":
+    skips = loadSkipFrom(skipFrom)
+    let pattern = p.key
+    p.next
+    processPattern(r, pattern, p.cmdLineRest, simulate)
+  of "r", "run":
+    let (cat, path) = splitTestFile(p.key)
+    processSingleTest(r, cat.Category, p.cmdLineRest, path, gTargets, targetsSet)
+  of "html":
+    generateHtml(resultsFile, optFailing)
+  else:
+    quit Usage
+
+  if optPrintResults:
+    if action == "html": openDefaultBrowser(resultsFile)
+    else: echo r, r.data
+  azure.finalize()
+  backend.close()
+  var failed = r.total - r.passed - r.skipped
+  if failed != 0:
+    echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ",
+      r.skipped, " failed: ", failed
+    quit(QuitFailure)
+  if isMainProcess:
+    echo "Used ", compilerPrefix, " to run the tests. Use --nim to override."
+
+if paramCount() == 0:
+  quit Usage
+main()
diff --git a/testament/testament.nim.cfg b/testament/testament.nim.cfg
new file mode 100644
index 000000000..c97284129
--- /dev/null
+++ b/testament/testament.nim.cfg
@@ -0,0 +1,6 @@
+# don't move this file without updating the logic in `isNimRepoTests`
+
+path = "$nim" # For compiler/nodejs
+-d:ssl # For azure
+# my SSL doesn't have this feature and I don't care:
+-d:nimDisableCertificateValidation
diff --git a/testament/testamenthtml.nimf b/testament/testamenthtml.nimf
new file mode 100644
index 000000000..9190f370e
--- /dev/null
+++ b/testament/testamenthtml.nimf
@@ -0,0 +1,297 @@
+#? stdtmpl(subsChar = '%', metaChar = '#', emit = "outfile.write")
+#import strutils
+#
+#proc htmlQuote*(raw: string): string =
+#  result = raw.multiReplace(
+#    ("&", "&amp;"),
+#    ("\"", "&quot;"),
+#    ("'", "&apos;"),
+#    ("<", "&lt;"),
+#    (">", "&gt;")
+#  )
+#
+#end proc
+#proc generateHtmlBegin*(outfile: File) =
+<!DOCTYPE html>
+<html>
+<head>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Testament Test Results</title>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.0/jquery.min.js" integrity="sha256-ihAoc6M/JPfrIiIeayPE9xjin4UWjsx2mjW/rtmxLM4=" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha256-916EbMg70RQy9LHiGkXzG8hSg9EdNy97GazNG/aiY1w=" crossorigin="anonymous" />
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap-theme.min.css" integrity="sha256-ZT4HPpdCOt2lvDkXokHuhJfdOKSPFLzeAJik5U/Q+l4=" crossorigin="anonymous" />
+    <script>
+        /**
+        * Callback function that is executed for each Element in an array.
+        * @callback executeForElement
+        * @param {Element} elem Element to operate on
+        */
+
+        /**
+        *
+        * @param {number} index
+        * @param {Element[]} elemArray
+        * @param {executeForElement} executeOnItem
+        */
+        function executeAllAsync(elemArray, index, executeOnItem) {
+            for (var i = 0; index < elemArray.length && i < 100; i++ , index++) {
+                var item = elemArray[index];
+                executeOnItem(item);
+            }
+            if (index < elemArray.length) {
+                setTimeout(executeAllAsync, 0, elemArray, index, executeOnItem);
+            }
+        }
+
+        /** @param {Element} elem */
+        function executeShowOnElement(elem) {
+            while (elem.classList.contains("hidden")) {
+                elem.classList.remove("hidden");
+            }
+        }
+
+        /** @param {Element} elem */
+        function executeHideOnElement(elem) {
+            if (!elem.classList.contains("hidden")) {
+                elem.classList.add("hidden");
+            }
+        }
+
+        /** @param {Element} elem */
+        function executeExpandOnElement(elem) {
+            $(elem).collapse("show");
+        }
+
+        /** @param {Element} elem */
+        function executeCollapseOnElement(elem) {
+            $(elem).collapse("hide");
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        * @param {executeForElement} executeOnEachPanel
+        */
+        function wholePanelAll(category, executeOnEachPanel) {
+            var selector = "div.panel";
+            if (typeof category === "string" && category) {
+                selector += "-" + category;
+            }
+
+            var jqPanels = $(selector);
+            /** @type {Element[]} */
+            var elemArray = jqPanels.toArray();
+
+            setTimeout(executeAllAsync, 0, elemArray, 0, executeOnEachPanel);
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        * @param {executeForElement} executeOnEachPanel
+        */
+        function panelBodyAll(category, executeOnEachPanelBody) {
+            var selector = "div.panel";
+            if (typeof category === "string" && category) {
+                selector += "-" + category;
+            }
+
+            var jqPanels = $(selector);
+
+            var jqPanelBodies = $("div.panel-body", jqPanels);
+            /** @type {Element[]} */
+            var elemArray = jqPanelBodies.toArray();
+
+            setTimeout(executeAllAsync, 0, elemArray, 0, executeOnEachPanelBody);
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        */
+        function showAll(category) {
+            wholePanelAll(category, executeShowOnElement);
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        */
+        function hideAll(category) {
+            wholePanelAll(category, executeHideOnElement);
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        */
+        function expandAll(category) {
+            panelBodyAll(category, executeExpandOnElement);
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        */
+        function collapseAll(category) {
+            panelBodyAll(category, executeCollapseOnElement);
+        }
+    </script>
+</head>
+<body>
+    <div class="container">
+        <h1>Testament Test Results <small>Nim Tester</small></h1>
+#end proc
+#proc generateHtmlAllTestsBegin*(outfile: File, machine, commit, branch: string,
+#  totalCount: BiggestInt,
+#  successCount: BiggestInt, successPercentage: string,
+#  ignoredCount: BiggestInt, ignoredPercentage: string,
+#  failedCount: BiggestInt, failedPercentage: string, onlyFailing = false) =
+        <dl class="dl-horizontal">
+            <dt>Hostname</dt>
+            <dd>%machine</dd>
+            <dt>Git Commit</dt>
+            <dd><code>%commit</code></dd>
+            <dt title="Git Branch reference">Branch ref.</dt>
+            <dd>%branch</dd>
+        </dl>
+        <dl class="dl-horizontal">
+            <dt>All Tests</dt>
+            <dd>
+                <span class="glyphicon glyphicon-th-list"></span>
+                %totalCount
+            </dd>
+            <dt>Successful Tests</dt>
+            <dd>
+                <span class="glyphicon glyphicon-ok-sign"></span>
+                %successCount (%successPercentage)
+            </dd>
+            <dt>Skipped Tests</dt>
+            <dd>
+                <span class="glyphicon glyphicon-question-sign"></span>
+                %ignoredCount (%ignoredPercentage)
+            </dd>
+            <dt>Failed Tests</dt>
+            <dd>
+                <span class="glyphicon glyphicon-exclamation-sign"></span>
+                %failedCount (%failedPercentage)
+            </dd>
+        </dl>
+        <div class="table-responsive">
+            <table class="table table-condensed">
+#  if not onlyFailing:
+                <tr>
+                    <th class="text-right" style="vertical-align:middle">All Tests</th>
+                    <td>
+                        <div class="btn-group">
+                            <button class="btn btn-default" type="button" onclick="showAll();">Show All</button>
+                            <button class="btn btn-default" type="button" onclick="hideAll();">Hide All</button>
+                            <button class="btn btn-default" type="button" onclick="expandAll();">Expand All</button>
+                            <button class="btn btn-default" type="button" onclick="collapseAll();">Collapse All</button>
+                        </div>
+                    </td>
+                </tr>
+                <tr>
+                    <th class="text-right" style="vertical-align:middle">Successful Tests</th>
+                    <td>
+                        <div class="btn-group">
+                            <button class="btn btn-default" type="button" onclick="showAll('success');">Show All</button>
+                            <button class="btn btn-default" type="button" onclick="hideAll('success');">Hide All</button>
+                            <button class="btn btn-default" type="button" onclick="expandAll('success');">Expand All</button>
+                            <button class="btn btn-default" type="button" onclick="collapseAll('success');">Collapse All</button>
+                        </div>
+                    </td>
+                </tr>
+#  end if
+                <tr>
+                    <th class="text-right" style="vertical-align:middle">Skipped Tests</th>
+                    <td>
+                        <div class="btn-group">
+                            <button class="btn btn-default" type="button" onclick="showAll('info');">Show All</button>
+                            <button class="btn btn-default" type="button" onclick="hideAll('info');">Hide All</button>
+                            <button class="btn btn-default" type="button" onclick="expandAll('info');">Expand All</button>
+                            <button class="btn btn-default" type="button" onclick="collapseAll('info');">Collapse All</button>
+                        </div>
+                    </td>
+                </tr>
+                <tr>
+                    <th class="text-right" style="vertical-align:middle">Failed Tests</th>
+                    <td>
+                        <div class="btn-group">
+                            <button class="btn btn-default" type="button" onclick="showAll('danger');">Show All</button>
+                            <button class="btn btn-default" type="button" onclick="hideAll('danger');">Hide All</button>
+                            <button class="btn btn-default" type="button" onclick="expandAll('danger');">Expand All</button>
+                            <button class="btn btn-default" type="button" onclick="collapseAll('danger');">Collapse All</button>
+                        </div>
+                    </td>
+                </tr>
+            </table>
+        </div>
+        <div class="panel-group">
+#end proc
+#proc generateHtmlTestresultPanelBegin*(outfile: File, trId, name, target, category,
+#  action, resultDescription, timestamp, result, resultSign,
+#  panelCtxClass, textCtxClass, bgCtxClass: string) =
+            <div id="panel-testResult-%trId" class="panel panel-%panelCtxClass">
+                <div class="panel-heading" style="cursor:pointer" data-toggle="collapse" data-target="#panel-body-testResult-%trId" aria-controls="panel-body-testResult-%trId" aria-expanded="false">
+                    <div class="row">
+                        <h4 class="col-xs-3 col-sm-1 panel-title">
+                            <span class="glyphicon glyphicon-%resultSign-sign"></span>
+                            <strong>%resultDescription</strong>
+                        </h4>
+                        <h4 class="col-xs-1 panel-title"><span class="badge">%target</span></h4>
+                        <h4 class="col-xs-5 col-sm-7 panel-title" title="%name"><code class="text-%textCtxClass">%name</code></h4>
+                        <h4 class="col-xs-3 col-sm-3 panel-title text-right"><span class="badge">%category</span></h4>
+                    </div>
+                </div>
+                <div id="panel-body-testResult-%trId" class="panel-body collapse bg-%bgCtxClass">
+                    <dl class="dl-horizontal">
+                        <dt>Name</dt>
+                        <dd><code class="text-%textCtxClass">%name</code></dd>
+                        <dt>Category</dt>
+                        <dd><span class="badge">%category</span></dd>
+                        <dt>Timestamp</dt>
+                        <dd>%timestamp</dd>
+                        <dt>Nim Action</dt>
+                        <dd><code class="text-%textCtxClass">%action</code></dd>
+                        <dt>Nim Backend Target</dt>
+                        <dd><span class="badge">%target</span></dd>
+                        <dt>Code</dt>
+                        <dd><code class="text-%textCtxClass">%result</code></dd>
+                    </dl>
+#end proc
+#proc generateHtmlTestresultOutputDetails*(outfile: File, expected, gotten: string) =
+                    <div class="table-responsive">
+                        <table class="table table-condensed">
+                            <thead>
+                                <tr>
+                                    <th>Expected</th>
+                                    <th>Actual</th>
+                                </tr>
+                            </thead>
+                            <tbody>
+                                <tr>
+                                    <td><pre>%expected</pre></td>
+                                    <td><pre>%gotten</pre></td>
+                                </tr>
+                            </tbody>
+                        </table>
+                    </div>
+#end proc
+#proc generateHtmlTestresultOutputNone*(outfile: File) =
+                    <p class="sr-only">No output details</p>
+#end proc
+#proc generateHtmlTestresultPanelEnd*(outfile: File) =
+                </div>
+            </div>
+#end proc
+#proc generateHtmlAllTestsEnd*(outfile: File) =
+        </div>
+#end proc
+#proc generateHtmlEnd*(outfile: File, timestamp: string) =
+        <hr />
+        <footer>
+            <p>
+                Report generated by: <code>testament</code> &ndash; Nim Tester
+                <br />
+                Made with Nim. Generated on: %timestamp
+            </p>
+        </footer>
+    </div>
+</body>
+</html>
diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim
new file mode 100644
index 000000000..477da1e23
--- /dev/null
+++ b/testament/tests/shouldfail/tccodecheck.nim
@@ -0,0 +1,8 @@
+discard """
+  ccodecheck: "baz"
+"""
+
+proc foo(): void {.exportc: "bar".}=
+  echo "Hello World"
+
+foo()
diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim
new file mode 100644
index 000000000..809ddec74
--- /dev/null
+++ b/testament/tests/shouldfail/tcolumn.nim
@@ -0,0 +1,8 @@
+discard """
+  errormsg: "undeclared identifier: 'undeclared'"
+  line: 9
+  column: 7
+"""
+
+# test should fail because the line directive is wrong
+echo undeclared
diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim
new file mode 100644
index 000000000..6c130d107
--- /dev/null
+++ b/testament/tests/shouldfail/terrormsg.nim
@@ -0,0 +1,8 @@
+discard """
+  errormsg: "wrong error message"
+  line: 9
+  column: 6
+"""
+
+# test should fail because the line directive is wrong
+echo undeclared
diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim
new file mode 100644
index 000000000..605f046db
--- /dev/null
+++ b/testament/tests/shouldfail/texitcode1.nim
@@ -0,0 +1,3 @@
+discard """
+  exitcode: 1
+"""
diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim
new file mode 100644
index 000000000..b40a4f44f
--- /dev/null
+++ b/testament/tests/shouldfail/tfile.nim
@@ -0,0 +1,6 @@
+discard """
+  errormsg: "undeclared identifier: 'undefined'"
+  file: "notthisfile.nim"
+"""
+
+echo undefined
diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim
new file mode 100644
index 000000000..fe782eb03
--- /dev/null
+++ b/testament/tests/shouldfail/tline.nim
@@ -0,0 +1,8 @@
+discard """
+  errormsg: "undeclared identifier: 'undeclared'"
+  line: 10
+  column: 6
+"""
+
+# test should fail because the line directive is wrong
+echo undeclared
diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim
new file mode 100644
index 000000000..92022ee97
--- /dev/null
+++ b/testament/tests/shouldfail/tmaxcodesize.nim
@@ -0,0 +1,5 @@
+discard """
+  maxcodesize: 1
+"""
+
+echo "Hello World"
diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim
new file mode 100644
index 000000000..0a65bfb70
--- /dev/null
+++ b/testament/tests/shouldfail/tnimout.nim
@@ -0,0 +1,7 @@
+discard """
+  nimout: "Hello World!"
+  action: compile
+"""
+
+static:
+  echo "something else"
diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim
new file mode 100644
index 000000000..4fc93f6d2
--- /dev/null
+++ b/testament/tests/shouldfail/tnimoutfull.nim
@@ -0,0 +1,14 @@
+discard """
+  nimout: '''
+msg1
+msg2
+'''
+  action: compile
+  nimoutFull: true
+"""
+
+# should fail because `msg3` is not in nimout and `nimoutFill: true` was given
+static:
+  echo "msg1"
+  echo "msg2"
+  echo "msg3"
diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim
new file mode 100644
index 000000000..eaf9e8652
--- /dev/null
+++ b/testament/tests/shouldfail/toutput.nim
@@ -0,0 +1,7 @@
+discard """
+  output: '''
+  done
+  '''
+"""
+
+echo "broken"
diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim
new file mode 100644
index 000000000..47324ecee
--- /dev/null
+++ b/testament/tests/shouldfail/toutputsub.nim
@@ -0,0 +1,5 @@
+discard """
+  outputsub: "something else"
+"""
+
+echo "Hello World!"
diff --git a/testament/tests/shouldfail/treject.nim b/testament/tests/shouldfail/treject.nim
new file mode 100644
index 000000000..1e7258f70
--- /dev/null
+++ b/testament/tests/shouldfail/treject.nim
@@ -0,0 +1,7 @@
+discard """
+  action: "reject"
+"""
+
+# Because we set action="reject", we expect this line not to compile. But the
+# line does compile, therefore the test fails.
+assert true
diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim
new file mode 100644
index 000000000..69dfbc0a0
--- /dev/null
+++ b/testament/tests/shouldfail/tsortoutput.nim
@@ -0,0 +1,11 @@
+discard """
+  sortoutput: true
+  output: '''
+2
+1
+'''
+"""
+
+# this test should ensure that the output is actually sorted
+echo "2"
+echo "1"
diff --git a/testament/tests/shouldfail/ttimeout.nim b/testament/tests/shouldfail/ttimeout.nim
new file mode 100644
index 000000000..fd3e1a598
--- /dev/null
+++ b/testament/tests/shouldfail/ttimeout.nim
@@ -0,0 +1,7 @@
+discard """
+  timeout: "0.1"
+"""
+
+import os
+
+os.sleep(1000)
diff --git a/testament/tests/shouldfail/tvalgrind.nim b/testament/tests/shouldfail/tvalgrind.nim
new file mode 100644
index 000000000..d551ff12e
--- /dev/null
+++ b/testament/tests/shouldfail/tvalgrind.nim
@@ -0,0 +1,17 @@
+discard """
+  valgrind: true
+  cmd: "nim $target --gc:arc -d:useMalloc $options $file"
+"""
+
+# this is the same check used by testament/specs.nim whether or not valgrind
+# tests are supported
+when defined(linux) and sizeof(int) == 8:
+  # discarding this allocation will cause valgrind to fail (which is what we
+  # want), but valgrind only runs on 64-bit Linux machines...
+  discard alloc(1)
+else:
+  # ...so on all other OS/architectures, simulate any non-zero exit code to
+  # mimic how valgrind would have failed on this test. We cannot use things like
+  # `disabled: "freebsd"` in the Testament configs above or else the tests will
+  # be SKIP-ed rather than FAIL-ed
+  quit(1) # choose 1 to match valgrind's `--error-exit=1`, but could be anything