Testament pre parallel (#9137)

* testament: move to root dir (it's not a test)

* osproc: fix process index passed to afterRunEvent for parallel runs

it was passing the index of the process, not index of all commands

* testament: complete file move

8 files changed, 1979 insertions, 0 deletions

diff --git a/testament/backend.nim b/testament/backend.nim
new file mode 100644
index 000000000..385f1171c
--- /dev/null
+++ b/testament/backend.nim
@@ -0,0 +1,75 @@
+#
+#
+#              The Nim Tester
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    Look at license.txt for more info.
+#    All rights reserved.
+
+import strutils, os, osproc, json
+
+type
+  MachineId* = distinct string
+  CommitId = distinct string
+
+proc `$`*(id: MachineId): string {.borrow.}
+#proc `$`(id: CommitId): string {.borrow.} # not used
+
+var
+  thisMachine: MachineId
+  thisCommit: CommitId
+  thisBranch: string
+
+{.experimental.}
+proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip
+
+proc getMachine*(): MachineId =
+  var name = "hostname"()
+  if name.len == 0:
+    name = when defined(posix): getenv"HOSTNAME".string
+           else: getenv"COMPUTERNAME".string
+  if name.len == 0:
+    quit "cannot determine the machine name"
+
+  result = MachineId(name)
+
+proc getCommit(): CommitId =
+  const commLen = "commit ".len
+  let hash = "git log -n 1"()[commLen..commLen+10]
+  thisBranch = "git symbolic-ref --short HEAD"()
+  if hash.len == 0 or thisBranch.len == 0: quit "cannot determine git HEAD"
+  result = CommitId(hash)
+
+var
+  results: File
+  currentCategory: string
+  entries: int
+
+proc writeTestResult*(name, category, target,
+                      action, result, expected, given: string) =
+  createDir("testresults")
+  if currentCategory != category:
+    if currentCategory.len > 0:
+      results.writeLine("]")
+      close(results)
+    currentCategory = category
+    results = open("testresults" / category.addFileExt"json", fmWrite)
+    results.writeLine("[")
+    entries = 0
+
+  let jentry = %*{"name": name, "category": category, "target": target,
+    "action": action, "result": result, "expected": expected, "given": given,
+    "machine": thisMachine.string, "commit": thisCommit.string, "branch": thisBranch}
+  if entries > 0:
+    results.writeLine(",")
+  results.write($jentry)
+  inc entries
+
+proc open*() =
+  thisMachine = getMachine()
+  thisCommit = getCommit()
+
+proc close*() =
+  if currentCategory.len > 0:
+    results.writeLine("]")
+    close(results)
diff --git a/testament/caasdriver.nim b/testament/caasdriver.nim
new file mode 100644
index 000000000..30383bddb
--- /dev/null
+++ b/testament/caasdriver.nim
@@ -0,0 +1,195 @@
+import osproc, streams, os, strutils, re
+{.experimental.}
+
+## Compiler as a service tester.
+##
+## Please read docs/idetools.txt for information about this.
+
+
+type
+  TRunMode = enum
+    ProcRun, CaasRun, SymbolProcRun
+
+  NimSession* = object
+    nim: Process # Holds the open process for CaasRun sessions, nil otherwise.
+    mode: TRunMode # Stores the type of run mode the session was started with.
+    lastOutput: string # Preserves the last output, needed for ProcRun mode.
+    filename: string # Appended to each command starting with '>'. Also a var.
+    modname: string # Like filename but without extension.
+    nimcache: string # Input script based name for the nimcache dir.
+
+const
+  modes = [CaasRun, ProcRun, SymbolProcRun]
+  filenameReplaceVar = "$TESTNIM"
+  moduleReplaceVar = "$MODULE"
+  silentReplaceVar = "$SILENT"
+  silentReplaceText = "--verbosity:0 --hints:off"
+
+var
+  TesterDir = getAppDir() / ".."
+  NimBin = TesterDir / "../bin/nim"
+
+proc replaceVars(session: var NimSession, text: string): string =
+  result = text.replace(filenameReplaceVar, session.filename)
+  result = result.replace(moduleReplaceVar, session.modname)
+  result = result.replace(silentReplaceVar, silentReplaceText)
+
+proc startNimSession(project, script: string, mode: TRunMode):
+                        NimSession =
+  let (dir, name, ext) = project.splitFile
+  result.mode = mode
+  result.lastOutput = ""
+  result.filename = name & ext
+  result.modname = name
+
+  let (nimcacheDir, nimcacheName, nimcacheExt) = script.splitFile
+  result.nimcache = "SymbolProcRun." & nimcacheName
+
+  if mode == SymbolProcRun:
+    removeDir(nimcacheDir / result.nimcache)
+  else:
+    removeDir(nimcacheDir / "nimcache")
+
+  if mode == CaasRun:
+    result.nim = startProcess(NimBin, workingDir = dir,
+      args = ["serve", "--server.type:stdin", name])
+
+proc doCaasCommand(session: var NimSession, command: string): string =
+  assert session.mode == CaasRun
+  session.nim.inputStream.write(session.replaceVars(command) & "\n")
+  session.nim.inputStream.flush
+
+  result = ""
+
+  while true:
+    var line = TaintedString("")
+    if session.nim.outputStream.readLine(line):
+      if line.string == "": break
+      result.add(line.string & "\n")
+    else:
+      result = "FAILED TO EXECUTE: " & command & "\n" & result
+      break
+
+proc doProcCommand(session: var NimSession, command: string): string =
+  try:
+    assert session.mode == ProcRun or session.mode == SymbolProcRun
+  except:
+    result = "FAILED TO EXECUTE: " & command & "\n" & result
+  var
+    process = startProcess(NimBin, args = session.replaceVars(command).split)
+    stream = outputStream(process)
+    line = TaintedString("")
+
+  result = ""
+  while stream.readLine(line):
+    if result.len > 0: result &= "\n"
+    result &= line.string
+
+  process.close()
+
+proc doCommand(session: var NimSession, command: string) =
+  if session.mode == CaasRun:
+    if not session.nim.running:
+      session.lastOutput = "FAILED TO EXECUTE: " & command & "\n" &
+          "Exit code " & $session.nim.peekExitCode
+      return
+    session.lastOutput = doCaasCommand(session,
+                                       command & " " & session.filename)
+  else:
+    var command = command
+    # For symbol runs we prepend the necessary parameters to avoid clobbering
+    # the normal nimcache.
+    if session.mode == SymbolProcRun:
+      command = "--symbolFiles:on --nimcache:" & session.nimcache &
+                " " & command
+    session.lastOutput = doProcCommand(session,
+                                       command & " " & session.filename)
+
+proc destroy(session: var NimSession) {.destructor.} =
+  if session.mode == CaasRun:
+    session.nim.close
+
+proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): bool =
+  result = true
+
+  var f = open(script)
+  var project = TaintedString("")
+
+  if f.readLine(project):
+    var
+      s = startNimSession(script.parentDir / project.string, script, mode)
+      tline = TaintedString("")
+      ln = 1
+
+    while f.readLine(tline):
+      var line = tline.string
+      inc ln
+
+      # Filter lines by run mode, removing the prefix if the mode is current.
+      for testMode in modes:
+        if line.startsWith($testMode):
+          if testMode != mode:
+            line = ""
+          else:
+            line = line[len($testMode)..len(line) - 1].strip
+          break
+
+      if line.strip.len == 0: continue
+
+      if line.startsWith("#"):
+        output.writeLine line
+        continue
+      elif line.startsWith(">"):
+        s.doCommand(line.substr(1).strip)
+        output.writeLine line, "\n", if verbose: s.lastOutput else: ""
+      else:
+        var expectMatch = true
+        var pattern = s.replaceVars(line)
+        if line.startsWith("!"):
+          pattern = pattern.substr(1).strip
+          expectMatch = false
+
+        let actualMatch =
+          s.lastOutput.find(re(pattern, flags = {reStudy})) != -1
+
+        if expectMatch == actualMatch:
+          output.writeLine "SUCCESS ", line
+        else:
+          output.writeLine "FAILURE ", line
+          result = false
+
+iterator caasTestsRunner*(filter = "", verbose = false): tuple[test,
+                                              output: string, status: bool,
+                                              mode: TRunMode] =
+  for scenario in os.walkFiles(TesterDir / "caas/*.txt"):
+    if filter.len > 0 and find(scenario, filter) == -1: continue
+    for mode in modes:
+      var outStream = newStringStream()
+      let r = doScenario(scenario, outStream, mode, verbose)
+      yield (scenario, outStream.data, r, mode)
+
+when isMainModule:
+  var
+    filter = ""
+    failures = 0
+    verbose = false
+
+  for i in 0..paramCount() - 1:
+    let param = string(paramStr(i + 1))
+    case param
+    of "verbose": verbose = true
+    else: filter = param
+
+  if verbose and len(filter) > 0:
+    echo "Running only test cases matching filter '$1'" % [filter]
+
+  for test, output, result, mode in caasTestsRunner(filter, verbose):
+    if not result or verbose:
+      echo "Mode ", $mode, " (", if result: "succeeded)" else: "failed)"
+      echo test
+      echo output
+      echo "---------\n"
+    if not result:
+      failures += 1
+
+  quit(failures)
diff --git a/testament/categories.nim b/testament/categories.nim
new file mode 100644
index 000000000..59ce70315
--- /dev/null
+++ b/testament/categories.nim
@@ -0,0 +1,539 @@
+#
+#
+#            Nim Tester
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Include for the tester that contains test suites that test special features
+## of the compiler.
+
+# included from tester.nim
+# ---------------- ROD file tests ---------------------------------------------
+
+const
+  rodfilesDir = "tests/rodfiles"
+
+proc delNimCache(filename, options: string) =
+  for target in low(TTarget)..high(TTarget):
+    let dir = nimcacheDir(filename, options, target)
+    try:
+      removeDir(dir)
+    except OSError:
+      echo "[Warning] could not delete: ", dir
+
+proc runRodFiles(r: var TResults, cat: Category, options: string) =
+  template test(filename: string, clearCacheFirst=false) =
+    if clearCacheFirst: delNimCache(filename, options)
+    testSpec r, makeTest(rodfilesDir / filename, options, cat, actionRun)
+
+
+  # test basic recompilation scheme:
+  test "hallo", true
+  test "hallo"
+  when false:
+    # test incremental type information:
+    test "hallo2"
+
+  # test type converters:
+  test "aconv", true
+  test "bconv"
+
+  # test G, A, B example from the documentation; test init sections:
+  test "deada", true
+  test "deada2"
+
+  when false:
+    # test method generation:
+    test "bmethods", true
+    test "bmethods2"
+
+    # test generics:
+    test "tgeneric1", true
+    test "tgeneric2"
+
+proc compileRodFiles(r: var TResults, cat: Category, options: string) =
+  template test(filename: untyped, clearCacheFirst=true) =
+    if clearCacheFirst: delNimCache(filename, options)
+    testSpec r, makeTest(rodfilesDir / filename, options, cat)
+
+  # test DLL interfacing:
+  test "gtkex1", true
+  test "gtkex2"
+
+# --------------------- flags tests -------------------------------------------
+
+proc flagTests(r: var TResults, cat: Category, options: string) =
+  # --genscript
+  const filename = "tests"/"flags"/"tgenscript"
+  const genopts = " --genscript"
+  let nimcache = nimcacheDir(filename, genopts, targetC)
+  testSpec r, makeTest(filename, genopts, cat)
+
+  when defined(windows):
+    testExec r, makeTest(filename, " cmd /c cd " & nimcache &
+                         " && compile_tgenscript.bat", cat)
+
+  elif defined(posix):
+    testExec r, makeTest(filename, " sh -c \"cd " & nimcache &
+                         " && sh compile_tgenscript.sh\"", cat)
+
+  # Run
+  testExec r, makeTest(filename, " " & nimcache / "tgenscript", cat)
+
+# --------------------- DLL generation tests ----------------------------------
+
+proc safeCopyFile(src, dest: string) =
+  try:
+    copyFile(src, dest)
+  except OSError:
+    echo "[Warning] could not copy: ", src, " to ", dest
+
+proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
+  const rpath = when defined(macosx):
+      " --passL:-rpath --passL:@loader_path"
+    else:
+      ""
+
+  testSpec c, makeTest("lib/nimrtl.nim",
+    options & " --app:lib -d:createNimRtl --threads:on", cat)
+  testSpec c, makeTest("tests/dll/server.nim",
+    options & " --app:lib -d:useNimRtl --threads:on" & rpath, cat)
+
+
+  when defined(Windows):
+    # windows looks in the dir of the exe (yay!):
+    var nimrtlDll = DynlibFormat % "nimrtl"
+    safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll)
+  else:
+    # posix relies on crappy LD_LIBRARY_PATH (ugh!):
+    const libpathenv = when defined(haiku):
+                         "LIBRARY_PATH"
+                       else:
+                         "LD_LIBRARY_PATH"
+    var libpath = getEnv(libpathenv).string
+    # Temporarily add the lib directory to LD_LIBRARY_PATH:
+    putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: ""))
+    defer: putEnv(libpathenv, libpath)
+    var nimrtlDll = DynlibFormat % "nimrtl"
+    safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll)
+
+  testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl --threads:on" & rpath,
+                       cat, actionRun)
+
+proc dllTests(r: var TResults, cat: Category, options: string) =
+  # dummy compile result:
+  var c = initResults()
+
+  runBasicDLLTest c, r, cat, options
+  runBasicDLLTest c, r, cat, options & " -d:release"
+  when not defined(windows):
+    # still cannot find a recent Windows version of boehm.dll:
+    runBasicDLLTest c, r, cat, options & " --gc:boehm"
+    runBasicDLLTest c, r, cat, options & " -d:release --gc:boehm"
+
+# ------------------------------ GC tests -------------------------------------
+
+proc gcTests(r: var TResults, cat: Category, options: string) =
+  template testWithNone(filename: untyped) =
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " --gc:none", cat, actionRun)
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " -d:release --gc:none", cat, actionRun)
+
+  template testWithoutMs(filename: untyped) =
+    testSpec r, makeTest("tests/gc" / filename, options, cat, actionRun)
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " -d:release", cat, actionRun)
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " -d:release -d:useRealtimeGC", cat, actionRun)
+
+  template testWithoutBoehm(filename: untyped) =
+    testWithoutMs filename
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " --gc:markAndSweep", cat, actionRun)
+    testSpec r, makeTest("tests/gc" / filename, options &
+                  " -d:release --gc:markAndSweep", cat, actionRun)
+  template test(filename: untyped) =
+    testWithoutBoehm filename
+    when not defined(windows) and not defined(android):
+      # AR: cannot find any boehm.dll on the net, right now, so disabled
+      # for windows:
+      testSpec r, makeTest("tests/gc" / filename, options &
+                    " --gc:boehm", cat, actionRun)
+      testSpec r, makeTest("tests/gc" / filename, options &
+                    " -d:release --gc:boehm", cat, actionRun)
+
+  testWithoutBoehm "foreign_thr"
+  test "gcemscripten"
+  test "growobjcrash"
+  test "gcbench"
+  test "gcleak"
+  test "gcleak2"
+  testWithoutBoehm "gctest"
+  testWithNone "gctest"
+  test "gcleak3"
+  test "gcleak4"
+  # Disabled because it works and takes too long to run:
+  #test "gcleak5"
+  testWithoutBoehm "weakrefs"
+  test "cycleleak"
+  testWithoutBoehm "closureleak"
+  testWithoutMs "refarrayleak"
+
+  testWithoutBoehm "tlists"
+  testWithoutBoehm "thavlak"
+
+  test "stackrefleak"
+  test "cyclecollector"
+
+proc longGCTests(r: var TResults, cat: Category, options: string) =
+  when defined(windows):
+    let cOptions = "-ldl -DWIN"
+  else:
+    let cOptions = "-ldl"
+
+  var c = initResults()
+  # According to ioTests, this should compile the file
+  testNoSpec c, makeTest("tests/realtimeGC/shared", options, cat, actionCompile)
+  testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat, actionRun)
+  testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat, actionRun)
+
+# ------------------------- threading tests -----------------------------------
+
+proc threadTests(r: var TResults, cat: Category, options: string) =
+  template test(filename: untyped) =
+    testSpec r, makeTest(filename, options, cat, actionRun)
+    testSpec r, makeTest(filename, options & " -d:release", cat, actionRun)
+    testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat, actionRun)
+  for t in os.walkFiles("tests/threads/t*.nim"):
+    test(t)
+
+# ------------------------- IO tests ------------------------------------------
+
+proc ioTests(r: var TResults, cat: Category, options: string) =
+  # We need readall_echo to be compiled for this test to run.
+  # dummy compile result:
+  var c = initResults()
+  testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat)
+  testSpec r, makeTest("tests/system/tio", options, cat)
+
+# ------------------------- async tests ---------------------------------------
+proc asyncTests(r: var TResults, cat: Category, options: string) =
+  template test(filename: untyped) =
+    testSpec r, makeTest(filename, options, cat)
+  for t in os.walkFiles("tests/async/t*.nim"):
+    test(t)
+
+# ------------------------- debugger tests ------------------------------------
+
+proc debuggerTests(r: var TResults, cat: Category, options: string) =
+  testNoSpec r, makeTest("tools/nimgrep", options & " --debugger:on", cat)
+
+# ------------------------- JS tests ------------------------------------------
+
+proc jsTests(r: var TResults, cat: Category, options: string) =
+  template test(filename: untyped) =
+    testSpec r, makeTest(filename, options & " -d:nodejs", cat,
+                         actionRun), targetJS
+    testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat,
+                         actionRun), targetJS
+
+  for t in os.walkFiles("tests/js/t*.nim"):
+    test(t)
+  for testfile in ["exception/texceptions", "exception/texcpt1",
+                   "exception/texcsub", "exception/tfinally",
+                   "exception/tfinally2", "exception/tfinally3",
+                   "exception/tunhandledexc",
+                   "actiontable/tactiontable", "method/tmultim1",
+                   "method/tmultim3", "method/tmultim4",
+                   "varres/tvarres0", "varres/tvarres3", "varres/tvarres4",
+                   "varres/tvartup", "misc/tints", "misc/tunsignedinc",
+                   "async/tjsandnativeasync"]:
+    test "tests/" & testfile & ".nim"
+
+  for testfile in ["strutils", "json", "random", "times", "logging"]:
+    test "lib/pure/" & testfile & ".nim"
+
+# ------------------------- nim in action -----------
+
+proc testNimInAction(r: var TResults, cat: Category, options: string) =
+  let options = options & " --nilseqs:on"
+
+  template test(filename: untyped, action: untyped) =
+    testSpec r, makeTest(filename, options, cat, action)
+
+  template testJS(filename: untyped) =
+    testSpec r, makeTest(filename, options, cat, actionCompile), targetJS
+
+  template testCPP(filename: untyped) =
+    testSpec r, makeTest(filename, options, cat, actionCompile), targetCPP
+
+  let tests = [
+    "niminaction/Chapter1/various1",
+    "niminaction/Chapter2/various2",
+    "niminaction/Chapter2/resultaccept",
+    "niminaction/Chapter2/resultreject",
+    "niminaction/Chapter2/explicit_discard",
+    "niminaction/Chapter2/no_def_eq",
+    "niminaction/Chapter2/no_iterator",
+    "niminaction/Chapter2/no_seq_type",
+    "niminaction/Chapter3/ChatApp/src/server",
+    "niminaction/Chapter3/ChatApp/src/client",
+    "niminaction/Chapter3/various3",
+    "niminaction/Chapter6/WikipediaStats/concurrency_regex",
+    "niminaction/Chapter6/WikipediaStats/concurrency",
+    "niminaction/Chapter6/WikipediaStats/naive",
+    "niminaction/Chapter6/WikipediaStats/parallel_counts",
+    "niminaction/Chapter6/WikipediaStats/race_condition",
+    "niminaction/Chapter6/WikipediaStats/sequential_counts",
+    "niminaction/Chapter6/WikipediaStats/unguarded_access",
+    "niminaction/Chapter7/Tweeter/src/tweeter",
+    "niminaction/Chapter7/Tweeter/src/createDatabase",
+    "niminaction/Chapter7/Tweeter/tests/database_test",
+    "niminaction/Chapter8/sdl/sdl_test"
+    ]
+
+  # Verify that the files have not been modified. Death shall fall upon
+  # whoever edits these hashes without dom96's permission, j/k. But please only
+  # edit when making a conscious breaking change, also please try to make your
+  # commit message clear and notify me so I can easily compile an errata later.
+  var testHashes: seq[string] = @[]
+
+  for test in tests:
+    testHashes.add(getMD5(readFile("tests" / test.addFileExt("nim")).string))
+
+  const refHashes = @[
+    "51afdfa84b3ca3d810809d6c4e5037ba", "30f07e4cd5eaec981f67868d4e91cfcf",
+    "d14e7c032de36d219c9548066a97e846", "2e40bfd5daadb268268727da91bb4e81",
+    "c5d3853ed0aba04bf6d35ba28a98dca0", "058603145ff92d46c009006b06e5b228",
+    "7b94a029b94ddb7efafddd546c965ff6", "586d74514394e49f2370dfc01dd9e830",
+    "e1901837b757c9357dc8d259fd0ef0f6", "097670c7ae12e825debaf8ec3995227b",
+    "a8cb7b78cc78d28535ab467361db5d6e", "bfaec2816a1848991b530c1ad17a0184",
+    "47cb71bb4c1198d6d29cdbee05aa10b9", "87e4436809f9d73324cfc4f57f116770",
+    "7b7db5cddc8cf8fa9b6776eef1d0a31d", "e6e40219f0f2b877869b738737b7685e",
+    "6532ee87d819f2605a443d5e94f9422a", "9a8fe78c588d08018843b64b57409a02",
+    "03a801275b8b76b4170c870cd0da079d", "20bb7d3e2d38d43b0cb5fcff4909a4a8",
+    "af6844598f534fab6942abfa4dfe9ab2", "2a7a17f84f6503d9bc89a5ab8feea127"
+  ]
+  doAssert testHashes == refHashes, "Nim in Action tests were changed."
+
+  # Run the tests.
+  for testfile in tests:
+    test "tests/" & testfile & ".nim", actionCompile
+
+  let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim"
+  testJS jsFile
+
+  let cppFile = "tests/niminaction/Chapter8/sfml/sfml_test.nim"
+  testCPP cppFile
+
+
+
+
+# ------------------------- manyloc -------------------------------------------
+#proc runSpecialTests(r: var TResults, options: string) =
+#  for t in ["lib/packages/docutils/highlite"]:
+#    testSpec(r, t, options)
+
+proc findMainFile(dir: string): string =
+  # finds the file belonging to ".nim.cfg"; if there is no such file
+  # it returns the some ".nim" file if there is only one:
+  const cfgExt = ".nim.cfg"
+  result = ""
+  var nimFiles = 0
+  for kind, file in os.walkDir(dir):
+    if kind == pcFile:
+      if file.endsWith(cfgExt): return file[.. ^(cfgExt.len+1)] & ".nim"
+      elif file.endsWith(".nim"):
+        if result.len == 0: result = file
+        inc nimFiles
+  if nimFiles != 1: result.setlen(0)
+
+proc manyLoc(r: var TResults, cat: Category, options: string) =
+  for kind, dir in os.walkDir("tests/manyloc"):
+    if kind == pcDir:
+      when defined(windows):
+        if dir.endsWith"nake": continue
+      if dir.endsWith"named_argument_bug": continue
+      let mainfile = findMainFile(dir)
+      if mainfile != "":
+        testNoSpec r, makeTest(mainfile, options, cat)
+
+proc compileExample(r: var TResults, pattern, options: string, cat: Category) =
+  for test in os.walkFiles(pattern):
+    testNoSpec r, makeTest(test, options, cat)
+
+proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
+  for test in os.walkFiles(pattern):
+    let name = extractFilename(test)
+    if name notin disabledFiles:
+      let contents = readFile(test).string
+      if contents.contains("when isMainModule"):
+        testSpec r, makeTest(test, options, cat, actionRunNoSpec)
+      else:
+        testNoSpec r, makeTest(test, options, cat, actionCompile)
+
+# ----------------------------- nimble ----------------------------------------
+type PackageFilter = enum
+  pfCoreOnly
+  pfExtraOnly
+  pfAll
+
+var nimbleDir = getEnv("NIMBLE_DIR").string
+if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
+let
+  nimbleExe = findExe("nimble")
+  #packageDir = nimbleDir / "pkgs" # not used
+  packageIndex = nimbleDir / "packages.json"
+
+proc waitForExitEx(p: Process): int =
+  var outp = outputStream(p)
+  var line = newStringOfCap(120).TaintedString
+  while true:
+    if outp.readLine(line):
+      discard
+    else:
+      result = peekExitCode(p)
+      if result != -1: break
+  close(p)
+
+proc getPackageDir(package: string): string =
+  ## TODO - Replace this with dom's version comparison magic.
+  var commandOutput = execCmdEx("nimble path $#" % package)
+  if commandOutput.exitCode != QuitSuccess:
+    return ""
+  else:
+    result = commandOutput[0].string
+
+iterator listPackages(filter: PackageFilter): tuple[name, url: string] =
+  let packageList = parseFile(packageIndex)
+
+  for package in packageList.items():
+    let
+      name = package["name"].str
+      url = package["url"].str
+      isCorePackage = "nim-lang" in normalize(url)
+    case filter:
+    of pfCoreOnly:
+      if isCorePackage:
+        yield (name, url)
+    of pfExtraOnly:
+      if not isCorePackage:
+        yield (name, url)
+    of pfAll:
+      yield (name, url)
+
+proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) =
+  if nimbleExe == "":
+    echo("[Warning] - Cannot run nimble tests: Nimble binary not found.")
+    return
+
+  if execCmd("$# update" % nimbleExe) == QuitFailure:
+    echo("[Warning] - Cannot run nimble tests: Nimble update failed.")
+    return
+
+  let packageFileTest = makeTest("PackageFileParsed", "", cat)
+  try:
+    for name, url in listPackages(filter):
+      var test = makeTest(name, "", cat)
+      echo(url)
+      let
+        installProcess = startProcess(nimbleExe, "", ["install", "-y", name])
+        installStatus = waitForExitEx(installProcess)
+      installProcess.close
+      if installStatus != QuitSuccess:
+        r.addResult(test, targetC, "", "", reInstallFailed)
+        continue
+
+      let
+        buildPath = getPackageDir(name).strip
+        buildProcess = startProcess(nimbleExe, buildPath, ["build"])
+        buildStatus = waitForExitEx(buildProcess)
+      buildProcess.close
+      if buildStatus != QuitSuccess:
+        r.addResult(test, targetC, "", "", reBuildFailed)
+      r.addResult(test, targetC, "", "", reSuccess)
+    r.addResult(packageFileTest, targetC, "", "", reSuccess)
+  except JsonParsingError:
+    echo("[Warning] - Cannot run nimble tests: Invalid package file.")
+    r.addResult(packageFileTest, targetC, "", "", reBuildFailed)
+
+
+# ----------------------------------------------------------------------------
+
+const AdditionalCategories = ["debugger", "examples", "lib"]
+
+proc `&.?`(a, b: string): string =
+  # candidate for the stdlib?
+  result = if b.startswith(a): b else: a & b
+
+#proc `&?.`(a, b: string): string = # not used
+  # candidate for the stdlib?
+  #result = if a.endswith(b): a else: a & b
+
+proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
+  let test = "tests" & DirSep &.? cat.string / test
+  let target = if cat.string.normalize == "js": targetJS else: targetC
+
+  if existsFile(test): testSpec r, makeTest(test, options, cat), target
+  else: echo "[Warning] - ", test, " test does not exist"
+
+proc processCategory(r: var TResults, cat: Category, options: string) =
+  case cat.string.normalize
+  of "rodfiles":
+    when false:
+      compileRodFiles(r, cat, options)
+      runRodFiles(r, cat, options)
+  of "js":
+    # only run the JS tests on Windows or Linux because Travis is bad
+    # and other OSes like Haiku might lack nodejs:
+    if not defined(linux) and isTravis:
+      discard
+    else:
+      jsTests(r, cat, options)
+  of "dll":
+    dllTests(r, cat, options)
+  of "flags":
+    flagTests(r, cat, options)
+  of "gc":
+    gcTests(r, cat, options)
+  of "longgc":
+    longGCTests(r, cat, options)
+  of "debugger":
+    debuggerTests(r, cat, options)
+  of "manyloc":
+    manyLoc r, cat, options
+  of "threads":
+    threadTests r, cat, options & " --threads:on"
+  of "io":
+    ioTests r, cat, options
+  of "async":
+    asyncTests r, cat, options
+  of "lib":
+    testStdlib(r, "lib/pure/*.nim", options, cat)
+    testStdlib(r, "lib/packages/docutils/highlite", options, cat)
+  of "examples":
+    compileExample(r, "examples/*.nim", options, cat)
+    compileExample(r, "examples/gtk/*.nim", options, cat)
+    compileExample(r, "examples/talk/*.nim", options, cat)
+  of "nimble-core":
+    testNimblePackages(r, cat, pfCoreOnly)
+  of "nimble-extra":
+    testNimblePackages(r, cat, pfExtraOnly)
+  of "nimble-all":
+    testNimblePackages(r, cat, pfAll)
+  of "niminaction":
+    testNimInAction(r, cat, options)
+  of "untestable":
+    # We can't test it because it depends on a third party.
+    discard # TODO: Move untestable tests to someplace else, i.e. nimble repo.
+  else:
+    var testsRun = 0
+    for name in os.walkFiles("tests" & DirSep &.? cat.string / "t*.nim"):
+      testSpec r, makeTest(name, options, cat)
+      inc testsRun
+    if testsRun == 0:
+      echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run"
diff --git a/testament/htmlgen.nim b/testament/htmlgen.nim
new file mode 100644
index 000000000..4a10fe00c
--- /dev/null
+++ b/testament/htmlgen.nim
@@ -0,0 +1,148 @@
+#
+#
+#            Nim Tester
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## HTML generator for the tester.
+
+import cgi, backend, strutils, json, os, tables, times
+
+import "testamenthtml.templ"
+
+proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) =
+  let
+    trId = htmlQuote(testResultRow["category"].str & "_" & testResultRow["name"].str).
+        multiReplace({".": "_", " ": "_", ":": "_"})
+    name = testResultRow["name"].str.htmlQuote()
+    category = testResultRow["category"].str.htmlQuote()
+    target = testResultRow["target"].str.htmlQuote()
+    action = testResultRow["action"].str.htmlQuote()
+    result = htmlQuote testResultRow["result"].str
+    expected = testResultRow["expected"].getStr
+    gotten = testResultRow["given"].getStr
+    timestamp = "unknown"
+  var
+    panelCtxClass, textCtxClass, bgCtxClass: string
+    resultSign, resultDescription: string
+  case result
+  of "reSuccess":
+    panelCtxClass = "success"
+    textCtxClass = "success"
+    bgCtxClass = "success"
+    resultSign = "ok"
+    resultDescription = "PASS"
+  of "reIgnored":
+    panelCtxClass = "info"
+    textCtxClass = "info"
+    bgCtxClass = "info"
+    resultSign = "question"
+    resultDescription = "SKIP"
+  else:
+    panelCtxClass = "danger"
+    textCtxClass = "danger"
+    bgCtxClass = "danger"
+    resultSign = "exclamation"
+    resultDescription = "FAIL"
+
+  outfile.generateHtmlTestresultPanelBegin(
+    trId, name, target, category, action, resultDescription,
+    timestamp, result, resultSign, panelCtxClass, textCtxClass, bgCtxClass
+  )
+  if expected.isNilOrWhitespace() and gotten.isNilOrWhitespace():
+    outfile.generateHtmlTestresultOutputNone()
+  else:
+    outfile.generateHtmlTestresultOutputDetails(
+      expected.strip().htmlQuote,
+      gotten.strip().htmlQuote
+    )
+  outfile.generateHtmlTestresultPanelEnd()
+
+type
+  AllTests = object
+    data: JSonNode
+    totalCount, successCount, ignoredCount, failedCount: int
+    successPercentage, ignoredPercentage, failedPercentage: BiggestFloat
+
+proc allTestResults(onlyFailing = false): AllTests =
+  result.data = newJArray()
+  for file in os.walkFiles("testresults/*.json"):
+    let data = parseFile(file)
+    if data.kind != JArray:
+      echo "[ERROR] ignoring json file that is not an array: ", file
+    else:
+      for elem in data:
+        let state = elem["result"].str
+        inc result.totalCount
+        if state.contains("reSuccess"): inc result.successCount
+        elif state.contains("reIgnored"): inc result.ignoredCount
+        if not onlyFailing or not(state.contains("reSuccess")):
+          result.data.add elem
+  result.successPercentage = 100 *
+    (result.successCount.toBiggestFloat / result.totalCount.toBiggestFloat)
+  result.ignoredPercentage = 100 *
+    (result.ignoredCount.toBiggestFloat / result.totalCount.toBiggestFloat)
+  result.failedCount = result.totalCount -
+    result.successCount - result.ignoredCount
+  result.failedPercentage = 100 *
+    (result.failedCount.toBiggestFloat / result.totalCount.toBiggestFloat)
+
+proc generateTestResultsPanelGroupPartial(outfile: File, allResults: JsonNode) =
+  for testresultRow in allResults:
+    generateTestResultPanelPartial(outfile, testresultRow)
+
+proc generateAllTestsContent(outfile: File, allResults: AllTests,
+  onlyFailing = false) =
+  if allResults.data.len < 1: return # Nothing to do if there is no data.
+  # Only results from one test run means that test run environment info is the
+  # same for all tests
+  let
+    firstRow = allResults.data[0]
+    commit = htmlQuote firstRow["commit"].str
+    branch = htmlQuote firstRow["branch"].str
+    machine = htmlQuote firstRow["machine"].str
+
+  outfile.generateHtmlAllTestsBegin(
+    machine, commit, branch,
+    allResults.totalCount,
+    allResults.successCount,
+    formatBiggestFloat(allResults.successPercentage, ffDecimal, 2) & "%",
+    allResults.ignoredCount,
+    formatBiggestFloat(allResults.ignoredPercentage, ffDecimal, 2) & "%",
+    allResults.failedCount,
+    formatBiggestFloat(allResults.failedPercentage, ffDecimal, 2) & "%",
+    onlyFailing
+  )
+  generateTestResultsPanelGroupPartial(outfile, allResults.data)
+  outfile.generateHtmlAllTestsEnd()
+
+proc generateHtml*(filename: string, onlyFailing: bool) =
+  let
+    currentTime = getTime().local()
+    timestring = htmlQuote format(currentTime, "yyyy-MM-dd HH:mm:ss 'UTC'zzz")
+  var outfile = open(filename, fmWrite)
+
+  outfile.generateHtmlBegin()
+
+  generateAllTestsContent(outfile, allTestResults(onlyFailing), onlyFailing)
+
+  outfile.generateHtmlEnd(timestring)
+
+  outfile.flushFile()
+  close(outfile)
+
+proc dumpJsonTestResults*(prettyPrint, onlyFailing: bool) =
+  var
+    outfile = stdout
+    jsonString: string
+
+  let results = allTestResults(onlyFailing)
+  if prettyPrint:
+    jsonString = results.data.pretty()
+  else:
+    jsonString = $ results.data
+
+  outfile.writeLine(jsonString)
diff --git a/testament/specs.nim b/testament/specs.nim
new file mode 100644
index 000000000..c51a3343e
--- /dev/null
+++ b/testament/specs.nim
@@ -0,0 +1,202 @@
+#
+#
+#            Nim Tester
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import parseutils, strutils, os, osproc, streams, parsecfg
+
+
+var compilerPrefix* = "compiler" / "nim "
+
+let isTravis* = existsEnv("TRAVIS")
+let isAppVeyor* = existsEnv("APPVEYOR")
+
+proc cmdTemplate*(): string =
+  compilerPrefix & "$target --lib:lib --hints:on -d:testing --nimblePath:tests/deps $options $file"
+
+type
+  TTestAction* = enum
+    actionCompile = "compile"
+    actionRun = "run"
+    actionReject = "reject"
+    actionRunNoSpec = "runNoSpec"
+  TResultEnum* = enum
+    reNimcCrash,     # nim compiler seems to have crashed
+    reMsgsDiffer,       # error messages differ
+    reFilesDiffer,      # expected and given filenames differ
+    reLinesDiffer,      # expected and given line numbers differ
+    reOutputsDiffer,
+    reExitcodesDiffer,
+    reInvalidPeg,
+    reCodegenFailure,
+    reCodeNotFound,
+    reExeNotFound,
+    reInstallFailed     # package installation failed
+    reBuildFailed       # package building failed
+    reIgnored,          # test is ignored
+    reSuccess           # test was successful
+  TTarget* = enum
+    targetC = "C"
+    targetCpp = "C++"
+    targetObjC = "ObjC"
+    targetJS = "JS"
+
+  TSpec* = object
+    action*: TTestAction
+    file*, cmd*: string
+    outp*: string
+    line*, column*: int
+    tfile*: string
+    tline*, tcolumn*: int
+    exitCode*: int
+    msg*: string
+    ccodeCheck*: string
+    maxCodeSize*: int
+    err*: TResultEnum
+    substr*, sortoutput*: bool
+    targets*: set[TTarget]
+    nimout*: string
+
+const
+  targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"]
+  targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"]
+
+when not declared(parseCfgBool):
+  # candidate for the stdlib:
+  proc parseCfgBool(s: string): bool =
+    case normalize(s)
+    of "y", "yes", "true", "1", "on": result = true
+    of "n", "no", "false", "0", "off": result = false
+    else: raise newException(ValueError, "cannot interpret as a bool: " & s)
+
+proc extractSpec(filename: string): string =
+  const tripleQuote = "\"\"\""
+  var x = readFile(filename).string
+  var a = x.find(tripleQuote)
+  var b = x.find(tripleQuote, a+3)
+  # look for """ only in the first section
+  if a >= 0 and b > a and a < 40:
+    result = x.substr(a+3, b-1).replace("'''", tripleQuote)
+  else:
+    #echo "warning: file does not contain spec: " & filename
+    result = ""
+
+when not defined(nimhygiene):
+  {.pragma: inject.}
+
+template parseSpecAux(fillResult: untyped) =
+  var ss = newStringStream(extractSpec(filename))
+  var p {.inject.}: CfgParser
+  open(p, ss, filename, 1)
+  while true:
+    var e {.inject.} = next(p)
+    case e.kind
+    of cfgEof: break
+    of cfgSectionStart, cfgOption, cfgError:
+      echo ignoreMsg(p, e)
+    of cfgKeyValuePair:
+      fillResult
+  close(p)
+
+proc specDefaults*(result: var TSpec) =
+  result.msg = ""
+  result.outp = ""
+  result.nimout = ""
+  result.ccodeCheck = ""
+  result.cmd = cmdTemplate()
+  result.line = 0
+  result.column = 0
+  result.tfile = ""
+  result.tline = 0
+  result.tcolumn = 0
+  result.maxCodeSize = 0
+
+proc parseTargets*(value: string): set[TTarget] =
+  for v in value.normalize.splitWhitespace:
+    case v
+    of "c": result.incl(targetC)
+    of "cpp", "c++": result.incl(targetCpp)
+    of "objc": result.incl(targetObjC)
+    of "js": result.incl(targetJS)
+    else: echo "target ignored: " & v
+
+proc parseSpec*(filename: string): TSpec =
+  specDefaults(result)
+  result.file = filename
+  parseSpecAux:
+    case normalize(e.key)
+    of "action":
+      case e.value.normalize
+      of "compile": result.action = actionCompile
+      of "run": result.action = actionRun
+      of "reject": result.action = actionReject
+      else: echo ignoreMsg(p, e)
+    of "file": result.file = e.value
+    of "line": discard parseInt(e.value, result.line)
+    of "column": discard parseInt(e.value, result.column)
+    of "tfile": result.tfile = e.value
+    of "tline": discard parseInt(e.value, result.tline)
+    of "tcolumn": discard parseInt(e.value, result.tcolumn)
+    of "output":
+      result.action = actionRun
+      result.outp = e.value
+    of "outputsub":
+      result.action = actionRun
+      result.outp = e.value
+      result.substr = true
+    of "sortoutput":
+      result.sortoutput = parseCfgBool(e.value)
+    of "exitcode":
+      discard parseInt(e.value, result.exitCode)
+      result.action = actionRun
+    of "msg":
+      result.msg = e.value
+      if result.action != actionRun:
+        result.action = actionCompile
+    of "errormsg", "errmsg":
+      result.msg = e.value
+      result.action = actionReject
+    of "nimout":
+      result.nimout = e.value
+    of "disabled":
+      case e.value.normalize
+      of "y", "yes", "true", "1", "on": result.err = reIgnored
+      of "n", "no", "false", "0", "off": discard
+      of "win", "windows":
+        when defined(windows): result.err = reIgnored
+      of "linux":
+        when defined(linux): result.err = reIgnored
+      of "bsd":
+        when defined(bsd): result.err = reIgnored
+      of "macosx":
+        when defined(macosx): result.err = reIgnored
+      of "unix":
+        when defined(unix): result.err = reIgnored
+      of "posix":
+        when defined(posix): result.err = reIgnored
+      of "travis":
+        if isTravis: result.err = reIgnored
+      of "appveyor":
+        if isAppVeyor: result.err = reIgnored
+      else:
+        raise newException(ValueError, "cannot interpret as a bool: " & e.value)
+    of "cmd":
+      if e.value.startsWith("nim "):
+        result.cmd = compilerPrefix & e.value[4..^1]
+      else:
+        result.cmd = e.value
+    of "ccodecheck": result.ccodeCheck = e.value
+    of "maxcodesize": discard parseInt(e.value, result.maxCodeSize)
+    of "target", "targets":
+      for v in e.value.normalize.splitWhitespace:
+        case v
+        of "c": result.targets.incl(targetC)
+        of "cpp", "c++": result.targets.incl(targetCpp)
+        of "objc": result.targets.incl(targetObjC)
+        of "js": result.targets.incl(targetJS)
+        else: echo ignoreMsg(p, e)
+    else: echo ignoreMsg(p, e)
diff --git a/testament/testamenthtml.templ b/testament/testamenthtml.templ
new file mode 100644
index 000000000..9190f370e
--- /dev/null
+++ b/testament/testamenthtml.templ
@@ -0,0 +1,297 @@
+#? stdtmpl(subsChar = '%', metaChar = '#', emit = "outfile.write")
+#import strutils
+#
+#proc htmlQuote*(raw: string): string =
+#  result = raw.multiReplace(
+#    ("&", "&amp;"),
+#    ("\"", "&quot;"),
+#    ("'", "&apos;"),
+#    ("<", "&lt;"),
+#    (">", "&gt;")
+#  )
+#
+#end proc
+#proc generateHtmlBegin*(outfile: File) =
+<!DOCTYPE html>
+<html>
+<head>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Testament Test Results</title>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.0/jquery.min.js" integrity="sha256-ihAoc6M/JPfrIiIeayPE9xjin4UWjsx2mjW/rtmxLM4=" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha256-916EbMg70RQy9LHiGkXzG8hSg9EdNy97GazNG/aiY1w=" crossorigin="anonymous" />
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap-theme.min.css" integrity="sha256-ZT4HPpdCOt2lvDkXokHuhJfdOKSPFLzeAJik5U/Q+l4=" crossorigin="anonymous" />
+    <script>
+        /**
+        * Callback function that is executed for each Element in an array.
+        * @callback executeForElement
+        * @param {Element} elem Element to operate on
+        */
+
+        /**
+        *
+        * @param {number} index
+        * @param {Element[]} elemArray
+        * @param {executeForElement} executeOnItem
+        */
+        function executeAllAsync(elemArray, index, executeOnItem) {
+            for (var i = 0; index < elemArray.length && i < 100; i++ , index++) {
+                var item = elemArray[index];
+                executeOnItem(item);
+            }
+            if (index < elemArray.length) {
+                setTimeout(executeAllAsync, 0, elemArray, index, executeOnItem);
+            }
+        }
+
+        /** @param {Element} elem */
+        function executeShowOnElement(elem) {
+            while (elem.classList.contains("hidden")) {
+                elem.classList.remove("hidden");
+            }
+        }
+
+        /** @param {Element} elem */
+        function executeHideOnElement(elem) {
+            if (!elem.classList.contains("hidden")) {
+                elem.classList.add("hidden");
+            }
+        }
+
+        /** @param {Element} elem */
+        function executeExpandOnElement(elem) {
+            $(elem).collapse("show");
+        }
+
+        /** @param {Element} elem */
+        function executeCollapseOnElement(elem) {
+            $(elem).collapse("hide");
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        * @param {executeForElement} executeOnEachPanel
+        */
+        function wholePanelAll(category, executeOnEachPanel) {
+            var selector = "div.panel";
+            if (typeof category === "string" && category) {
+                selector += "-" + category;
+            }
+
+            var jqPanels = $(selector);
+            /** @type {Element[]} */
+            var elemArray = jqPanels.toArray();
+
+            setTimeout(executeAllAsync, 0, elemArray, 0, executeOnEachPanel);
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        * @param {executeForElement} executeOnEachPanel
+        */
+        function panelBodyAll(category, executeOnEachPanelBody) {
+            var selector = "div.panel";
+            if (typeof category === "string" && category) {
+                selector += "-" + category;
+            }
+
+            var jqPanels = $(selector);
+
+            var jqPanelBodies = $("div.panel-body", jqPanels);
+            /** @type {Element[]} */
+            var elemArray = jqPanelBodies.toArray();
+
+            setTimeout(executeAllAsync, 0, elemArray, 0, executeOnEachPanelBody);
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        */
+        function showAll(category) {
+            wholePanelAll(category, executeShowOnElement);
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        */
+        function hideAll(category) {
+            wholePanelAll(category, executeHideOnElement);
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        */
+        function expandAll(category) {
+            panelBodyAll(category, executeExpandOnElement);
+        }
+
+        /**
+        * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success)
+        */
+        function collapseAll(category) {
+            panelBodyAll(category, executeCollapseOnElement);
+        }
+    </script>
+</head>
+<body>
+    <div class="container">
+        <h1>Testament Test Results <small>Nim Tester</small></h1>
+#end proc
+#proc generateHtmlAllTestsBegin*(outfile: File, machine, commit, branch: string,
+#  totalCount: BiggestInt,
+#  successCount: BiggestInt, successPercentage: string,
+#  ignoredCount: BiggestInt, ignoredPercentage: string,
+#  failedCount: BiggestInt, failedPercentage: string, onlyFailing = false) =
+        <dl class="dl-horizontal">
+            <dt>Hostname</dt>
+            <dd>%machine</dd>
+            <dt>Git Commit</dt>
+            <dd><code>%commit</code></dd>
+            <dt title="Git Branch reference">Branch ref.</dt>
+            <dd>%branch</dd>
+        </dl>
+        <dl class="dl-horizontal">
+            <dt>All Tests</dt>
+            <dd>
+                <span class="glyphicon glyphicon-th-list"></span>
+                %totalCount
+            </dd>
+            <dt>Successful Tests</dt>
+            <dd>
+                <span class="glyphicon glyphicon-ok-sign"></span>
+                %successCount (%successPercentage)
+            </dd>
+            <dt>Skipped Tests</dt>
+            <dd>
+                <span class="glyphicon glyphicon-question-sign"></span>
+                %ignoredCount (%ignoredPercentage)
+            </dd>
+            <dt>Failed Tests</dt>
+            <dd>
+                <span class="glyphicon glyphicon-exclamation-sign"></span>
+                %failedCount (%failedPercentage)
+            </dd>
+        </dl>
+        <div class="table-responsive">
+            <table class="table table-condensed">
+#  if not onlyFailing:
+                <tr>
+                    <th class="text-right" style="vertical-align:middle">All Tests</th>
+                    <td>
+                        <div class="btn-group">
+                            <button class="btn btn-default" type="button" onclick="showAll();">Show All</button>
+                            <button class="btn btn-default" type="button" onclick="hideAll();">Hide All</button>
+                            <button class="btn btn-default" type="button" onclick="expandAll();">Expand All</button>
+                            <button class="btn btn-default" type="button" onclick="collapseAll();">Collapse All</button>
+                        </div>
+                    </td>
+                </tr>
+                <tr>
+                    <th class="text-right" style="vertical-align:middle">Successful Tests</th>
+                    <td>
+                        <div class="btn-group">
+                            <button class="btn btn-default" type="button" onclick="showAll('success');">Show All</button>
+                            <button class="btn btn-default" type="button" onclick="hideAll('success');">Hide All</button>
+                            <button class="btn btn-default" type="button" onclick="expandAll('success');">Expand All</button>
+                            <button class="btn btn-default" type="button" onclick="collapseAll('success');">Collapse All</button>
+                        </div>
+                    </td>
+                </tr>
+#  end if
+                <tr>
+                    <th class="text-right" style="vertical-align:middle">Skipped Tests</th>
+                    <td>
+                        <div class="btn-group">
+                            <button class="btn btn-default" type="button" onclick="showAll('info');">Show All</button>
+                            <button class="btn btn-default" type="button" onclick="hideAll('info');">Hide All</button>
+                            <button class="btn btn-default" type="button" onclick="expandAll('info');">Expand All</button>
+                            <button class="btn btn-default" type="button" onclick="collapseAll('info');">Collapse All</button>
+                        </div>
+                    </td>
+                </tr>
+                <tr>
+                    <th class="text-right" style="vertical-align:middle">Failed Tests</th>
+                    <td>
+                        <div class="btn-group">
+                            <button class="btn btn-default" type="button" onclick="showAll('danger');">Show All</button>
+                            <button class="btn btn-default" type="button" onclick="hideAll('danger');">Hide All</button>
+                            <button class="btn btn-default" type="button" onclick="expandAll('danger');">Expand All</button>
+                            <button class="btn btn-default" type="button" onclick="collapseAll('danger');">Collapse All</button>
+                        </div>
+                    </td>
+                </tr>
+            </table>
+        </div>
+        <div class="panel-group">
+#end proc
+#proc generateHtmlTestresultPanelBegin*(outfile: File, trId, name, target, category,
+#  action, resultDescription, timestamp, result, resultSign,
+#  panelCtxClass, textCtxClass, bgCtxClass: string) =
+            <div id="panel-testResult-%trId" class="panel panel-%panelCtxClass">
+                <div class="panel-heading" style="cursor:pointer" data-toggle="collapse" data-target="#panel-body-testResult-%trId" aria-controls="panel-body-testResult-%trId" aria-expanded="false">
+                    <div class="row">
+                        <h4 class="col-xs-3 col-sm-1 panel-title">
+                            <span class="glyphicon glyphicon-%resultSign-sign"></span>
+                            <strong>%resultDescription</strong>
+                        </h4>
+                        <h4 class="col-xs-1 panel-title"><span class="badge">%target</span></h4>
+                        <h4 class="col-xs-5 col-sm-7 panel-title" title="%name"><code class="text-%textCtxClass">%name</code></h4>
+                        <h4 class="col-xs-3 col-sm-3 panel-title text-right"><span class="badge">%category</span></h4>
+                    </div>
+                </div>
+                <div id="panel-body-testResult-%trId" class="panel-body collapse bg-%bgCtxClass">
+                    <dl class="dl-horizontal">
+                        <dt>Name</dt>
+                        <dd><code class="text-%textCtxClass">%name</code></dd>
+                        <dt>Category</dt>
+                        <dd><span class="badge">%category</span></dd>
+                        <dt>Timestamp</dt>
+                        <dd>%timestamp</dd>
+                        <dt>Nim Action</dt>
+                        <dd><code class="text-%textCtxClass">%action</code></dd>
+                        <dt>Nim Backend Target</dt>
+                        <dd><span class="badge">%target</span></dd>
+                        <dt>Code</dt>
+                        <dd><code class="text-%textCtxClass">%result</code></dd>
+                    </dl>
+#end proc
+#proc generateHtmlTestresultOutputDetails*(outfile: File, expected, gotten: string) =
+                    <div class="table-responsive">
+                        <table class="table table-condensed">
+                            <thead>
+                                <tr>
+                                    <th>Expected</th>
+                                    <th>Actual</th>
+                                </tr>
+                            </thead>
+                            <tbody>
+                                <tr>
+                                    <td><pre>%expected</pre></td>
+                                    <td><pre>%gotten</pre></td>
+                                </tr>
+                            </tbody>
+                        </table>
+                    </div>
+#end proc
+#proc generateHtmlTestresultOutputNone*(outfile: File) =
+                    <p class="sr-only">No output details</p>
+#end proc
+#proc generateHtmlTestresultPanelEnd*(outfile: File) =
+                </div>
+            </div>
+#end proc
+#proc generateHtmlAllTestsEnd*(outfile: File) =
+        </div>
+#end proc
+#proc generateHtmlEnd*(outfile: File, timestamp: string) =
+        <hr />
+        <footer>
+            <p>
+                Report generated by: <code>testament</code> &ndash; Nim Tester
+                <br />
+                Made with Nim. Generated on: %timestamp
+            </p>
+        </footer>
+    </div>
+</body>
+</html>
diff --git a/testament/tester.nim b/testament/tester.nim
new file mode 100644
index 000000000..024d02ed4
--- /dev/null
+++ b/testament/tester.nim
@@ -0,0 +1,522 @@
+#
+#
+#            Nim Tester
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This program verifies Nim against the testcases.
+
+import
+  parseutils, strutils, pegs, os, osproc, streams, parsecfg, json,
+  marshal, backend, parseopt, specs, htmlgen, browsers, terminal,
+  algorithm, compiler/nodejs, times, sets, md5
+
+const
+  resultsFile = "testresults.html"
+  #jsonFile = "testresults.json" # not used
+  Usage = """Usage:
+  tester [options] command [arguments]
+
+Command:
+  all                         run all tests
+  c|cat|category <category>   run all the tests of a certain category
+  r|run <test>                run single test file
+  html                        generate $1 from the database
+Arguments:
+  arguments are passed to the compiler
+Options:
+  --print                   also print results to the console
+  --failing                 only show failing/ignored tests
+  --targets:"c c++ js objc" run tests for specified targets (default: all)
+  --nim:path                use a particular nim executable (default: compiler/nim)
+""" % resultsFile
+
+type
+  Category = distinct string
+  TResults = object
+    total, passed, skipped: int
+    data: string
+
+  TTest = object
+    name: string
+    cat: Category
+    options: string
+    action: TTestAction
+    startTime: float
+
+# ----------------------------------------------------------------------------
+
+let
+  pegLineError =
+    peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}"
+  pegLineTemplate =
+    peg"""
+      {[^(]*} '(' {\d+} ', ' {\d+} ') '
+      'template/generic instantiation' ( ' of `' [^`]+ '`' )? ' from here' .*
+    """
+  pegOtherError = peg"'Error:' \s* {.*}"
+  pegSuccess = peg"'Hint: operation successful'.*"
+  pegOfInterest = pegLineError / pegOtherError
+
+var targets = {low(TTarget)..high(TTarget)}
+
+proc normalizeMsg(s: string): string =
+  result = newStringOfCap(s.len+1)
+  for x in splitLines(s):
+    if result.len > 0: result.add '\L'
+    result.add x.strip
+
+proc getFileDir(filename: string): string =
+  result = filename.splitFile().dir
+  if not result.isAbsolute():
+    result = getCurrentDir() / result
+
+proc nimcacheDir(filename, options: string, target: TTarget): string =
+  ## Give each test a private nimcache dir so they don't clobber each other's.
+  let hashInput = options & $target
+  return "nimcache" / (filename & '_' & hashInput.getMD5)
+
+proc callCompiler(cmdTemplate, filename, options: string,
+                  target: TTarget, extraOptions=""): TSpec =
+  let nimcache = nimcacheDir(filename, options, target)
+  let options = options & " " & ("--nimCache:" & nimcache).quoteShell & extraOptions
+  let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
+                       "options", options, "file", filename.quoteShell,
+                       "filedir", filename.getFileDir()])
+  var p = startProcess(command=c[0], args=c[1.. ^1],
+                       options={poStdErrToStdOut, poUsePath})
+  let outp = p.outputStream
+  var suc = ""
+  var err = ""
+  var tmpl = ""
+  var x = newStringOfCap(120)
+  result.nimout = ""
+  while outp.readLine(x.TaintedString) or running(p):
+    result.nimout.add(x & "\n")
+    if x =~ pegOfInterest:
+      # `err` should contain the last error/warning message
+      err = x
+    elif x =~ pegLineTemplate and err == "":
+      # `tmpl` contains the last template expansion before the error
+      tmpl = x
+    elif x =~ pegSuccess:
+      suc = x
+  close(p)
+  result.msg = ""
+  result.file = ""
+  result.outp = ""
+  result.line = 0
+  result.column = 0
+  result.tfile = ""
+  result.tline = 0
+  result.tcolumn = 0
+  if tmpl =~ pegLineTemplate:
+    result.tfile = extractFilename(matches[0])
+    result.tline = parseInt(matches[1])
+    result.tcolumn = parseInt(matches[2])
+  if err =~ pegLineError:
+    result.file = extractFilename(matches[0])
+    result.line = parseInt(matches[1])
+    result.column = parseInt(matches[2])
+    result.msg = matches[3]
+  elif err =~ pegOtherError:
+    result.msg = matches[0]
+  elif suc =~ pegSuccess:
+    result.err = reSuccess
+
+proc callCCompiler(cmdTemplate, filename, options: string,
+                  target: TTarget): TSpec =
+  let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
+                       "options", options, "file", filename.quoteShell,
+                       "filedir", filename.getFileDir()])
+  var p = startProcess(command="gcc", args=c[5 .. ^1],
+                       options={poStdErrToStdOut, poUsePath})
+  let outp = p.outputStream
+  var x = newStringOfCap(120)
+  result.nimout = ""
+  result.msg = ""
+  result.file = ""
+  result.outp = ""
+  result.line = -1
+  while outp.readLine(x.TaintedString) or running(p):
+    result.nimout.add(x & "\n")
+  close(p)
+  if p.peekExitCode == 0:
+    result.err = reSuccess
+
+proc initResults: TResults =
+  result.total = 0
+  result.passed = 0
+  result.skipped = 0
+  result.data = ""
+
+#proc readResults(filename: string): TResults = # not used
+#  result = marshal.to[TResults](readFile(filename).string)
+
+#proc writeResults(filename: string, r: TResults) = # not used
+#  writeFile(filename, $$r)
+
+proc `$`(x: TResults): string =
+  result = ("Tests passed: $1 / $3 <br />\n" &
+            "Tests skipped: $2 / $3 <br />\n") %
+            [$x.passed, $x.skipped, $x.total]
+
+proc addResult(r: var TResults, test: TTest, target: TTarget,
+               expected, given: string, success: TResultEnum) =
+  let name = test.name.extractFilename & " " & $target & test.options
+  let duration = epochTime() - test.startTime
+  let durationStr = duration.formatFloat(ffDecimal, precision = 8)
+  backend.writeTestResult(name = name,
+                          category = test.cat.string,
+                          target = $target,
+                          action = $test.action,
+                          result = $success,
+                          expected = expected,
+                          given = given)
+  r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
+  if success == reSuccess:
+    styledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)"
+  elif success == reIgnored:
+    styledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name
+  else:
+    styledEcho styleBright, fgRed, "FAIL: ", fgCyan, name
+    styledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
+    styledEcho styleBright, fgRed, "Failure: ", $success
+    styledEcho fgYellow, "Expected:"
+    styledEcho styleBright, expected, "\n"
+    styledEcho fgYellow, "Gotten:"
+    styledEcho styleBright, given, "\n"
+
+  if existsEnv("APPVEYOR"):
+    let (outcome, msg) =
+      if success == reSuccess:
+        ("Passed", "")
+      elif success == reIgnored:
+        ("Skipped", "")
+      else:
+        ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
+    var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options,
+                         "-Framework", "nim-testament", "-FileName",
+                         test.cat.string,
+                         "-Outcome", outcome, "-ErrorMessage", msg,
+                         "-Duration", $(duration*1000).int],
+                         options={poStdErrToStdOut, poUsePath, poParentStreams})
+    discard waitForExit(p)
+    close(p)
+
+proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) =
+  if strip(expected.msg) notin strip(given.msg):
+    r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer)
+  elif expected.nimout.len > 0 and expected.nimout.normalizeMsg notin given.nimout.normalizeMsg:
+    r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer)
+  elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and
+      "internal error:" notin expected.msg:
+    r.addResult(test, target, expected.file, given.file, reFilesDiffer)
+  elif expected.line != given.line and expected.line != 0 or
+       expected.column != given.column and expected.column != 0:
+    r.addResult(test, target, $expected.line & ':' & $expected.column,
+                      $given.line & ':' & $given.column,
+                      reLinesDiffer)
+  elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and
+      "internal error:" notin expected.msg:
+    r.addResult(test, target, expected.tfile, given.tfile, reFilesDiffer)
+  elif expected.tline != given.tline and expected.tline != 0 or
+       expected.tcolumn != given.tcolumn and expected.tcolumn != 0:
+    r.addResult(test, target, $expected.tline & ':' & $expected.tcolumn,
+                      $given.tline & ':' & $given.tcolumn,
+                      reLinesDiffer)
+  else:
+    r.addResult(test, target, expected.msg, given.msg, reSuccess)
+    inc(r.passed)
+
+proc generatedFile(test: TTest, target: TTarget): string =
+  let (_, name, _) = test.name.splitFile
+  let ext = targetToExt[target]
+  result = nimcacheDir(test.name, test.options, target) /
+    (if target == targetJS: "" else: "compiler_") &
+    name.changeFileExt(ext)
+
+proc needsCodegenCheck(spec: TSpec): bool =
+  result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0
+
+proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var string,
+                  given: var TSpec) =
+  try:
+    let genFile = generatedFile(test, target)
+    let contents = readFile(genFile).string
+    let check = spec.ccodeCheck
+    if check.len > 0:
+      if check[0] == '\\':
+        # little hack to get 'match' support:
+        if not contents.match(check.peg):
+          given.err = reCodegenFailure
+      elif contents.find(check.peg) < 0:
+        given.err = reCodegenFailure
+      expectedMsg = check
+    if spec.maxCodeSize > 0 and contents.len > spec.maxCodeSize:
+      given.err = reCodegenFailure
+      given.msg = "generated code size: " & $contents.len
+      expectedMsg = "max allowed size: " & $spec.maxCodeSize
+  except ValueError:
+    given.err = reInvalidPeg
+    echo getCurrentExceptionMsg()
+  except IOError:
+    given.err = reCodeNotFound
+    echo getCurrentExceptionMsg()
+
+proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) =
+  let exp = expectedNimout.strip.replace("\C\L", "\L")
+  let giv = given.nimout.strip.replace("\C\L", "\L")
+  if exp notin giv:
+    given.err = reMsgsDiffer
+
+proc makeDeterministic(s: string): string =
+  var x = splitLines(s)
+  sort(x, system.cmp)
+  result = join(x, "\n")
+
+proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
+                         expected: TSpec; r: var TResults) =
+  var expectedmsg: string = ""
+  var givenmsg: string = ""
+  if given.err == reSuccess:
+    if expected.needsCodegenCheck:
+      codegenCheck(test, target, expected, expectedmsg, given)
+      givenmsg = given.msg
+    if expected.nimout.len > 0:
+      expectedmsg = expected.nimout
+      givenmsg = given.nimout.strip
+      nimoutCheck(test, expectedmsg, given)
+  else:
+    givenmsg = given.nimout.strip
+  if given.err == reSuccess: inc(r.passed)
+  r.addResult(test, target, expectedmsg, givenmsg, given.err)
+
+proc testSpec(r: var TResults, test: TTest, target = targetC) =
+  let tname = test.name.addFileExt(".nim")
+  #echo "TESTING ", tname
+  var expected: TSpec
+  if test.action != actionRunNoSpec:
+    expected = parseSpec(tname)
+    if test.action == actionRun and expected.action == actionCompile:
+      expected.action = actionRun
+  else:
+    specDefaults expected
+    expected.action = actionRunNoSpec
+
+  if expected.err == reIgnored:
+    r.addResult(test, target, "", "", reIgnored)
+    inc(r.skipped)
+    inc(r.total)
+    return
+
+  if expected.targets == {}:
+    expected.targets.incl(target)
+
+  for target in expected.targets:
+    inc(r.total)
+    if target notin targets:
+      r.addResult(test, target, "", "", reIgnored)
+      inc(r.skipped)
+      continue
+
+    case expected.action
+    of actionCompile:
+      var given = callCompiler(expected.cmd, test.name, test.options, target,
+        extraOptions=" --stdout --hint[Path]:off --hint[Processing]:off")
+      compilerOutputTests(test, target, given, expected, r)
+    of actionRun, actionRunNoSpec:
+      # In this branch of code "early return" pattern is clearer than deep
+      # nested conditionals - the empty rows in between to clarify the "danger"
+      var given = callCompiler(expected.cmd, test.name, test.options,
+                               target)
+
+      if given.err != reSuccess:
+        r.addResult(test, target, "", given.msg, given.err)
+        continue
+
+      let isJsTarget = target == targetJS
+      var exeFile: string
+      if isJsTarget:
+        let (_, file, _) = splitFile(tname)
+        exeFile = nimcacheDir(test.name, test.options, target) / file & ".js"
+      else:
+        exeFile = changeFileExt(tname, ExeExt)
+
+      if not existsFile(exeFile):
+        r.addResult(test, target, expected.outp, "executable not found", reExeNotFound)
+        continue
+
+      let nodejs = if isJsTarget: findNodeJs() else: ""
+      if isJsTarget and nodejs == "":
+        r.addResult(test, target, expected.outp, "nodejs binary not in PATH",
+                    reExeNotFound)
+        continue
+
+      let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile
+      var (buf, exitCode) = execCmdEx(exeCmd, options = {poStdErrToStdOut})
+
+      # Treat all failure codes from nodejs as 1. Older versions of nodejs used
+      # to return other codes, but for us it is sufficient to know that it's not 0.
+      if exitCode != 0: exitCode = 1
+
+      let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string))
+                 else: strip(buf.string)
+      let expectedOut = strip(expected.outp)
+
+      if exitCode != expected.exitCode:
+        r.addResult(test, target, "exitcode: " & $expected.exitCode,
+                          "exitcode: " & $exitCode & "\n\nOutput:\n" &
+                          bufB, reExitCodesDiffer)
+        continue
+
+      if bufB != expectedOut and expected.action != actionRunNoSpec:
+        if not (expected.substr and expectedOut in bufB):
+          given.err = reOutputsDiffer
+          r.addResult(test, target, expected.outp, bufB, reOutputsDiffer)
+          continue
+
+      compilerOutputTests(test, target, given, expected, r)
+      continue
+
+    of actionReject:
+      var given = callCompiler(expected.cmd, test.name, test.options,
+                               target)
+      cmpMsgs(r, expected, given, test, target)
+      continue
+
+proc testNoSpec(r: var TResults, test: TTest, target = targetC) =
+  # does not extract the spec because the file is not supposed to have any
+  #let tname = test.name.addFileExt(".nim")
+  inc(r.total)
+  let given = callCompiler(cmdTemplate(), test.name, test.options, target)
+  r.addResult(test, target, "", given.msg, given.err)
+  if given.err == reSuccess: inc(r.passed)
+
+proc testC(r: var TResults, test: TTest) =
+  # runs C code. Doesn't support any specs, just goes by exit code.
+  let tname = test.name.addFileExt(".c")
+  inc(r.total)
+  styledEcho "Processing ", fgCyan, extractFilename(tname)
+  var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC)
+  if given.err != reSuccess:
+    r.addResult(test, targetC, "", given.msg, given.err)
+  elif test.action == actionRun:
+    let exeFile = changeFileExt(test.name, ExeExt)
+    var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath})
+    if exitCode != 0: given.err = reExitCodesDiffer
+  if given.err == reSuccess: inc(r.passed)
+
+proc testExec(r: var TResults, test: TTest) =
+  # runs executable or script, just goes by exit code
+  inc(r.total)
+  let (outp, errC) = execCmdEx(test.options.strip())
+  var given: TSpec
+  specDefaults(given)
+  if errC == 0:
+    given.err = reSuccess
+  else:
+    given.err = reExitCodesDiffer
+    given.msg = outp.string
+
+  if given.err == reSuccess: inc(r.passed)
+  r.addResult(test, targetC, "", given.msg, given.err)
+
+proc makeTest(test, options: string, cat: Category, action = actionCompile,
+              env: string = ""): TTest =
+  # start with 'actionCompile', will be overwritten in the spec:
+  result = TTest(cat: cat, name: test, options: options,
+                 action: action, startTime: epochTime())
+
+when defined(windows):
+  const
+    # array of modules disabled from compilation test of stdlib.
+    disabledFiles = ["coro.nim", "fsmonitor.nim"]
+else:
+  const
+    # array of modules disabled from compilation test of stdlib.
+    disabledFiles = ["-"]
+
+include categories
+
+# proc runCaasTests(r: var TResults) =
+#   for test, output, status, mode in caasTestsRunner():
+#     r.addResult(test, "", output & "-> " & $mode,
+#                 if status: reSuccess else: reOutputsDiffer)
+
+proc main() =
+  os.putenv "NIMTEST_COLOR", "never"
+  os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
+
+  backend.open()
+  var optPrintResults = false
+  var optFailing = false
+
+  var targetsStr = ""
+
+  var p = initOptParser()
+  p.next()
+  while p.kind == cmdLongoption:
+    case p.key.string.normalize
+    of "print", "verbose": optPrintResults = true
+    of "failing": optFailing = true
+    of "pedantic": discard "now always enabled"
+    of "targets":
+      targetsStr = p.val.string
+      targets = parseTargets(targetsStr)
+    of "nim": compilerPrefix = p.val.string
+    else: quit Usage
+    p.next()
+  if p.kind != cmdArgument: quit Usage
+  var action = p.key.string.normalize
+  p.next()
+  var r = initResults()
+  case action
+  of "all":
+    let testsDir = "tests" & DirSep
+    var myself = quoteShell(findExe("testament" / "tester"))
+    if targetsStr.len > 0:
+      myself &= " " & quoteShell("--targets:" & targetsStr)
+
+    myself &= " " & quoteShell("--nim:" & compilerPrefix)
+
+    var cmds: seq[string] = @[]
+    let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: ""
+    for kind, dir in walkDir(testsDir):
+      assert testsDir.startsWith(testsDir)
+      let cat = dir[testsDir.len .. ^1]
+      if kind == pcDir and cat notin ["testdata", "nimcache"]:
+        cmds.add(myself & " cat " & quoteShell(cat) & rest)
+    for cat in AdditionalCategories:
+      cmds.add(myself & " cat " & quoteShell(cat) & rest)
+    quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams})
+  of "c", "cat", "category":
+    var cat = Category(p.key)
+    p.next
+    processCategory(r, cat, p.cmdLineRest.string)
+  of "r", "run":
+    let (dir, file) = splitPath(p.key.string)
+    let (_, subdir) = splitPath(dir)
+    var cat = Category(subdir)
+    processSingleTest(r, cat, p.cmdLineRest.string, file)
+  of "html":
+    generateHtml(resultsFile, optFailing)
+  else:
+    quit Usage
+
+  if optPrintResults:
+    if action == "html": openDefaultBrowser(resultsFile)
+    else: echo r, r.data
+  backend.close()
+  var failed = r.total - r.passed - r.skipped
+  if failed != 0:
+    echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ",
+      r.skipped, " failed: ", failed
+    quit(QuitFailure)
+
+if paramCount() == 0:
+  quit Usage
+main()
diff --git a/testament/tester.nim.cfg b/testament/tester.nim.cfg
new file mode 100644
index 000000000..27fd67075
--- /dev/null
+++ b/testament/tester.nim.cfg
@@ -0,0 +1 @@
+path = "$nim" # For compiler/nodejs