new tester produces json output

5 files changed, 26 insertions, 715 deletions

diff --git a/tests/specials.nim b/tests/specials.nim
deleted file mode 100644
index 9ced66bbb..000000000
--- a/tests/specials.nim
+++ /dev/null
@@ -1,238 +0,0 @@
-#
-#
-#            Nimrod Tester
-#        (c) Copyright 2013 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Include for the tester that contains test suites that test special features
-## of the compiler.
-
-# included from tester.nim
-# ---------------- ROD file tests ---------------------------------------------
-
-const
-  rodfilesDir = "tests/rodfiles"
-  nimcacheDir = rodfilesDir / "nimcache"
-
-proc delNimCache() =
-  try:
-    removeDir(nimcacheDir)
-  except EOS:
-    echo "[Warning] could not delete: ", nimcacheDir
-    
-proc runRodFiles(r: var TResults, options: string) =
-  template test(filename: expr): stmt =
-    runSingleTest(r, rodfilesDir / filename, options)
-  
-  delNimCache()
-  
-  # test basic recompilation scheme:
-  test "hallo"
-  test "hallo"
-  # test incremental type information:
-  test "hallo2"
-  delNimCache()
-  
-  # test type converters:
-  test "aconv"
-  test "bconv"
-  delNimCache()
-  
-  # test G, A, B example from the documentation; test init sections:
-  test "deada"
-  test "deada2"
-  delNimCache()
-  
-  # test method generation:
-  test "bmethods"
-  test "bmethods2"
-  delNimCache()
-  
-  # test generics:
-  test "tgeneric1"
-  test "tgeneric2"
-  delNimCache()
-
-proc compileRodFiles(r: var TResults, options: string) =
-  template test(filename: expr): stmt =
-    compileSingleTest(r, rodfilesDir / filename, options)
-    
-  delNimCache()
-  # test DLL interfacing:
-  test "gtkex1"
-  test "gtkex2"
-  delNimCache()
-
-# --------------------- DLL generation tests ----------------------------------
-
-proc safeCopyFile(src, dest: string) =
-  try:
-    copyFile(src, dest)
-  except EOS:
-    echo "[Warning] could not copy: ", src, " to ", dest
-
-proc runBasicDLLTest(c, r: var TResults, options: string) =
-  compileSingleTest c, "lib/nimrtl.nim", options & " --app:lib -d:createNimRtl"
-  compileSingleTest c, "tests/dll/server.nim", 
-    options & " --app:lib -d:useNimRtl"
-  
-  when defined(Windows): 
-    # windows looks in the dir of the exe (yay!):
-    var nimrtlDll = DynlibFormat % "nimrtl"
-    safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll)
-  else:
-    # posix relies on crappy LD_LIBRARY_PATH (ugh!):
-    var libpath = getenv"LD_LIBRARY_PATH".string
-    if peg"\i '/nimrod' (!'/')* '/lib'" notin libpath:
-      echo "[Warning] insufficient LD_LIBRARY_PATH"
-    var serverDll = DynlibFormat % "server"
-    safeCopyFile("tests/dll" / serverDll, "lib" / serverDll)
-  
-  runSingleTest r, "tests/dll/client.nim", options & " -d:useNimRtl"
-
-proc runDLLTests(r: var TResults, options: string) =
-  # dummy compile result:
-  var c = initResults()
-  
-  runBasicDLLTest c, r, options
-  runBasicDLLTest c, r, options & " -d:release"
-  runBasicDLLTest c, r, options & " --gc:boehm"
-  runBasicDLLTest c, r, options & " -d:release --gc:boehm"
-
-proc compileDLLTests(r: var TResults, options: string) =
-  # dummy run result:
-  var c = initResults()
-  
-  runBasicDLLTest r, c, options
-  runBasicDLLTest r, c, options & " -d:release"
-  runBasicDLLTest r, c, options & " --gc:boehm"
-  runBasicDLLTest r, c, options & " -d:release --gc:boehm"
-
-# ------------------------------ GC tests -------------------------------------
-
-proc runGcTests(r: var TResults, options: string) =
-  template test(filename: expr): stmt =
-    runSingleTest(r, "tests/gc" / filename, options)
-    runSingleTest(r, "tests/gc" / filename, options & " -d:release")
-    runSingleTest(r, "tests/gc" / filename, options &
-                  " -d:release -d:useRealtimeGC")
-    runSingleTest(r, "tests/gc" / filename, options &
-                  " --gc:markAndSweep")
-    runSingleTest(r, "tests/gc" / filename, options &
-                  " -d:release --gc:markAndSweep")
-  
-  test "gcbench"
-  test "gcleak"
-  test "gcleak2"
-  test "gctest"
-  test "gcleak3"
-  test "weakrefs"
-  test "cycleleak"
-  test "closureleak"
-
-# ------------------------- threading tests -----------------------------------
-
-proc runThreadTests(r: var TResults, options: string) =
-  template test(filename: expr): stmt =
-    runSingleTest(r, "tests/threads" / filename, options)
-    runSingleTest(r, "tests/threads" / filename, options & " -d:release")
-    runSingleTest(r, "tests/threads" / filename, options & " --tlsEmulation:on")
-  
-  test "tactors"
-  test "tactors2"
-  test "threadex"
-  # deactivated because output capturing still causes problems sometimes:
-  #test "trecursive_actor"
-  #test "threadring"
-  #test "tthreadanalysis"
-  #test "tthreadsort"
-
-proc rejectThreadTests(r: var TResults, options: string) =
-  rejectSingleTest(r, "tests/threads/tthreadanalysis2", options)
-  rejectSingleTest(r, "tests/threads/tthreadanalysis3", options)
-  rejectSingleTest(r, "tests/threads/tthreadheapviolation1", options)
-
-# ------------------------- IO tests ------------------------------------------
-
-proc runIOTests(r: var TResults, options: string) =
-  # We need readall_echo to be compiled for this test to run.
-  # dummy compile result:
-  var c = initResults()
-  compileSingleTest(c, "tests/system/helpers/readall_echo", options)
-  runSingleTest(r, "tests/system/io", options)
-  
-# ------------------------- debugger tests ------------------------------------
-
-proc compileDebuggerTests(r: var TResults, options: string) =
-  compileSingleTest(r, "tools/nimgrep", options & 
-                    " --debugger:on")
-
-# ------------------------- JS tests ------------------------------------------
-
-proc runJsTests(r: var TResults, options: string) =
-  template test(filename: expr): stmt =
-    runSingleTest(r, filename, options & " -d:nodejs", targetJS)
-    runSingleTest(r, filename, options & " -d:nodejs -d:release", targetJS)
-    
-  for t in os.walkFiles("tests/js/t*.nim"):
-    test(t)
-  for testfile in ["texceptions", "texcpt1", "texcsub", "tfinally",
-                   "tfinally2", "tfinally3", "tactiontable", "tmultim1",
-                   "tmultim3", "tmultim4"]:
-    test "tests/run/" & testfile & ".nim"
-
-# ------------------------- register special tests here -----------------------
-proc runSpecialTests(r: var TResults, options: string) =
-  runRodFiles(r, options)
-  #runDLLTests(r, options)
-  runGCTests(r, options)
-  runThreadTests(r, options & " --threads:on")
-  runIOTests(r, options)
-
-  for t in os.walkFiles("tests/patterns/t*.nim"):
-    runSingleTest(r, t, options)
-  for t in ["lib/packages/docutils/highlite"]:
-    runSingleTest(r, t, options)
-
-proc rejectSpecialTests(r: var TResults, options: string) =
-  rejectThreadTests(r, options)
-
-proc findMainFile(dir: string): string =
-  # finds the file belonging to ".nimrod.cfg"; if there is no such file
-  # it returns the some ".nim" file if there is only one: 
-  const cfgExt = ".nimrod.cfg"
-  result = ""
-  var nimFiles = 0
-  for kind, file in os.walkDir(dir):
-    if kind == pcFile:
-      if file.endsWith(cfgExt): return file[.. -(cfgExt.len+1)] & ".nim"
-      elif file.endsWith(".nim"):
-        if result.len == 0: result = file
-        inc nimFiles
-  if nimFiles != 1: result.setlen(0)
-
-proc compileManyLoc(r: var TResults, options: string) =
-  for kind, dir in os.walkDir("tests/manyloc"):
-    if kind == pcDir:
-      let mainfile = findMainFile(dir)
-      if mainfile != ".nim":
-        compileSingleTest(r, mainfile, options)
-
-proc compileSpecialTests(r: var TResults, options: string) =
-  compileRodFiles(r, options)
-
-  compileSingleTest(r, "compiler/c2nim/c2nim.nim", options)
-  compileSingleTest(r, "compiler/pas2nim/pas2nim.nim", options)
-
-  compileDLLTests(r, options)
-  compileDebuggerTests(r, options)
-  
-  compileManyLoc(r, options)
-
-  #var given = callCompiler("nimrod i", "nimrod i", options)
-  #r.addResult("nimrod i", given.msg, if given.err: reFailure else: reSuccess)
-  #if not given.err: inc(r.passed)
-
diff --git a/tests/testament/backend.nim b/tests/testament/backend.nim
index bc1f92eba..5199bb9d6 100644
--- a/tests/testament/backend.nim
+++ b/tests/testament/backend.nim
@@ -49,10 +49,10 @@ proc createDb() =
   #  """, [])
 
 type
-  MachineId = distinct int64
+  MachineId* = distinct int64
   CommitId = distinct int64
 
-proc `$`(id: MachineId): string {.borrow.}
+proc `$`*(id: MachineId): string {.borrow.}
 proc `$`(id: CommitId): string {.borrow.}
 
 var
@@ -61,7 +61,7 @@ var
 
 proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip
 
-proc getMachine: MachineId =
+proc getMachine*(db: TDbConn): MachineId =
   var name = "hostname"()
   if name.len == 0:
     name = when defined(posix): getenv"HOSTNAME".string
@@ -76,7 +76,7 @@ proc getMachine: MachineId =
     result = db.insertId(sql"insert into Machine(name, os, cpu) values (?,?,?)",
                          name, system.hostOS, system.hostCPU).MachineId
 
-proc getCommit: CommitId =
+proc getCommit(db: TDbConn): CommitId =
   const commLen = "commit ".len
   let hash = "git log -n 1"()[commLen..commLen+10]
   let branch = "git symbolic-ref --short HEAD"()
@@ -115,7 +115,7 @@ proc open*() =
   db = open(connection="testament.db", user="testament", password="",
             database="testament")
   createDb()
-  thisMachine = getMachine()
-  thisCommit = getCommit()
+  thisMachine = getMachine(db)
+  thisCommit = getCommit(db)
 
 proc close*() = close(db)
diff --git a/tests/testament/htmlgen.nim b/tests/testament/htmlgen.nim
index bc2d8bd37..eb674a171 100644
--- a/tests/testament/htmlgen.nim
+++ b/tests/testament/htmlgen.nim
@@ -159,3 +159,22 @@ proc generateHtml*(filename: string, commit: int) =
   outfile.write(HtmlEnd)
   close(db)
   close(outfile)
+
+proc generateJson*(filename: string, commit: int) =
+  const selRow = """select count(*),
+                           sum(result = 'reSuccess'), 
+                           sum(result = 'reIgnored')
+                    from TestResult
+                    where [commit] = ? and machine = ?
+                    order by category"""
+  var db = open(connection="testament.db", user="testament", password="",
+                database="testament")
+  let lastCommit = db.getCommit(commit)
+
+  var outfile = open(filename, fmWrite)
+
+  let data = db.getRow(sql(selRow), lastCommit, $backend.getMachine(db))
+
+  outfile.writeln("""{"total": $#, "passed": $#, "skipped": $#}""" % data)
+  close(db)
+  close(outfile)
diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim
index 54a6de2d0..fac97cf2a 100644
--- a/tests/testament/tester.nim
+++ b/tests/testament/tester.nim
@@ -208,20 +208,6 @@ proc makeTest(test, options: string, cat: Category, action = actionCompile,
 
 include categories
 
-proc toJson(res: TResults): PJsonNode =
-  result = newJObject()
-  result["total"] = newJInt(res.total)
-  result["passed"] = newJInt(res.passed)
-  result["skipped"] = newJInt(res.skipped)
-
-proc outputJson(reject, compile, run: TResults) =
-  var doc = newJObject()
-  doc["reject"] = toJson(reject)
-  doc["compile"] = toJson(compile)
-  doc["run"] = toJson(run)
-  var s = pretty(doc)
-  writeFile(jsonFile, s)
-
 # proc runCaasTests(r: var TResults) =
 #   for test, output, status, mode in caasTestsRunner():
 #     r.addResult(test, "", output & "-> " & $mode,
@@ -259,6 +245,7 @@ proc main() =
     var commit = 0
     discard parseInt(p.cmdLineRest.string, commit)
     generateHtml(resultsFile, commit)
+    generateJson(jsonFile, commit)
   else:
     quit usage
 
diff --git a/tests/tester.nim b/tests/tester.nim
deleted file mode 100644
index 0e125b1bb..000000000
--- a/tests/tester.nim
+++ /dev/null
@@ -1,457 +0,0 @@
-#
-#
-#            Nimrod Tester
-#        (c) Copyright 2013 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This program verifies Nimrod against the testcases.
-
-import
-  parseutils, strutils, pegs, os, osproc, streams, parsecfg, browsers, json,
-  marshal, cgi, parseopt #, caas
-
-const
-  cmdTemplate = r"nimrod cc --hints:on $# $#"
-  resultsFile = "testresults.html"
-  jsonFile = "testresults.json"
-  Usage = "usage: tester [--print] " &
-                    "reject|compile|run|" &
-                    "merge|special|rodfiles| [nimrod options]\n" &
-          "   or: tester test|comp|rej singleTest"
-
-type
-  TTestAction = enum
-    actionCompile, actionRun, actionReject
-  TResultEnum = enum
-    reNimrodcCrash,     # nimrod compiler seems to have crashed
-    reMsgsDiffer,       # error messages differ
-    reFilesDiffer,      # expected and given filenames differ
-    reLinesDiffer,      # expected and given line numbers differ
-    reOutputsDiffer,
-    reExitcodesDiffer,
-    reInvalidPeg,
-    reCodegenFailure,
-    reCodeNotFound,
-    reExeNotFound,
-    reIgnored,          # test is ignored
-    reSuccess           # test was successful
-
-  TTarget = enum
-    targetC, targetCpp, targetObjC, targetJS
-
-  TSpec = object
-    action: TTestAction
-    file, cmd: string
-    outp: string
-    line, exitCode: int
-    msg: string
-    ccodeCheck: string
-    err: TResultEnum
-    substr: bool
-  TResults = object
-    total, passed, skipped: int
-    data: string
-
-# ----------------------- Spec parser ----------------------------------------
-
-when not defined(parseCfgBool):
-  # candidate for the stdlib:
-  proc parseCfgBool(s: string): bool =
-    case normalize(s)
-    of "y", "yes", "true", "1", "on": result = true
-    of "n", "no", "false", "0", "off": result = false
-    else: raise newException(EInvalidValue, "cannot interpret as a bool: " & s)
-
-proc extractSpec(filename: string): string =
-  const tripleQuote = "\"\"\""
-  var x = readFile(filename).string
-  var a = x.find(tripleQuote)
-  var b = x.find(tripleQuote, a+3)
-  # look for """ only in the first section
-  if a >= 0 and b > a and a < 40:
-    result = x.substr(a+3, b-1).replace("'''", tripleQuote)
-  else:
-    #echo "warning: file does not contain spec: " & filename
-    result = ""
-
-when not defined(nimhygiene):
-  {.pragma: inject.}
-
-template parseSpecAux(fillResult: stmt) {.immediate.} =
-  var ss = newStringStream(extractSpec(filename))
-  var p {.inject.}: TCfgParser
-  open(p, ss, filename, 1)
-  while true:
-    var e {.inject.} = next(p)
-    case e.kind
-    of cfgEof: break
-    of cfgSectionStart, cfgOption, cfgError:
-      echo ignoreMsg(p, e)
-    of cfgKeyValuePair:
-      fillResult
-  close(p)
-
-proc parseSpec(filename: string): TSpec =
-  result.file = filename
-  result.msg = ""
-  result.outp = ""
-  result.ccodeCheck = ""
-  result.cmd = cmdTemplate
-  parseSpecAux:
-    case normalize(e.key)
-    of "action":
-      case e.value.normalize
-      of "compile": result.action = actionCompile
-      of "run": result.action = actionRun
-      of "reject": result.action = actionReject
-      else: echo ignoreMsg(p, e)
-    of "file": result.file = e.value
-    of "line": discard parseInt(e.value, result.line)
-    of "output": result.outp = e.value
-    of "outputsub":
-      result.outp = e.value
-      result.substr = true
-    of "exitcode": 
-      discard parseInt(e.value, result.exitCode)
-    of "errormsg", "msg": result.msg = e.value
-    of "disabled":
-      if parseCfgBool(e.value): result.err = reIgnored
-    of "cmd": result.cmd = e.value
-    of "ccodecheck": result.ccodeCheck = e.value
-    else: echo ignoreMsg(p, e)
-
-# ----------------------------------------------------------------------------
-
-let
-  pegLineError = 
-    peg"{[^(]*} '(' {\d+} ', ' \d+ ') ' ('Error'/'Warning') ':' \s* {.*}"
-  pegOtherError = peg"'Error:' \s* {.*}"
-  pegSuccess = peg"'Hint: operation successful'.*"
-  pegOfInterest = pegLineError / pegOtherError
-
-proc callCompiler(cmdTemplate, filename, options: string): TSpec =
-  let c = parseCmdLine(cmdTemplate % [options, filename])
-  var p = startProcess(command=c[0], args=c[1.. -1],
-                       options={poStdErrToStdOut, poUseShell})
-  let outp = p.outputStream
-  var suc = ""
-  var err = ""
-  var x = newStringOfCap(120)
-  while outp.readLine(x.TaintedString) or running(p):
-    if x =~ pegOfInterest:
-      # `err` should contain the last error/warning message
-      err = x
-    elif x =~ pegSuccess:
-      suc = x
-  close(p)
-  result.msg = ""
-  result.file = ""
-  result.outp = ""
-  result.line = -1
-  if err =~ pegLineError:
-    result.file = extractFilename(matches[0])
-    result.line = parseInt(matches[1])
-    result.msg = matches[2]
-  elif err =~ pegOtherError:
-    result.msg = matches[0]
-  elif suc =~ pegSuccess:
-    result.err = reSuccess
-
-proc initResults: TResults =
-  result.total = 0
-  result.passed = 0
-  result.skipped = 0
-  result.data = ""
-
-proc readResults(filename: string): TResults =
-  result = marshal.to[TResults](readFile(filename).string)
-
-proc writeResults(filename: string, r: TResults) =
-  writeFile(filename, $$r)
-
-proc `$`(x: TResults): string =
-  result = ("Tests passed: $1 / $3 <br />\n" &
-            "Tests skipped: $2 / $3 <br />\n") %
-            [$x.passed, $x.skipped, $x.total]
-
-proc colorResult(r: TResultEnum): string =
-  case r
-  of reIgnored: result = "<span style=\"color:fuchsia\">ignored</span>"
-  of reSuccess: result = "<span style=\"color:green\">yes</span>"
-  else: result = "<span style=\"color:red\">no</span>"
-
-const
-  TableHeader4 = "<table border=\"1\"><tr><td>Test</td><td>Expected</td>" &
-                 "<td>Given</td><td>Success</td></tr>\n"
-  TableHeader3 = "<table border=\"1\"><tr><td>Test</td>" &
-                 "<td>Given</td><td>Success</td></tr>\n"
-  TableFooter = "</table>\n"
-  HtmlBegin = """<html>
-    <head> 
-      <title>Test results</title>
-      <style type="text/css">
-      <!--""" & slurp("css/boilerplate.css") & "\n" &
-                slurp("css/style.css") &
-      """-->
-    </style>
-
-    </head>
-    <body>"""
-  
-  HtmlEnd = "</body></html>"
-
-proc td(s: string): string =
-  result = s.substr(0, 200).XMLEncode
-
-proc addResult(r: var TResults, test, expected, given: string,
-               success: TResultEnum) =
-  r.data.addf("<tr><td>$#</td><td>$#</td><td>$#</td><td>$#</td></tr>\n", [
-    XMLEncode(test), td(expected), td(given), success.colorResult])
-
-proc addResult(r: var TResults, test, given: string,
-               success: TResultEnum) =
-  r.data.addf("<tr><td>$#</td><td>$#</td><td>$#</td></tr>\n", [
-    XMLEncode(test), td(given), success.colorResult])
-
-proc listResults(reject, compile, run: TResults) =
-  var s = HtmlBegin
-  s.add("<h1>Tests to Reject</h1>\n")
-  s.add($reject)
-  s.add(TableHeader4 & reject.data & TableFooter)
-  s.add("<br /><br /><br /><h1>Tests to Compile</h1>\n")
-  s.add($compile)
-  s.add(TableHeader3 & compile.data & TableFooter)
-  s.add("<br /><br /><br /><h1>Tests to Run</h1>\n")
-  s.add($run)
-  s.add(TableHeader4 & run.data & TableFooter)
-  s.add(HtmlEnd)
-  writeFile(resultsFile, s)
-
-proc cmpMsgs(r: var TResults, expected, given: TSpec, test: string) =
-  if strip(expected.msg) notin strip(given.msg):
-    r.addResult(test, expected.msg, given.msg, reMsgsDiffer)
-  elif extractFilename(expected.file) != extractFilename(given.file) and
-      "internal error:" notin expected.msg:
-    r.addResult(test, expected.file, given.file, reFilesDiffer)
-  elif expected.line != given.line and expected.line != 0:
-    r.addResult(test, $expected.line, $given.line, reLinesDiffer)
-  else:
-    r.addResult(test, expected.msg, given.msg, reSuccess)
-    inc(r.passed)
-
-proc rejectSingleTest(r: var TResults, test, options: string) =
-  let test = test.addFileExt(".nim")
-  var t = extractFilename(test)
-  inc(r.total)
-  echo t
-  var expected = parseSpec(test)
-  if expected.err == reIgnored:
-    r.addResult(t, "", "", reIgnored)
-    inc(r.skipped)
-  else:
-    var given = callCompiler(expected.cmd, test, options)
-    cmpMsgs(r, expected, given, t)
-
-proc reject(r: var TResults, dir, options: string) =
-  ## handle all the tests that the compiler should reject
-  for test in os.walkFiles(dir / "t*.nim"): rejectSingleTest(r, test, options)
-
-proc codegenCheck(test, check, ext: string, given: var TSpec) =
-  if check.len > 0:
-    try:
-      let (path, name, ext2) = test.splitFile
-      echo path / "nimcache" / name.changeFileExt(ext)
-      let contents = readFile(path / "nimcache" / name.changeFileExt(ext)).string
-      if contents.find(check.peg) < 0:
-        given.err = reCodegenFailure
-    except EInvalidValue:
-      given.err = reInvalidPeg
-    except EIO:
-      given.err = reCodeNotFound
-  
-proc codegenChecks(test: string, expected: TSpec, given: var TSpec) =
-  codegenCheck(test, expected.ccodeCheck, ".c", given)
-  
-proc compile(r: var TResults, pattern, options: string) =
-  for test in os.walkFiles(pattern):
-    let t = extractFilename(test)
-    echo t
-    inc(r.total)
-    let expected = parseSpec(test)
-    if expected.err == reIgnored:
-      r.addResult(t, "", reIgnored)
-      inc(r.skipped)
-    else:
-      var given = callCompiler(expected.cmd, test, options)
-      if given.err == reSuccess:
-        codegenChecks(test, expected, given)
-      r.addResult(t, given.msg, given.err)
-      if given.err == reSuccess: inc(r.passed)
-
-proc compileSingleTest(r: var TResults, test, options: string) =
-  # does not extract the spec because the file is not supposed to have any
-  let test = test.addFileExt(".nim")
-  let t = extractFilename(test)
-  inc(r.total)
-  echo t
-  let given = callCompiler(cmdTemplate, test, options)
-  r.addResult(t, given.msg, given.err)
-  if given.err == reSuccess: inc(r.passed)
-
-proc runSingleTest(r: var TResults, test, options: string, target: TTarget) =
-  var test = test.addFileExt(".nim")
-  var t = extractFilename(test)
-  echo t
-  inc(r.total)
-  var expected = parseSpec(test)
-  if expected.err == reIgnored:
-    r.addResult(t, "", "", reIgnored)
-    inc(r.skipped)
-  else:
-    var given = callCompiler(expected.cmd, test, options)
-    if given.err != reSuccess:
-      r.addResult(t, "", given.msg, given.err)
-    else:
-      var exeFile: string
-      if target == targetC:
-        exeFile = changeFileExt(test, ExeExt)
-      else:
-        let (dir, file, ext) = splitFile(test)
-        exeFile = dir / "nimcache" / file & ".js"
-      
-      if existsFile(exeFile):
-        var (buf, exitCode) = execCmdEx(
-          (if target==targetJS: "node " else: "") & exeFile)
-        if exitCode != expected.ExitCode:
-          r.addResult(t, "exitcode: " & $expected.ExitCode,
-                         "exitcode: " & $exitCode, reExitCodesDiffer)
-        else:
-          if strip(buf.string) != strip(expected.outp):
-            if not (expected.substr and expected.outp in buf.string):
-              given.err = reOutputsDiffer
-          if given.err == reSuccess:
-            codeGenChecks(test, expected, given)
-          if given.err == reSuccess: inc(r.passed)
-          r.addResult(t, expected.outp, buf.string, given.err)
-      else:
-        r.addResult(t, expected.outp, "executable not found", reExeNotFound)
-
-proc runSingleTest(r: var TResults, test, options: string) =
-  runSingleTest(r, test, options, targetC)
-
-proc run(r: var TResults, dir, options: string) =
-  for test in os.walkFiles(dir / "t*.nim"): runSingleTest(r, test, options)
-
-include specials
-
-proc compileExample(r: var TResults, pattern, options: string) =
-  for test in os.walkFiles(pattern): compileSingleTest(r, test, options)
-
-proc toJson(res: TResults): PJsonNode =
-  result = newJObject()
-  result["total"] = newJInt(res.total)
-  result["passed"] = newJInt(res.passed)
-  result["skipped"] = newJInt(res.skipped)
-
-proc outputJSON(reject, compile, run: TResults) =
-  var doc = newJObject()
-  doc["reject"] = toJson(reject)
-  doc["compile"] = toJson(compile)
-  doc["run"] = toJson(run)
-  var s = pretty(doc)
-  writeFile(jsonFile, s)
-
-# proc runCaasTests(r: var TResults) =
-#   for test, output, status, mode in caasTestsRunner():
-#     r.addResult(test, "", output & "-> " & $mode,
-#                 if status: reSuccess else: reOutputsDiffer)
-
-proc main() =
-  os.putenv "NIMTEST_NO_COLOR", "1"
-  os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
-
-  const
-    compileJson = "compile.json"
-    runJson = "run.json"
-    rejectJson = "reject.json"
-  
-  var optPrintResults = false
-  var p = initOptParser()
-  p.next()
-  if p.kind == cmdLongoption:
-    case p.key.string
-    of "print": optPrintResults = true
-    else: quit usage
-    p.next()
-  if p.kind != cmdArgument: quit usage
-  var action = p.key.string.normalize
-  p.next()
-  var r = initResults()
-  case action
-  of "reject":
-    reject(r, "tests/reject", p.cmdLineRest.string)
-    rejectSpecialTests(r, p.cmdLineRest.string)
-    writeResults(rejectJson, r)
-  of "compile":
-    compile(r, "tests/compile/t*.nim", p.cmdLineRest.string)
-    compile(r, "tests/ccg/t*.nim", p.cmdLineRest.string)
-    compile(r, "tests/js.nim", p.cmdLineRest.string)
-    compileExample(r, "lib/pure/*.nim", p.cmdLineRest.string)
-    compileExample(r, "examples/*.nim", p.cmdLineRest.string)
-    compileExample(r, "examples/gtk/*.nim", p.cmdLineRest.string)
-    compileExample(r, "examples/talk/*.nim", p.cmdLineRest.string)
-    compileSpecialTests(r, p.cmdLineRest.string)
-    writeResults(compileJson, r)
-  of "run":
-    run(r, "tests/run", p.cmdLineRest.string)
-    runSpecialTests(r, p.cmdLineRest.string)
-    writeResults(runJson, r)
-  of "special":
-    runSpecialTests(r, p.cmdLineRest.string)
-    # runCaasTests(r)
-    writeResults(runJson, r)
-  of "rodfiles":
-    runRodFiles(r, p.cmdLineRest.string)
-    writeResults(runJson, r)
-  of "js":
-    if existsFile(runJSon):
-      r = readResults(runJson)
-    runJsTests(r, p.cmdLineRest.string)
-    writeResults(runJson, r)
-  of "merge":
-    var rejectRes = readResults(rejectJson)
-    var compileRes = readResults(compileJson)
-    var runRes = readResults(runJson)
-    listResults(rejectRes, compileRes, runRes)
-    outputJSON(rejectRes, compileRes, runRes)
-  of "dll":
-    runDLLTests r, p.cmdLineRest.string
-  of "gc":
-    runGCTests(r, p.cmdLineRest.string)
-  of "test":
-    if p.kind != cmdArgument: quit usage
-    var testFile = p.key.string
-    p.next()
-    runSingleTest(r, testFile, p.cmdLineRest.string)
-  of "comp", "rej":
-    if p.kind != cmdArgument: quit usage
-    var testFile = p.key.string
-    p.next()
-    if peg"'/reject/'" in testFile or action == "rej":
-      rejectSingleTest(r, testFile, p.cmdLineRest.string)
-    elif peg"'/compile/'" in testFile or action == "comp":
-      compileSingleTest(r, testFile, p.cmdLineRest.string)
-    else:
-      runSingleTest(r, testFile, p.cmdLineRest.string)
-  else:
-    quit usage
-
-  if optPrintResults: echo r, r.data
-
-if paramCount() == 0:
-  quit usage
-main()
-