WIP, most tests are now skipped

4 files changed, 167 insertions, 163 deletions

diff --git a/testament/backend.nim b/testament/backend.nim
index 385f1171c..a1de5859a 100644
--- a/testament/backend.nim
+++ b/testament/backend.nim
@@ -45,8 +45,7 @@ var
   currentCategory: string
   entries: int
 
-proc writeTestResult*(name, category, target,
-                      action, result, expected, given: string) =
+proc writeTestResult*(name, category, target, result, expected, given: string) =
   createDir("testresults")
   if currentCategory != category:
     if currentCategory.len > 0:
@@ -58,7 +57,7 @@ proc writeTestResult*(name, category, target,
     entries = 0
 
   let jentry = %*{"name": name, "category": category, "target": target,
-    "action": action, "result": result, "expected": expected, "given": given,
+    "result": result, "expected": expected, "given": given,
     "machine": thisMachine.string, "commit": thisCommit.string, "branch": thisBranch}
   if entries > 0:
     results.writeLine(",")
diff --git a/testament/categories.nim b/testament/categories.nim
index 6694e3f0e..b641430a1 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -27,7 +27,7 @@ proc delNimCache(filename, options: string) =
 proc runRodFiles(r: var TResults, cat: Category, options: string) =
   template test(filename: string, clearCacheFirst=false) =
     if clearCacheFirst: delNimCache(filename, options)
-    testSpec r, makeTest(rodfilesDir / filename, options, cat, actionRun)
+    testSpec r, makeTest(rodfilesDir / filename, options, cat)
 
 
   # test basic recompilation scheme:
@@ -97,10 +97,12 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
     else:
       ""
 
-  testNoSpec c, makeTest("lib/nimrtl.nim",
-    options & " --app:lib -d:createNimRtl --threads:on", cat, actionCompile)
-  testNoSpec c, makeTest("tests/dll/server.nim",
-    options & " --app:lib -d:useNimRtl --threads:on" & rpath, cat, actionCompile)
+  var test1 = makeTest("lib/nimrtl.nim", options & " --app:lib -d:createNimRtl --threads:on", cat)
+  test1.spec.action = actionCompile
+  testSpec c, test1
+  var test2 = makeTest("tests/dll/server.nim", options & " --app:lib -d:useNimRtl --threads:on" & rpath, cat)
+  test2.spec.action = actionCompile
+  testSpec c, test2
 
   when defined(Windows):
     # windows looks in the dir of the exe (yay!):
@@ -120,7 +122,7 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
     safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll)
 
   testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl --threads:on" & rpath,
-                       cat, actionRun)
+                       cat)
 
 proc dllTests(r: var TResults, cat: Category, options: string) =
   # dummy compile result:
@@ -138,32 +140,32 @@ proc dllTests(r: var TResults, cat: Category, options: string) =
 proc gcTests(r: var TResults, cat: Category, options: string) =
   template testWithNone(filename: untyped) =
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " --gc:none", cat, actionRun)
+                  " --gc:none", cat)
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " -d:release --gc:none", cat, actionRun)
+                  " -d:release --gc:none", cat)
 
   template testWithoutMs(filename: untyped) =
-    testSpec r, makeTest("tests/gc" / filename, options, cat, actionRun)
+    testSpec r, makeTest("tests/gc" / filename, options, cat)
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " -d:release", cat, actionRun)
+                  " -d:release", cat)
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " -d:release -d:useRealtimeGC", cat, actionRun)
+                  " -d:release -d:useRealtimeGC", cat)
 
   template testWithoutBoehm(filename: untyped) =
     testWithoutMs filename
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " --gc:markAndSweep", cat, actionRun)
+                  " --gc:markAndSweep", cat)
     testSpec r, makeTest("tests/gc" / filename, options &
-                  " -d:release --gc:markAndSweep", cat, actionRun)
+                  " -d:release --gc:markAndSweep", cat)
   template test(filename: untyped) =
     testWithoutBoehm filename
     when not defined(windows) and not defined(android):
       # AR: cannot find any boehm.dll on the net, right now, so disabled
       # for windows:
       testSpec r, makeTest("tests/gc" / filename, options &
-                    " --gc:boehm", cat, actionRun)
+                    " --gc:boehm", cat)
       testSpec r, makeTest("tests/gc" / filename, options &
-                    " -d:release --gc:boehm", cat, actionRun)
+                    " -d:release --gc:boehm", cat)
 
   testWithoutBoehm "foreign_thr"
   test "gcemscripten"
@@ -196,17 +198,18 @@ proc longGCTests(r: var TResults, cat: Category, options: string) =
 
   var c = initResults()
   # According to ioTests, this should compile the file
-  testNoSpec c, makeTest("tests/realtimeGC/shared", options, cat, actionCompile)
-  testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat, actionRun)
-  testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat, actionRun)
+  testSpec c, makeTest("tests/realtimeGC/shared", options, cat)
+  #        ^- why is this not appended to r? Should this be discarded?
+  testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun
+  testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat)
 
 # ------------------------- threading tests -----------------------------------
 
 proc threadTests(r: var TResults, cat: Category, options: string) =
   template test(filename: untyped) =
-    testSpec r, makeTest(filename, options, cat, actionRun)
-    testSpec r, makeTest(filename, options & " -d:release", cat, actionRun)
-    testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat, actionRun)
+    testSpec r, makeTest(filename, options, cat)
+    testSpec r, makeTest(filename, options & " -d:release", cat)
+    testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat)
   for t in os.walkFiles("tests/threads/t*.nim"):
     test(t)
 
@@ -235,10 +238,8 @@ proc debuggerTests(r: var TResults, cat: Category, options: string) =
 
 proc jsTests(r: var TResults, cat: Category, options: string) =
   template test(filename: untyped) =
-    testSpec r, makeTest(filename, options & " -d:nodejs", cat,
-                         actionRun), targetJS
-    testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat,
-                         actionRun), targetJS
+    testSpec r, makeTest(filename, options & " -d:nodejs", cat), {targetJS}
+    testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat), {targetJS}
 
   for t in os.walkFiles("tests/js/t*.nim"):
     test(t)
@@ -259,14 +260,14 @@ proc jsTests(r: var TResults, cat: Category, options: string) =
 proc testNimInAction(r: var TResults, cat: Category, options: string) =
   let options = options & " --nilseqs:on"
 
-  template test(filename: untyped, action: untyped) =
-    testSpec r, makeTest(filename, options, cat, action)
+  template test(filename: untyped) =
+    testSpec r, makeTest(filename, options, cat)
 
   template testJS(filename: untyped) =
-    testSpec r, makeTest(filename, options, cat, actionCompile), targetJS
+    testSpec r, makeTest(filename, options, cat), {targetJS}
 
   template testCPP(filename: untyped) =
-    testSpec r, makeTest(filename, options, cat, actionCompile), targetCPP
+    testSpec r, makeTest(filename, options, cat), {targetCPP}
 
   let tests = [
     "niminaction/Chapter1/various1",
@@ -318,7 +319,7 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) =
 
   # Run the tests.
   for testfile in tests:
-    test "tests/" & testfile & ".nim", actionCompile
+    test "tests/" & testfile & ".nim"
 
   let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim"
   testJS jsFile
@@ -361,14 +362,17 @@ proc compileExample(r: var TResults, pattern, options: string, cat: Category) =
     testNoSpec r, makeTest(test, options, cat)
 
 proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
-  for test in os.walkFiles(pattern):
-    let name = extractFilename(test)
+  for testFile in os.walkFiles(pattern):
+    let name = extractFilename(testFile)
     if name notin disabledFiles:
-      let contents = readFile(test).string
-      if contents.contains("when isMainModule"):
-        testSpec r, makeTest(test, options, cat, actionRunNoSpec)
-      else:
-        testNoSpec r, makeTest(test, options, cat, actionCompile)
+
+
+      let contents = readFile(testFile).string
+
+      var testObj = makeTest(testFile, options, cat)
+      if "when isMainModule" notin contents:
+        testObj.spec.action = actionCompile
+      testSpec r, testObj
 
 # ----------------------------- nimble ----------------------------------------
 type PackageFilter = enum
@@ -472,7 +476,7 @@ proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
   let test = "tests" & DirSep &.? cat.string / test
   let target = if cat.string.normalize == "js": targetJS else: targetC
 
-  if existsFile(test): testSpec r, makeTest(test, options, cat), target
+  if existsFile(test): testSpec r, makeTest(test, options, cat), {target}
   else: echo "[Warning] - ", test, " test does not exist"
 
 proc processCategory(r: var TResults, cat: Category, options: string) =
@@ -521,8 +525,6 @@ proc processCategory(r: var TResults, cat: Category, options: string) =
     testNimblePackages(r, cat, pfAll)
   of "niminaction":
     testNimInAction(r, cat, options)
-  of "testament":
-    testTestament(r, cat, options)
   of "untestable":
     # We can't test it because it depends on a third party.
     discard # TODO: Move untestable tests to someplace else, i.e. nimble repo.
diff --git a/testament/specs.nim b/testament/specs.nim
index a86544c27..9b2347bf9 100644
--- a/testament/specs.nim
+++ b/testament/specs.nim
@@ -9,7 +9,6 @@
 
 import parseutils, strutils, os, osproc, streams, parsecfg
 
-
 var compilerPrefix* = "compiler" / "nim"
 
 let isTravis* = existsEnv("TRAVIS")
@@ -97,32 +96,8 @@ proc extractSpec(filename: string): string =
 when not defined(nimhygiene):
   {.pragma: inject.}
 
-template parseSpecAux(fillResult: untyped) =
-  var ss = newStringStream(extractSpec(filename))
-  var p {.inject.}: CfgParser
-  open(p, ss, filename, 1)
-  while true:
-    var e {.inject.} = next(p)
-    case e.kind
-    of cfgEof: break
-    of cfgSectionStart, cfgOption, cfgError:
-      echo ignoreMsg(p, e)
-    of cfgKeyValuePair:
-      fillResult
-  close(p)
-
-proc specDefaults*(result: var TSpec) =
-  result.msg = ""
-  result.outp = ""
-  result.nimout = ""
-  result.ccodeCheck = ""
+proc defaultSpec*(): TSpec =
   result.cmd = cmdTemplate()
-  result.line = 0
-  result.column = 0
-  result.tfile = ""
-  result.tline = 0
-  result.tcolumn = 0
-  result.maxCodeSize = 0
 
 proc parseTargets*(value: string): set[TTarget] =
   for v in value.normalize.splitWhitespace:
@@ -134,79 +109,109 @@ proc parseTargets*(value: string): set[TTarget] =
     else: echo "target ignored: " & v
 
 proc parseSpec*(filename: string): TSpec =
-  specDefaults(result)
+  result = defaultSpec()
   result.file = filename
-  parseSpecAux:
-    case normalize(e.key)
-    of "action":
-      case e.value.normalize
-      of "compile": result.action = actionCompile
-      of "run": result.action = actionRun
-      of "reject": result.action = actionReject
-      else: echo ignoreMsg(p, e)
-    of "file": result.file = e.value
-    of "line": discard parseInt(e.value, result.line)
-    of "column": discard parseInt(e.value, result.column)
-    of "tfile": result.tfile = e.value
-    of "tline": discard parseInt(e.value, result.tline)
-    of "tcolumn": discard parseInt(e.value, result.tcolumn)
-    of "output":
-      result.outputCheck = ocEqual
-      result.outp = strip(e.value)
-    of "input":
-      result.input = e.value
-    of "outputsub":
-      result.outputCheck = ocSubstr
-      result.outp = strip(e.value)
-    of "sortoutput":
-      result.sortoutput = parseCfgBool(e.value)
-    of "exitcode":
-      discard parseInt(e.value, result.exitCode)
-      result.action = actionRun
-    of "msg":
-      result.msg = e.value
-      if result.action != actionRun:
-        result.action = actionCompile
-    of "errormsg", "errmsg":
-      result.msg = e.value
-      result.action = actionReject
-    of "nimout":
-      result.nimout = e.value
-    of "disabled":
-      case e.value.normalize
-      of "y", "yes", "true", "1", "on": result.err = reIgnored
-      of "n", "no", "false", "0", "off": discard
-      of "win", "windows":
-        when defined(windows): result.err = reIgnored
-      of "linux":
-        when defined(linux): result.err = reIgnored
-      of "bsd":
-        when defined(bsd): result.err = reIgnored
-      of "macosx":
-        when defined(macosx): result.err = reIgnored
-      of "unix":
-        when defined(unix): result.err = reIgnored
-      of "posix":
-        when defined(posix): result.err = reIgnored
-      of "travis":
-        if isTravis: result.err = reIgnored
-      of "appveyor":
-        if isAppVeyor: result.err = reIgnored
-      else:
-        raise newException(ValueError, "cannot interpret as a bool: " & e.value)
-    of "cmd":
-      if e.value.startsWith("nim "):
-        result.cmd = compilerPrefix & e.value[3..^1]
+  var ss = newStringStream(extractSpec(filename))
+  var p {.inject.}: CfgParser
+  open(p, ss, filename, 1)
+  while true:
+    var e = next(p)
+    case e.kind
+    of cfgKeyValuePair:
+      case normalize(e.key)
+      of "action":
+        case e.value.normalize
+        of "compile":
+          result.action = actionCompile
+        of "run":
+          result.action = actionRun
+        of "reject":
+          result.action = actionReject
+        else:
+          echo ignoreMsg(p, e)
+      of "file":
+        result.file = e.value
+      of "line":
+        discard parseInt(e.value, result.line)
+      of "column":
+        discard parseInt(e.value, result.column)
+      of "tfile":
+        result.tfile = e.value
+      of "tline":
+        discard parseInt(e.value, result.tline)
+      of "tcolumn":
+        discard parseInt(e.value, result.tcolumn)
+      of "output":
+        result.outputCheck = ocEqual
+        result.outp = strip(e.value)
+      of "input":
+        result.input = e.value
+      of "outputsub":
+        result.outputCheck = ocSubstr
+        result.outp = strip(e.value)
+      of "sortoutput":
+        result.sortoutput = parseCfgBool(e.value)
+      of "exitcode":
+        discard parseInt(e.value, result.exitCode)
+        result.action = actionRun
+      of "msg":
+        result.msg = e.value
+        if result.action != actionRun:
+          result.action = actionCompile
+      of "errormsg", "errmsg":
+        result.msg = e.value
+        result.action = actionReject
+      of "nimout":
+        result.nimout = e.value
+      of "disabled":
+        case e.value.normalize
+        of "y", "yes", "true", "1", "on": result.err = reIgnored
+        of "n", "no", "false", "0", "off": discard
+        of "win", "windows":
+          when defined(windows): result.err = reIgnored
+        of "linux":
+          when defined(linux): result.err = reIgnored
+        of "bsd":
+          when defined(bsd): result.err = reIgnored
+        of "macosx":
+          when defined(macosx): result.err = reIgnored
+        of "unix":
+          when defined(unix): result.err = reIgnored
+        of "posix":
+          when defined(posix): result.err = reIgnored
+        of "travis":
+          if isTravis: result.err = reIgnored
+        of "appveyor":
+          if isAppVeyor: result.err = reIgnored
+        else:
+          raise newException(ValueError, "cannot interpret as a bool: " & e.value)
+      of "cmd":
+        if e.value.startsWith("nim "):
+          result.cmd = compilerPrefix & e.value[3..^1]
+        else:
+          result.cmd = e.value
+      of "ccodecheck":
+        result.ccodeCheck = e.value
+      of "maxcodesize":
+        discard parseInt(e.value, result.maxCodeSize)
+      of "target", "targets":
+        for v in e.value.normalize.splitWhitespace:
+          case v
+          of "c":
+            result.targets.incl(targetC)
+          of "cpp", "c++":
+            result.targets.incl(targetCpp)
+          of "objc":
+            result.targets.incl(targetObjC)
+          of "js":
+            result.targets.incl(targetJS)
+          else:
+            echo ignoreMsg(p, e)
       else:
-        result.cmd = e.value
-    of "ccodecheck": result.ccodeCheck = e.value
-    of "maxcodesize": discard parseInt(e.value, result.maxCodeSize)
-    of "target", "targets":
-      for v in e.value.normalize.splitWhitespace:
-        case v
-        of "c": result.targets.incl(targetC)
-        of "cpp", "c++": result.targets.incl(targetCpp)
-        of "objc": result.targets.incl(targetObjC)
-        of "js": result.targets.incl(targetJS)
-        else: echo ignoreMsg(p, e)
-    else: echo ignoreMsg(p, e)
+        echo ignoreMsg(p, e)
+
+    of cfgSectionStart, cfgOption, cfgError:
+      echo ignoreMsg(p, e)
+    of cfgEof:
+      break
+  close(p)
diff --git a/testament/tester.nim b/testament/tester.nim
index 60cf46423..d8c0f2e11 100644
--- a/testament/tester.nim
+++ b/testament/tester.nim
@@ -48,7 +48,7 @@ type
     name: string
     cat: Category
     options: string
-    action: TTestAction
+    spec: TSpec
     startTime: float
 
 # ----------------------------------------------------------------------------
@@ -113,7 +113,7 @@ proc nimcacheDir(filename, options: string, target: TTarget): string =
 proc callCompiler(cmdTemplate, filename, options: string,
                   target: TTarget, extraOptions=""): TSpec =
   let nimcache = nimcacheDir(filename, options, target)
-  let options = options & " " & ("--nimCache:" & nimcache).quoteShell & extraOptions
+  let options = options & " " & quoteShell("--nimCache:" & nimcache) & extraOptions
   let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
                        "options", options, "file", filename.quoteShell,
                        "filedir", filename.getFileDir()])
@@ -222,7 +222,6 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
   backend.writeTestResult(name = name,
                           category = test.cat.string,
                           target = $target,
-                          action = $test.action,
                           result = $success,
                           expected = expected,
                           given = given)
@@ -346,15 +345,10 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
   r.addResult(test, target, expectedmsg, givenmsg, given.err)
 
 proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
-  let tname = test.name.addFileExt(".nim")
-  var expected: TSpec
-  if test.action != actionRunNoSpec:
-    expected = parseSpec(tname)
-  else:
-    specDefaults expected
-    expected.action = actionRunNoSpec
+  var expected = test.spec
 
   if expected.err == reIgnored:
+    echo expected
     # targetC is a lie
     r.addResult(test, targetC, "", "", reIgnored)
     inc(r.skipped)
@@ -401,10 +395,10 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
       let isJsTarget = target == targetJS
       var exeFile: string
       if isJsTarget:
-        let (_, file, _) = splitFile(tname)
-        exeFile = nimcacheDir(test.name, test.options, target) / file & ".js"
+        let file = addFileExt(test.name, "js")
+        exeFile = nimcacheDir(test.name, test.options, target) / file
       else:
-        exeFile = changeFileExt(tname, ExeExt)
+        exeFile = addFileExt(test.name, ExeExt)
 
       if not existsFile(exeFile):
         r.addResult(test, target, expected.outp, "executable not found", reExeNotFound)
@@ -468,7 +462,7 @@ proc testNoSpec(r: var TResults, test: TTest, target = targetC) =
   r.addResult(test, target, "", given.msg, given.err)
   if given.err == reSuccess: inc(r.passed)
 
-proc testC(r: var TResults, test: TTest) =
+proc testC(r: var TResults, test: TTest, action: TTestAction) =
   # runs C code. Doesn't support any specs, just goes by exit code.
   let tname = test.name.addFileExt(".c")
   inc(r.total)
@@ -476,7 +470,7 @@ proc testC(r: var TResults, test: TTest) =
   var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC)
   if given.err != reSuccess:
     r.addResult(test, targetC, "", given.msg, given.err)
-  elif test.action == actionRun:
+  elif action == actionRun:
     let exeFile = changeFileExt(test.name, ExeExt)
     var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath})
     if exitCode != 0: given.err = reExitCodesDiffer
@@ -486,8 +480,7 @@ proc testExec(r: var TResults, test: TTest) =
   # runs executable or script, just goes by exit code
   inc(r.total)
   let (outp, errC) = execCmdEx(test.options.strip())
-  var given: TSpec
-  specDefaults(given)
+  var given: TSpec = defaultSpec()
   if errC == 0:
     given.err = reSuccess
   else:
@@ -497,8 +490,12 @@ proc testExec(r: var TResults, test: TTest) =
   if given.err == reSuccess: inc(r.passed)
   r.addResult(test, targetC, "", given.msg, given.err)
 
-proc makeTest(test, options: string, cat: Category, env: string = ""): TTest =
-  result = TTest(cat: cat, name: test, options: options, startTime: epochTime())
+proc makeTest(test, options: string, cat: Category): TTest =
+  result.cat = cat
+  result.name = test
+  result.options = options
+  result.spec = parseSpec(addFileExt(test, ".nim"))
+  result.startTime = epochTime()
 
 when defined(windows):
   const
@@ -535,7 +532,8 @@ proc main() =
     of "targets":
       targetsStr = p.val.string
       targets = parseTargets(targetsStr)
-    of "nim": compilerPrefix = p.val.string
+    of "nim":
+      compilerPrefix = p.val.string
     of "directory":
       setCurrentDir(p.val.string)
     of "colors":