diff options
Diffstat (limited to 'tests/testament')
| -rw-r--r-- | tests/testament/backend.nim | 20 | ||||
| -rw-r--r-- | tests/testament/caasdriver.nim | 8 | ||||
| -rw-r--r-- | tests/testament/categories.nim | 41 | ||||
| -rw-r--r-- | tests/testament/htmlgen.nim | 12 | ||||
| -rw-r--r-- | tests/testament/specs.nim | 30 | ||||
| -rw-r--r-- | tests/testament/tester.nim | 238 |
6 files changed, 253 insertions, 96 deletions
diff --git a/tests/testament/backend.nim b/tests/testament/backend.nim index 11743c337..e2e2e2dd5 100644 --- a/tests/testament/backend.nim +++ b/tests/testament/backend.nim @@ -81,34 +81,34 @@ proc getCommit(db: TDbConn): CommitId = let hash = "git log -n 1"()[commLen..commLen+10] let branch = "git symbolic-ref --short HEAD"() if hash.len == 0 or branch.len == 0: quit "cannot determine git HEAD" - + let id = db.getValue(sql"select id from [Commit] where hash = ? and branch = ?", hash, branch) if id.len > 0: result = id.parseInt.CommitId else: - result = db.insertId(sql"insert into [Commit](hash, branch) values (?, ?)", + result = db.insertId(sql"insert into [Commit](hash, branch) values (?, ?)", hash, branch).CommitId -proc writeTestResult*(name, category, target, +proc writeTestResult*(name, category, target, action, result, expected, given: string) = - let id = db.getValue(sql"""select id from TestResult + let id = db.getValue(sql"""select id from TestResult where name = ? and category = ? and target = ? and - machine = ? and [commit] = ?""", + machine = ? and [commit] = ?""", name, category, target, thisMachine, thisCommit) if id.len > 0: db.exec(sql"""update TestResult - set action = ?, result = ?, expected = ?, given = ? + set action = ?, result = ?, expected = ?, given = ? where id = ?""", action, result, expected, given, id) else: - db.exec(sql"""insert into TestResult(name, category, target, - action, + db.exec(sql"""insert into TestResult(name, category, target, + action, result, expected, given, [commit], machine) - values (?,?,?,?,?,?,?,?,?) """, name, category, target, + values (?,?,?,?,?,?,?,?,?) """, name, category, target, action, - result, expected, given, + result, expected, given, thisCommit, thisMachine) proc open*() = diff --git a/tests/testament/caasdriver.nim b/tests/testament/caasdriver.nim index c61a9f108..30383bddb 100644 --- a/tests/testament/caasdriver.nim +++ b/tests/testament/caasdriver.nim @@ -137,11 +137,11 @@ proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): if line.strip.len == 0: continue if line.startsWith("#"): - output.writeln line + output.writeLine line continue elif line.startsWith(">"): s.doCommand(line.substr(1).strip) - output.writeln line, "\n", if verbose: s.lastOutput else: "" + output.writeLine line, "\n", if verbose: s.lastOutput else: "" else: var expectMatch = true var pattern = s.replaceVars(line) @@ -153,9 +153,9 @@ proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): s.lastOutput.find(re(pattern, flags = {reStudy})) != -1 if expectMatch == actualMatch: - output.writeln "SUCCESS ", line + output.writeLine "SUCCESS ", line else: - output.writeln "FAILURE ", line + output.writeLine "FAILURE ", line result = false iterator caasTestsRunner*(filter = "", verbose = false): tuple[test, diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index 4476fccf2..3166942ec 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -89,8 +89,11 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = var libpath = getEnv"LD_LIBRARY_PATH".string # Temporarily add the lib directory to LD_LIBRARY_PATH: putEnv("LD_LIBRARY_PATH", "lib:" & libpath) + defer: putEnv("LD_LIBRARY_PATH", libpath) var serverDll = DynlibFormat % "server" safeCopyFile("tests/dll" / serverDll, "lib" / serverDll) + var nimrtlDll = DynlibFormat % "nimrtl" + safeCopyFile("tests/dll" / nimrtlDll, "lib" / nimrtlDll) testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl", cat, actionRun) @@ -114,13 +117,23 @@ proc gcTests(r: var TResults, cat: Category, options: string) = testSpec r, makeTest("tests/gc" / filename, options & " -d:release -d:useRealtimeGC", cat, actionRun) - template test(filename: expr): stmt = + template testWithoutBoehm(filename: expr): stmt = testWithoutMs filename testSpec r, makeTest("tests/gc" / filename, options & " --gc:markAndSweep", cat, actionRun) testSpec r, makeTest("tests/gc" / filename, options & " -d:release --gc:markAndSweep", cat, actionRun) - + template test(filename: expr): stmt = + testWithoutBoehm filename + when not defined(windows): + # AR: cannot find any boehm.dll on the net, right now, so disabled + # for windows: + testSpec r, makeTest("tests/gc" / filename, options & + " --gc:boehm", cat, actionRun) + testSpec r, makeTest("tests/gc" / filename, options & + " -d:release --gc:boehm", cat, actionRun) + + test "gcemscripten" test "growobjcrash" test "gcbench" test "gcleak" @@ -130,14 +143,26 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "gcleak4" # Disabled because it works and takes too long to run: #test "gcleak5" - test "weakrefs" + testWithoutBoehm "weakrefs" test "cycleleak" - test "closureleak" + testWithoutBoehm "closureleak" testWithoutMs "refarrayleak" test "stackrefleak" test "cyclecollector" +proc longGCTests(r: var TResults, cat: Category, options: string) = + when defined(windows): + let cOptions = "gcc -ldl -DWIN" + else: + let cOptions = "gcc -ldl" + + var c = initResults() + # According to ioTests, this should compile the file + testNoSpec c, makeTest("tests/realtimeGC/shared", options, cat, actionCompile) + testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat, actionRun) + testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat, actionRun) + # ------------------------- threading tests ----------------------------------- proc threadTests(r: var TResults, cat: Category, options: string) = @@ -226,7 +251,7 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = for test in os.walkFiles(pattern): let contents = readFile(test).string if contents.contains("when isMainModule"): - testSpec r, makeTest(test, options, cat, actionRun) + testSpec r, makeTest(test, options, cat, actionRunNoSpec) else: testNoSpec r, makeTest(test, options, cat, actionCompile) @@ -327,7 +352,7 @@ proc `&?.`(a, b: string): string = # candidate for the stdlib? result = if a.endswith(b): a else: a & b -proc processCategory(r: var TResults, cat: Category, options: string) = +proc processCategory(r: var TResults, cat: Category, options: string, fileGlob: string = "t*.nim") = case cat.string.normalize of "rodfiles": discard # Disabled for now @@ -340,6 +365,8 @@ proc processCategory(r: var TResults, cat: Category, options: string) = dllTests(r, cat, options) of "gc": gcTests(r, cat, options) + of "longgc": + longGCTests(r, cat, options) of "debugger": debuggerTests(r, cat, options) of "manyloc": @@ -362,5 +389,5 @@ proc processCategory(r: var TResults, cat: Category, options: string) = of "nimble-all": testNimblePackages(r, cat, pfAll) else: - for name in os.walkFiles("tests" & DirSep &.? cat.string / "t*.nim"): + for name in os.walkFiles("tests" & DirSep &.? cat.string / fileGlob): testSpec r, makeTest(name, options, cat) diff --git a/tests/testament/htmlgen.nim b/tests/testament/htmlgen.nim index a9f739995..98ccf1170 100644 --- a/tests/testament/htmlgen.nim +++ b/tests/testament/htmlgen.nim @@ -138,7 +138,7 @@ proc generateHtml*(filename: string, commit: int; onlyFailing: bool) = # generate navigation: outfile.write("""<ul id="tabs">""") for m in db.rows(sql"select id, name, os, cpu from Machine order by id"): - outfile.writeln """<li><a href="#$#">$#: $#, $#</a></li>""" % m + outfile.writeLine """<li><a href="#$#">$#: $#, $#</a></li>""" % m outfile.write("</ul>") for currentMachine in db.rows(sql"select id from Machine order by id"): @@ -195,7 +195,7 @@ proc generateJson*(filename: string, commit: int) = let machine = $backend.getMachine(db) let data = db.getRow(sql(selRow), lastCommit, machine) - outfile.writeln("""{"total": $#, "passed": $#, "skipped": $#""" % data) + outfile.writeLine("""{"total": $#, "passed": $#, "skipped": $#""" % data) let results = newJArray() for row in db.rows(sql(selResults), lastCommit): @@ -208,7 +208,7 @@ proc generateJson*(filename: string, commit: int) = obj["expected"] = %row[5] obj["given"] = %row[6] results.add(obj) - outfile.writeln(""", "results": """) + outfile.writeLine(""", "results": """) outfile.write(results.pretty) if not previousCommit.isNil: @@ -220,9 +220,9 @@ proc generateJson*(filename: string, commit: int) = obj["old"] = %row[1] obj["new"] = %row[2] diff.add obj - outfile.writeln(""", "diff": """) - outfile.writeln(diff.pretty) + outfile.writeLine(""", "diff": """) + outfile.writeLine(diff.pretty) - outfile.writeln "}" + outfile.writeLine "}" close(db) close(outfile) diff --git a/tests/testament/specs.nim b/tests/testament/specs.nim index 2a8a4ea24..bab17d2cd 100644 --- a/tests/testament/specs.nim +++ b/tests/testament/specs.nim @@ -10,13 +10,14 @@ import parseutils, strutils, os, osproc, streams, parsecfg const - cmdTemplate* = r"nim $target --hints:on $options $file" + cmdTemplate* = r"compiler" / "nim $target --lib:lib --hints:on -d:testing $options $file" type TTestAction* = enum actionCompile = "compile" actionRun = "run" actionReject = "reject" + actionRunNoSpec = "runNoSpec" TResultEnum* = enum reNimcCrash, # nim compiler seems to have crashed reMsgsDiffer, # error messages differ @@ -42,7 +43,10 @@ type action*: TTestAction file*, cmd*: string outp*: string - line*, exitCode*: int + line*, column*: int + tfile*: string + tline*, tcolumn*: int + exitCode*: int msg*: string ccodeCheck*: string err*: TResultEnum @@ -77,7 +81,7 @@ proc extractSpec(filename: string): string = when not defined(nimhygiene): {.pragma: inject.} -template parseSpecAux(fillResult: stmt) {.immediate.} = +template parseSpecAux(fillResult: untyped) = var ss = newStringStream(extractSpec(filename)) var p {.inject.}: CfgParser open(p, ss, filename, 1) @@ -91,13 +95,21 @@ template parseSpecAux(fillResult: stmt) {.immediate.} = fillResult close(p) -proc parseSpec*(filename: string): TSpec = - result.file = filename +proc specDefaults*(result: var TSpec) = result.msg = "" result.outp = "" result.nimout = "" result.ccodeCheck = "" result.cmd = cmdTemplate + result.line = 0 + result.column = 0 + result.tfile = "" + result.tline = 0 + result.tcolumn = 0 + +proc parseSpec*(filename: string): TSpec = + specDefaults(result) + result.file = filename parseSpecAux: case normalize(e.key) of "action": @@ -108,7 +120,11 @@ proc parseSpec*(filename: string): TSpec = else: echo ignoreMsg(p, e) of "file": result.file = e.value of "line": discard parseInt(e.value, result.line) - of "output": + of "column": discard parseInt(e.value, result.column) + of "tfile": result.tfile = e.value + of "tline": discard parseInt(e.value, result.tline) + of "tcolumn": discard parseInt(e.value, result.tcolumn) + of "output": result.action = actionRun result.outp = e.value of "outputsub": @@ -117,7 +133,7 @@ proc parseSpec*(filename: string): TSpec = result.substr = true of "sortoutput": result.sortoutput = parseCfgBool(e.value) - of "exitcode": + of "exitcode": discard parseInt(e.value, result.exitCode) of "msg": result.msg = e.value diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index 7391b105e..636093a7f 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -12,7 +12,7 @@ import parseutils, strutils, pegs, os, osproc, streams, parsecfg, json, marshal, backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, compiler/nodejs + algorithm, compiler/nodejs, re const resultsFile = "testresults.html" @@ -23,6 +23,7 @@ const Command: all run all tests c|category <category> run all the tests of a certain category + r|run <test> run single test file html [commit] generate $1 from the database; uses the latest commit or a specific one (use -1 for the commit before latest etc) @@ -31,6 +32,7 @@ Arguments: Options: --print also print results to the console --failing only show failing/ignored tests + --pedantic return non-zero status code if there are failures """ % resultsFile type @@ -50,7 +52,9 @@ type let pegLineError = - peg"{[^(]*} '(' {\d+} ', ' \d+ ') ' ('Error') ':' \s* {.*}" + peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" + pegLineTemplate = + peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' 'template/generic instantiation from here'.*" pegOtherError = peg"'Error:' \s* {.*}" pegSuccess = peg"'Hint: operation successful'.*" pegOfInterest = pegLineError / pegOtherError @@ -64,6 +68,7 @@ proc callCompiler(cmdTemplate, filename, options: string, let outp = p.outputStream var suc = "" var err = "" + var tmpl = "" var x = newStringOfCap(120) result.nimout = "" while outp.readLine(x.TaintedString) or running(p): @@ -71,22 +76,53 @@ proc callCompiler(cmdTemplate, filename, options: string, if x =~ pegOfInterest: # `err` should contain the last error/warning message err = x + elif x =~ pegLineTemplate and err == "": + # `tmpl` contains the last template expansion before the error + tmpl = x elif x =~ pegSuccess: suc = x close(p) result.msg = "" result.file = "" result.outp = "" - result.line = -1 + result.line = 0 + result.column = 0 + result.tfile = "" + result.tline = 0 + result.tcolumn = 0 + if tmpl =~ pegLineTemplate: + result.tfile = extractFilename(matches[0]) + result.tline = parseInt(matches[1]) + result.tcolumn = parseInt(matches[2]) if err =~ pegLineError: result.file = extractFilename(matches[0]) result.line = parseInt(matches[1]) - result.msg = matches[2] + result.column = parseInt(matches[2]) + result.msg = matches[3] elif err =~ pegOtherError: result.msg = matches[0] elif suc =~ pegSuccess: result.err = reSuccess +proc callCCompiler(cmdTemplate, filename, options: string, + target: TTarget): TSpec = + let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], + "options", options, "file", filename.quoteShell]) + var p = startProcess(command="gcc", args=c[5.. ^1], + options={poStdErrToStdOut, poUsePath}) + let outp = p.outputStream + var x = newStringOfCap(120) + result.nimout = "" + result.msg = "" + result.file = "" + result.outp = "" + result.line = -1 + while outp.readLine(x.TaintedString) or running(p): + result.nimout.add(x & "\n") + close(p) + if p.peekExitCode == 0: + result.err = reSuccess + proc initResults: TResults = result.total = 0 result.passed = 0 @@ -115,23 +151,38 @@ proc addResult(r: var TResults, test: TTest, expected = expected, given = given) r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) - if success == reIgnored: - styledEcho styleBright, name, fgYellow, " [", $success, "]" - elif success != reSuccess: - styledEcho styleBright, name, fgRed, " [", $success, "]" - echo"Expected:" - styledEcho styleBright, expected - echo"Given:" - styledEcho styleBright, given + if success == reSuccess: + styledEcho fgGreen, "PASS: ", fgCyan, name + elif success == reIgnored: + styledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name + else: + styledEcho styleBright, fgRed, "FAIL: ", fgCyan, name + styledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" + styledEcho styleBright, fgRed, "Failure: ", $success + styledEcho fgYellow, "Expected:" + styledEcho styleBright, expected, "\n" + styledEcho fgYellow, "Gotten:" + styledEcho styleBright, given, "\n" proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) = if strip(expected.msg) notin strip(given.msg): r.addResult(test, expected.msg, given.msg, reMsgsDiffer) - elif extractFilename(expected.file) != extractFilename(given.file) and + elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: r.addResult(test, expected.file, given.file, reFilesDiffer) - elif expected.line != given.line and expected.line != 0: - r.addResult(test, $expected.line, $given.line, reLinesDiffer) + elif expected.line != given.line and expected.line != 0 or + expected.column != given.column and expected.column != 0: + r.addResult(test, $expected.line & ':' & $expected.column, + $given.line & ':' & $given.column, + reLinesDiffer) + elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and + "internal error:" notin expected.msg: + r.addResult(test, expected.tfile, given.tfile, reFilesDiffer) + elif expected.tline != given.tline and expected.tline != 0 or + expected.tcolumn != given.tcolumn and expected.tcolumn != 0: + r.addResult(test, $expected.tline & ':' & $expected.tcolumn, + $given.tline & ':' & $given.tcolumn, + reLinesDiffer) else: r.addResult(test, expected.msg, given.msg, reSuccess) inc(r.passed) @@ -183,72 +234,126 @@ proc compilerOutputTests(test: TTest, given: var TSpec, expected: TSpec; expectedmsg = expected.nimout givenmsg = given.nimout.strip nimoutCheck(test, expectedmsg, given) + else: + givenmsg = given.nimout.strip if given.err == reSuccess: inc(r.passed) r.addResult(test, expectedmsg, givenmsg, given.err) +proc analyzeAndConsolidateOutput(s: string): string = + result = "" + let rows = s.splitLines + for i in 0 ..< rows.len: + if (let pos = find(rows[i], "Traceback (most recent call last)"); pos != -1): + result = substr(rows[i], pos) & "\n" + for i in i+1 ..< rows.len: + result.add rows[i] & "\n" + if not (rows[i] =~ re"^[^(]+\(\d+\)\s+"): + return + elif (let pos = find(rows[i], "SIGSEGV: Illegal storage access."); pos != -1): + result = substr(rows[i], pos) + return + proc testSpec(r: var TResults, test: TTest) = # major entry point for a single test let tname = test.name.addFileExt(".nim") inc(r.total) - styledEcho "Processing ", fgCyan, extractFilename(tname) - var expected = parseSpec(tname) + var expected: TSpec + if test.action != actionRunNoSpec: + expected = parseSpec(tname) + else: + specDefaults expected + expected.action = actionRunNoSpec + if expected.err == reIgnored: r.addResult(test, "", "", reIgnored) inc(r.skipped) - else: - case expected.action - of actionCompile: - var given = callCompiler(expected.cmd, test.name, - test.options & " --hint[Path]:off --hint[Processing]:off", test.target) - compilerOutputTests(test, given, expected, r) - of actionRun: - var given = callCompiler(expected.cmd, test.name, test.options, - test.target) - if given.err != reSuccess: - r.addResult(test, "", given.msg, given.err) - else: - var exeFile: string - if test.target == targetJS: - let (dir, file, ext) = splitFile(tname) - exeFile = dir / "nimcache" / file & ".js" - else: - exeFile = changeFileExt(tname, ExeExt) - if existsFile(exeFile): - let nodejs = findNodeJs() - if test.target == targetJS and nodejs == "": - r.addResult(test, expected.outp, "nodejs binary not in PATH", - reExeNotFound) - return - var (buf, exitCode) = execCmdEx( - (if test.target == targetJS: nodejs & " " else: "") & exeFile) - if exitCode != expected.exitCode: - r.addResult(test, "exitcode: " & $expected.exitCode, - "exitcode: " & $exitCode, reExitCodesDiffer) - else: - var bufB = strip(buf.string) - if expected.sortoutput: bufB = makeDeterministic(bufB) - if bufB != strip(expected.outp): - if not (expected.substr and expected.outp in bufB): - given.err = reOutputsDiffer - compilerOutputTests(test, given, expected, r) - else: - r.addResult(test, expected.outp, "executable not found", reExeNotFound) - of actionReject: - var given = callCompiler(expected.cmd, test.name, test.options, - test.target) - cmpMsgs(r, expected, given, test) + return + + case expected.action + of actionCompile: + var given = callCompiler(expected.cmd, test.name, + test.options & " --hint[Path]:off --hint[Processing]:off", test.target) + compilerOutputTests(test, given, expected, r) + of actionRun, actionRunNoSpec: + # In this branch of code "early return" pattern is clearer than deep + # nested conditionals - the empty rows in between to clarify the "danger" + var given = callCompiler(expected.cmd, test.name, test.options, + test.target) + + if given.err != reSuccess: + r.addResult(test, "", given.msg, given.err) + return + + let isJsTarget = test.target == targetJS + var exeFile: string + if isJsTarget: + let (dir, file, ext) = splitFile(tname) + exeFile = dir / "nimcache" / file & ".js" # *TODO* hardcoded "nimcache" + else: + exeFile = changeFileExt(tname, ExeExt) + + if not existsFile(exeFile): + r.addResult(test, expected.outp, "executable not found", reExeNotFound) + return + + let nodejs = if isJsTarget: findNodeJs() else: "" + if isJsTarget and nodejs == "": + r.addResult(test, expected.outp, "nodejs binary not in PATH", + reExeNotFound) + return + + let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile + var (buf, exitCode) = execCmdEx(exeCmd, options = {poStdErrToStdOut}) + let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string)) + else: strip(buf.string) + let expectedOut = strip(expected.outp) + + if exitCode != expected.exitCode: + r.addResult(test, "exitcode: " & $expected.exitCode, + "exitcode: " & $exitCode & "\n\nOutput:\n" & + analyzeAndConsolidateOutput(bufB), + reExitCodesDiffer) + return + + if bufB != expectedOut: + if not (expected.substr and expectedOut in bufB): + given.err = reOutputsDiffer + r.addResult(test, expected.outp, bufB, reOutputsDiffer) + return + + compilerOutputTests(test, given, expected, r) + return + + of actionReject: + var given = callCompiler(expected.cmd, test.name, test.options, + test.target) + cmpMsgs(r, expected, given, test) + return proc testNoSpec(r: var TResults, test: TTest) = # does not extract the spec because the file is not supposed to have any let tname = test.name.addFileExt(".nim") inc(r.total) - styledEcho "Processing ", fgCyan, extractFilename(tname) let given = callCompiler(cmdTemplate, test.name, test.options, test.target) r.addResult(test, "", given.msg, given.err) if given.err == reSuccess: inc(r.passed) +proc testC(r: var TResults, test: TTest) = + # runs C code. Doesn't support any specs, just goes by exit code. + let tname = test.name.addFileExt(".c") + inc(r.total) + styledEcho "Processing ", fgCyan, extractFilename(tname) + var given = callCCompiler(cmdTemplate, test.name & ".c", test.options, test.target) + if given.err != reSuccess: + r.addResult(test, "", given.msg, given.err) + elif test.action == actionRun: + let exeFile = changeFileExt(test.name, ExeExt) + var (buf, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUseShell}) + if exitCode != 0: given.err = reExitCodesDiffer + if given.err == reSuccess: inc(r.passed) + proc makeTest(test, options: string, cat: Category, action = actionCompile, - target = targetC): TTest = + target = targetC, env: string = ""): TTest = # start with 'actionCompile', will be overwritten in the spec: result = TTest(cat: cat, name: test, options: options, target: target, action: action) @@ -267,12 +372,14 @@ proc main() = backend.open() var optPrintResults = false var optFailing = false + var optPedantic = false var p = initOptParser() p.next() while p.kind == cmdLongoption: case p.key.string.normalize of "print", "verbose": optPrintResults = true of "failing": optFailing = true + of "pedantic": optPedantic = true else: quit Usage p.next() if p.kind != cmdArgument: quit Usage @@ -293,6 +400,11 @@ proc main() = var cat = Category(p.key) p.next processCategory(r, cat, p.cmdLineRest.string) + of "r", "run": + let (dir, file) = splitPath(p.key.string) + let (_, subdir) = splitPath(dir) + var cat = Category(subdir) + processCategory(r, cat, p.cmdLineRest.string, file) of "html": var commit = 0 discard parseInt(p.cmdLineRest.string, commit) @@ -305,8 +417,10 @@ proc main() = if action == "html": openDefaultBrowser(resultsFile) else: echo r, r.data backend.close() + if optPedantic: + var failed = r.total - r.passed - r.skipped + if failed > 0 : quit(QuitFailure) if paramCount() == 0: quit Usage main() - |