diff options
Diffstat (limited to 'tests/tester.nim')
| -rw-r--r-- | tests/tester.nim | 457 |
1 files changed, 0 insertions, 457 deletions
diff --git a/tests/tester.nim b/tests/tester.nim deleted file mode 100644 index 0e125b1bb..000000000 --- a/tests/tester.nim +++ /dev/null @@ -1,457 +0,0 @@ -# -# -# Nimrod Tester -# (c) Copyright 2013 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This program verifies Nimrod against the testcases. - -import - parseutils, strutils, pegs, os, osproc, streams, parsecfg, browsers, json, - marshal, cgi, parseopt #, caas - -const - cmdTemplate = r"nimrod cc --hints:on $# $#" - resultsFile = "testresults.html" - jsonFile = "testresults.json" - Usage = "usage: tester [--print] " & - "reject|compile|run|" & - "merge|special|rodfiles| [nimrod options]\n" & - " or: tester test|comp|rej singleTest" - -type - TTestAction = enum - actionCompile, actionRun, actionReject - TResultEnum = enum - reNimrodcCrash, # nimrod compiler seems to have crashed - reMsgsDiffer, # error messages differ - reFilesDiffer, # expected and given filenames differ - reLinesDiffer, # expected and given line numbers differ - reOutputsDiffer, - reExitcodesDiffer, - reInvalidPeg, - reCodegenFailure, - reCodeNotFound, - reExeNotFound, - reIgnored, # test is ignored - reSuccess # test was successful - - TTarget = enum - targetC, targetCpp, targetObjC, targetJS - - TSpec = object - action: TTestAction - file, cmd: string - outp: string - line, exitCode: int - msg: string - ccodeCheck: string - err: TResultEnum - substr: bool - TResults = object - total, passed, skipped: int - data: string - -# ----------------------- Spec parser ---------------------------------------- - -when not defined(parseCfgBool): - # candidate for the stdlib: - proc parseCfgBool(s: string): bool = - case normalize(s) - of "y", "yes", "true", "1", "on": result = true - of "n", "no", "false", "0", "off": result = false - else: raise newException(EInvalidValue, "cannot interpret as a bool: " & s) - -proc extractSpec(filename: string): string = - const tripleQuote = "\"\"\"" - var x = readFile(filename).string - var a = x.find(tripleQuote) - var b = x.find(tripleQuote, a+3) - # look for """ only in the first section - if a >= 0 and b > a and a < 40: - result = x.substr(a+3, b-1).replace("'''", tripleQuote) - else: - #echo "warning: file does not contain spec: " & filename - result = "" - -when not defined(nimhygiene): - {.pragma: inject.} - -template parseSpecAux(fillResult: stmt) {.immediate.} = - var ss = newStringStream(extractSpec(filename)) - var p {.inject.}: TCfgParser - open(p, ss, filename, 1) - while true: - var e {.inject.} = next(p) - case e.kind - of cfgEof: break - of cfgSectionStart, cfgOption, cfgError: - echo ignoreMsg(p, e) - of cfgKeyValuePair: - fillResult - close(p) - -proc parseSpec(filename: string): TSpec = - result.file = filename - result.msg = "" - result.outp = "" - result.ccodeCheck = "" - result.cmd = cmdTemplate - parseSpecAux: - case normalize(e.key) - of "action": - case e.value.normalize - of "compile": result.action = actionCompile - of "run": result.action = actionRun - of "reject": result.action = actionReject - else: echo ignoreMsg(p, e) - of "file": result.file = e.value - of "line": discard parseInt(e.value, result.line) - of "output": result.outp = e.value - of "outputsub": - result.outp = e.value - result.substr = true - of "exitcode": - discard parseInt(e.value, result.exitCode) - of "errormsg", "msg": result.msg = e.value - of "disabled": - if parseCfgBool(e.value): result.err = reIgnored - of "cmd": result.cmd = e.value - of "ccodecheck": result.ccodeCheck = e.value - else: echo ignoreMsg(p, e) - -# ---------------------------------------------------------------------------- - -let - pegLineError = - peg"{[^(]*} '(' {\d+} ', ' \d+ ') ' ('Error'/'Warning') ':' \s* {.*}" - pegOtherError = peg"'Error:' \s* {.*}" - pegSuccess = peg"'Hint: operation successful'.*" - pegOfInterest = pegLineError / pegOtherError - -proc callCompiler(cmdTemplate, filename, options: string): TSpec = - let c = parseCmdLine(cmdTemplate % [options, filename]) - var p = startProcess(command=c[0], args=c[1.. -1], - options={poStdErrToStdOut, poUseShell}) - let outp = p.outputStream - var suc = "" - var err = "" - var x = newStringOfCap(120) - while outp.readLine(x.TaintedString) or running(p): - if x =~ pegOfInterest: - # `err` should contain the last error/warning message - err = x - elif x =~ pegSuccess: - suc = x - close(p) - result.msg = "" - result.file = "" - result.outp = "" - result.line = -1 - if err =~ pegLineError: - result.file = extractFilename(matches[0]) - result.line = parseInt(matches[1]) - result.msg = matches[2] - elif err =~ pegOtherError: - result.msg = matches[0] - elif suc =~ pegSuccess: - result.err = reSuccess - -proc initResults: TResults = - result.total = 0 - result.passed = 0 - result.skipped = 0 - result.data = "" - -proc readResults(filename: string): TResults = - result = marshal.to[TResults](readFile(filename).string) - -proc writeResults(filename: string, r: TResults) = - writeFile(filename, $$r) - -proc `$`(x: TResults): string = - result = ("Tests passed: $1 / $3 <br />\n" & - "Tests skipped: $2 / $3 <br />\n") % - [$x.passed, $x.skipped, $x.total] - -proc colorResult(r: TResultEnum): string = - case r - of reIgnored: result = "<span style=\"color:fuchsia\">ignored</span>" - of reSuccess: result = "<span style=\"color:green\">yes</span>" - else: result = "<span style=\"color:red\">no</span>" - -const - TableHeader4 = "<table border=\"1\"><tr><td>Test</td><td>Expected</td>" & - "<td>Given</td><td>Success</td></tr>\n" - TableHeader3 = "<table border=\"1\"><tr><td>Test</td>" & - "<td>Given</td><td>Success</td></tr>\n" - TableFooter = "</table>\n" - HtmlBegin = """<html> - <head> - <title>Test results</title> - <style type="text/css"> - <!--""" & slurp("css/boilerplate.css") & "\n" & - slurp("css/style.css") & - """--> - </style> - - </head> - <body>""" - - HtmlEnd = "</body></html>" - -proc td(s: string): string = - result = s.substr(0, 200).XMLEncode - -proc addResult(r: var TResults, test, expected, given: string, - success: TResultEnum) = - r.data.addf("<tr><td>$#</td><td>$#</td><td>$#</td><td>$#</td></tr>\n", [ - XMLEncode(test), td(expected), td(given), success.colorResult]) - -proc addResult(r: var TResults, test, given: string, - success: TResultEnum) = - r.data.addf("<tr><td>$#</td><td>$#</td><td>$#</td></tr>\n", [ - XMLEncode(test), td(given), success.colorResult]) - -proc listResults(reject, compile, run: TResults) = - var s = HtmlBegin - s.add("<h1>Tests to Reject</h1>\n") - s.add($reject) - s.add(TableHeader4 & reject.data & TableFooter) - s.add("<br /><br /><br /><h1>Tests to Compile</h1>\n") - s.add($compile) - s.add(TableHeader3 & compile.data & TableFooter) - s.add("<br /><br /><br /><h1>Tests to Run</h1>\n") - s.add($run) - s.add(TableHeader4 & run.data & TableFooter) - s.add(HtmlEnd) - writeFile(resultsFile, s) - -proc cmpMsgs(r: var TResults, expected, given: TSpec, test: string) = - if strip(expected.msg) notin strip(given.msg): - r.addResult(test, expected.msg, given.msg, reMsgsDiffer) - elif extractFilename(expected.file) != extractFilename(given.file) and - "internal error:" notin expected.msg: - r.addResult(test, expected.file, given.file, reFilesDiffer) - elif expected.line != given.line and expected.line != 0: - r.addResult(test, $expected.line, $given.line, reLinesDiffer) - else: - r.addResult(test, expected.msg, given.msg, reSuccess) - inc(r.passed) - -proc rejectSingleTest(r: var TResults, test, options: string) = - let test = test.addFileExt(".nim") - var t = extractFilename(test) - inc(r.total) - echo t - var expected = parseSpec(test) - if expected.err == reIgnored: - r.addResult(t, "", "", reIgnored) - inc(r.skipped) - else: - var given = callCompiler(expected.cmd, test, options) - cmpMsgs(r, expected, given, t) - -proc reject(r: var TResults, dir, options: string) = - ## handle all the tests that the compiler should reject - for test in os.walkFiles(dir / "t*.nim"): rejectSingleTest(r, test, options) - -proc codegenCheck(test, check, ext: string, given: var TSpec) = - if check.len > 0: - try: - let (path, name, ext2) = test.splitFile - echo path / "nimcache" / name.changeFileExt(ext) - let contents = readFile(path / "nimcache" / name.changeFileExt(ext)).string - if contents.find(check.peg) < 0: - given.err = reCodegenFailure - except EInvalidValue: - given.err = reInvalidPeg - except EIO: - given.err = reCodeNotFound - -proc codegenChecks(test: string, expected: TSpec, given: var TSpec) = - codegenCheck(test, expected.ccodeCheck, ".c", given) - -proc compile(r: var TResults, pattern, options: string) = - for test in os.walkFiles(pattern): - let t = extractFilename(test) - echo t - inc(r.total) - let expected = parseSpec(test) - if expected.err == reIgnored: - r.addResult(t, "", reIgnored) - inc(r.skipped) - else: - var given = callCompiler(expected.cmd, test, options) - if given.err == reSuccess: - codegenChecks(test, expected, given) - r.addResult(t, given.msg, given.err) - if given.err == reSuccess: inc(r.passed) - -proc compileSingleTest(r: var TResults, test, options: string) = - # does not extract the spec because the file is not supposed to have any - let test = test.addFileExt(".nim") - let t = extractFilename(test) - inc(r.total) - echo t - let given = callCompiler(cmdTemplate, test, options) - r.addResult(t, given.msg, given.err) - if given.err == reSuccess: inc(r.passed) - -proc runSingleTest(r: var TResults, test, options: string, target: TTarget) = - var test = test.addFileExt(".nim") - var t = extractFilename(test) - echo t - inc(r.total) - var expected = parseSpec(test) - if expected.err == reIgnored: - r.addResult(t, "", "", reIgnored) - inc(r.skipped) - else: - var given = callCompiler(expected.cmd, test, options) - if given.err != reSuccess: - r.addResult(t, "", given.msg, given.err) - else: - var exeFile: string - if target == targetC: - exeFile = changeFileExt(test, ExeExt) - else: - let (dir, file, ext) = splitFile(test) - exeFile = dir / "nimcache" / file & ".js" - - if existsFile(exeFile): - var (buf, exitCode) = execCmdEx( - (if target==targetJS: "node " else: "") & exeFile) - if exitCode != expected.ExitCode: - r.addResult(t, "exitcode: " & $expected.ExitCode, - "exitcode: " & $exitCode, reExitCodesDiffer) - else: - if strip(buf.string) != strip(expected.outp): - if not (expected.substr and expected.outp in buf.string): - given.err = reOutputsDiffer - if given.err == reSuccess: - codeGenChecks(test, expected, given) - if given.err == reSuccess: inc(r.passed) - r.addResult(t, expected.outp, buf.string, given.err) - else: - r.addResult(t, expected.outp, "executable not found", reExeNotFound) - -proc runSingleTest(r: var TResults, test, options: string) = - runSingleTest(r, test, options, targetC) - -proc run(r: var TResults, dir, options: string) = - for test in os.walkFiles(dir / "t*.nim"): runSingleTest(r, test, options) - -include specials - -proc compileExample(r: var TResults, pattern, options: string) = - for test in os.walkFiles(pattern): compileSingleTest(r, test, options) - -proc toJson(res: TResults): PJsonNode = - result = newJObject() - result["total"] = newJInt(res.total) - result["passed"] = newJInt(res.passed) - result["skipped"] = newJInt(res.skipped) - -proc outputJSON(reject, compile, run: TResults) = - var doc = newJObject() - doc["reject"] = toJson(reject) - doc["compile"] = toJson(compile) - doc["run"] = toJson(run) - var s = pretty(doc) - writeFile(jsonFile, s) - -# proc runCaasTests(r: var TResults) = -# for test, output, status, mode in caasTestsRunner(): -# r.addResult(test, "", output & "-> " & $mode, -# if status: reSuccess else: reOutputsDiffer) - -proc main() = - os.putenv "NIMTEST_NO_COLOR", "1" - os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES" - - const - compileJson = "compile.json" - runJson = "run.json" - rejectJson = "reject.json" - - var optPrintResults = false - var p = initOptParser() - p.next() - if p.kind == cmdLongoption: - case p.key.string - of "print": optPrintResults = true - else: quit usage - p.next() - if p.kind != cmdArgument: quit usage - var action = p.key.string.normalize - p.next() - var r = initResults() - case action - of "reject": - reject(r, "tests/reject", p.cmdLineRest.string) - rejectSpecialTests(r, p.cmdLineRest.string) - writeResults(rejectJson, r) - of "compile": - compile(r, "tests/compile/t*.nim", p.cmdLineRest.string) - compile(r, "tests/ccg/t*.nim", p.cmdLineRest.string) - compile(r, "tests/js.nim", p.cmdLineRest.string) - compileExample(r, "lib/pure/*.nim", p.cmdLineRest.string) - compileExample(r, "examples/*.nim", p.cmdLineRest.string) - compileExample(r, "examples/gtk/*.nim", p.cmdLineRest.string) - compileExample(r, "examples/talk/*.nim", p.cmdLineRest.string) - compileSpecialTests(r, p.cmdLineRest.string) - writeResults(compileJson, r) - of "run": - run(r, "tests/run", p.cmdLineRest.string) - runSpecialTests(r, p.cmdLineRest.string) - writeResults(runJson, r) - of "special": - runSpecialTests(r, p.cmdLineRest.string) - # runCaasTests(r) - writeResults(runJson, r) - of "rodfiles": - runRodFiles(r, p.cmdLineRest.string) - writeResults(runJson, r) - of "js": - if existsFile(runJSon): - r = readResults(runJson) - runJsTests(r, p.cmdLineRest.string) - writeResults(runJson, r) - of "merge": - var rejectRes = readResults(rejectJson) - var compileRes = readResults(compileJson) - var runRes = readResults(runJson) - listResults(rejectRes, compileRes, runRes) - outputJSON(rejectRes, compileRes, runRes) - of "dll": - runDLLTests r, p.cmdLineRest.string - of "gc": - runGCTests(r, p.cmdLineRest.string) - of "test": - if p.kind != cmdArgument: quit usage - var testFile = p.key.string - p.next() - runSingleTest(r, testFile, p.cmdLineRest.string) - of "comp", "rej": - if p.kind != cmdArgument: quit usage - var testFile = p.key.string - p.next() - if peg"'/reject/'" in testFile or action == "rej": - rejectSingleTest(r, testFile, p.cmdLineRest.string) - elif peg"'/compile/'" in testFile or action == "comp": - compileSingleTest(r, testFile, p.cmdLineRest.string) - else: - runSingleTest(r, testFile, p.cmdLineRest.string) - else: - quit usage - - if optPrintResults: echo r, r.data - -if paramCount() == 0: - quit usage -main() - |