diff options
Diffstat (limited to 'tests/tester.nim')
| -rwxr-xr-x | tests/tester.nim | 114 |
1 files changed, 75 insertions, 39 deletions
diff --git a/tests/tester.nim b/tests/tester.nim index 8156dab7f..60c84403e 100755 --- a/tests/tester.nim +++ b/tests/tester.nim @@ -1,7 +1,7 @@ # # # Nimrod Tester -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -25,22 +25,35 @@ const type TTestAction = enum actionCompile, actionRun, actionReject - TSpec {.pure.} = object + TResultEnum = enum + reNimrodcCrash, # nimrod compiler seems to have crashed + reMsgsDiffer, # error messages differ + reFilesDiffer, # expected and given filenames differ + reLinesDiffer, # expected and given line numbers differ + reOutputsDiffer, + reExitcodesDiffer, + reInvalidPeg, + reCodegenFailure, + reCodeNotFound, + reExeNotFound, + reIgnored, # test is ignored + reSuccess # test was successful + TTarget = enum + targetC, targetCpp, targetObjC, targetJS + + TSpec = object action: TTestAction file, cmd: string outp: string line, exitCode: int msg: string - err: bool - disabled: bool + ccodeCheck: string + err: TResultEnum substr: bool - TResults {.pure.} = object + TResults = object total, passed, skipped: int data: string - TResultEnum = enum - reFailure, reIgnored, reSuccess - # ----------------------- Spec parser ---------------------------------------- when not defined(parseCfgBool): @@ -82,9 +95,9 @@ template parseSpecAux(fillResult: stmt) {.immediate.} = proc parseSpec(filename: string): TSpec = result.file = filename - result.err = true result.msg = "" result.outp = "" + result.ccodeCheck = "" result.cmd = cmdTemplate parseSpecAux: case normalize(e.key) @@ -103,8 +116,10 @@ proc parseSpec(filename: string): TSpec = of "exitcode": discard parseInt(e.value, result.exitCode) of "errormsg", "msg": result.msg = e.value - of "disabled": result.disabled = parseCfgBool(e.value) + of "disabled": + if parseCfgBool(e.value): result.err = reIgnored of "cmd": result.cmd = e.value + of "ccodecheck": result.ccodeCheck = e.value else: echo ignoreMsg(p, e) # ---------------------------------------------------------------------------- @@ -134,7 +149,6 @@ proc callCompiler(cmdTemplate, filename, options: string): TSpec = result.msg = "" result.file = "" result.outp = "" - result.err = true result.line = -1 if err =~ pegLineError: result.file = extractFilename(matches[0]) @@ -143,7 +157,7 @@ proc callCompiler(cmdTemplate, filename, options: string): TSpec = elif err =~ pegOtherError: result.msg = matches[0] elif suc =~ pegSuccess: - result.err = false + result.err = reSuccess proc initResults: TResults = result.total = 0 @@ -164,9 +178,9 @@ proc `$`(x: TResults): string = proc colorResult(r: TResultEnum): string = case r - of reFailure: result = "<span style=\"color:red\">no</span>" of reIgnored: result = "<span style=\"color:fuchsia\">ignored</span>" of reSuccess: result = "<span style=\"color:green\">yes</span>" + else: result = "<span style=\"color:red\">no</span>" const TableHeader4 = "<table border=\"1\"><tr><td>Test</td><td>Expected</td>" & @@ -217,12 +231,12 @@ proc listResults(reject, compile, run: TResults) = proc cmpMsgs(r: var TResults, expected, given: TSpec, test: string) = if strip(expected.msg) notin strip(given.msg): - r.addResult(test, expected.msg, given.msg, reFailure) + r.addResult(test, expected.msg, given.msg, reMsgsDiffer) elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: - r.addResult(test, expected.file, given.file, reFailure) + r.addResult(test, expected.file, given.file, reFilesDiffer) elif expected.line != given.line and expected.line != 0: - r.addResult(test, $expected.line, $given.line, reFailure) + r.addResult(test, $expected.line, $given.line, reLinesDiffer) else: r.addResult(test, expected.msg, given.msg, reSuccess) inc(r.passed) @@ -233,7 +247,7 @@ proc rejectSingleTest(r: var TResults, test, options: string) = inc(r.total) echo t var expected = parseSpec(test) - if expected.disabled: + if expected.err == reIgnored: r.addResult(t, "", "", reIgnored) inc(r.skipped) else: @@ -244,31 +258,47 @@ proc reject(r: var TResults, dir, options: string) = ## handle all the tests that the compiler should reject for test in os.walkFiles(dir / "t*.nim"): rejectSingleTest(r, test, options) +proc codegenCheck(test, check, ext: string, given: var TSpec) = + if check.len > 0: + try: + let (path, name, ext2) = test.splitFile + echo path / "nimcache" / name.changeFileExt(ext) + let contents = readFile(path / "nimcache" / name.changeFileExt(ext)).string + if contents.find(check.peg) < 0: + given.err = reCodegenFailure + except EInvalidValue: + given.err = reInvalidPeg + except EIO: + given.err = reCodeNotFound + +proc codegenChecks(test: string, expected: TSpec, given: var TSpec) = + codegenCheck(test, expected.ccodeCheck, ".c", given) + proc compile(r: var TResults, pattern, options: string) = for test in os.walkFiles(pattern): let t = extractFilename(test) echo t inc(r.total) let expected = parseSpec(test) - if expected.disabled: + if expected.err == reIgnored: r.addResult(t, "", reIgnored) inc(r.skipped) else: var given = callCompiler(expected.cmd, test, options) - r.addResult(t, given.msg, if given.err: reFailure else: reSuccess) - if not given.err: inc(r.passed) + if given.err == reSuccess: + codegenChecks(test, expected, given) + r.addResult(t, given.msg, given.err) + if given.err == reSuccess: inc(r.passed) proc compileSingleTest(r: var TResults, test, options: string) = + # does not extract the spec because the file is not supposed to have any let test = test.addFileExt(".nim") let t = extractFilename(test) inc(r.total) echo t let given = callCompiler(cmdTemplate, test, options) - r.addResult(t, given.msg, if given.err: reFailure else: reSuccess) - if not given.err: inc(r.passed) - -type - TTarget = enum targetC, targetJS + r.addResult(t, given.msg, given.err) + if given.err == reSuccess: inc(r.passed) proc runSingleTest(r: var TResults, test, options: string, target: TTarget) = var test = test.addFileExt(".nim") @@ -276,13 +306,13 @@ proc runSingleTest(r: var TResults, test, options: string, target: TTarget) = echo t inc(r.total) var expected = parseSpec(test) - if expected.disabled: + if expected.err == reIgnored: r.addResult(t, "", "", reIgnored) inc(r.skipped) else: var given = callCompiler(expected.cmd, test, options) - if given.err: - r.addResult(t, "", given.msg, reFailure) + if given.err != reSuccess: + r.addResult(t, "", given.msg, given.err) else: var exeFile: string if target == targetC: @@ -296,25 +326,26 @@ proc runSingleTest(r: var TResults, test, options: string, target: TTarget) = (if target==targetJS: "node " else: "") & exeFile) if exitCode != expected.ExitCode: r.addResult(t, "exitcode: " & $expected.ExitCode, - "exitcode: " & $exitCode, reFailure) + "exitcode: " & $exitCode, reExitCodesDiffer) else: - var success = strip(buf.string) == strip(expected.outp) - if expected.substr and not success: - success = expected.outp in buf.string - if success: inc(r.passed) - r.addResult(t, expected.outp, - buf.string, if success: reSuccess else: reFailure) + if strip(buf.string) != strip(expected.outp): + if not (expected.substr and expected.outp in buf.string): + given.err = reOutputsDiffer + if given.err == reSuccess: + codeGenChecks(test, expected, given) + if given.err == reSuccess: inc(r.passed) + r.addResult(t, expected.outp, buf.string, given.err) else: - r.addResult(t, expected.outp, "executable not found", reFailure) + r.addResult(t, expected.outp, "executable not found", reExeNotFound) proc runSingleTest(r: var TResults, test, options: string) = runSingleTest(r, test, options, targetC) - + proc run(r: var TResults, dir, options: string) = for test in os.walkFiles(dir / "t*.nim"): runSingleTest(r, test, options) include specials - + proc compileExample(r: var TResults, pattern, options: string) = for test in os.walkFiles(pattern): compileSingleTest(r, test, options) @@ -392,7 +423,12 @@ proc main() = runDLLTests r, p.cmdLineRest.string of "gc": runGCTests(r, p.cmdLineRest.string) - of "test", "comp", "rej": + of "test": + if p.kind != cmdArgument: quit usage + var testFile = p.key.string + p.next() + runSingleTest(r, testFile, p.cmdLineRest.string) + of "comp", "rej": if p.kind != cmdArgument: quit usage var testFile = p.key.string p.next() |