diff options
author | Andreas Rumpf <andreas@andreas-desktop> | 2010-03-02 21:39:23 +0100 |
---|---|---|
committer | Andreas Rumpf <andreas@andreas-desktop> | 2010-03-02 21:39:23 +0100 |
commit | 6c08ba8290d8ddebbbf3b327401a6e3087c07188 (patch) | |
tree | 01302b6a55b4823f20f1a60ff83160f20fdfc1c7 /tests | |
parent | d83b170e7fffae08b4914e00b74a86c16c645890 (diff) | |
download | Nim-6c08ba8290d8ddebbbf3b327401a6e3087c07188.tar.gz |
fixed colors module
Diffstat (limited to 'tests')
-rwxr-xr-x | tests/tester.nim | 257 |
1 files changed, 116 insertions, 141 deletions
diff --git a/tests/tester.nim b/tests/tester.nim index f15f015b5..75ad5f27e 100755 --- a/tests/tester.nim +++ b/tests/tester.nim @@ -8,73 +8,38 @@ # ## This program verifies Nimrod against the testcases. -## The testcases may contain the directives '#ERROR' or '#ERROR_IN'. -## '#ERROR' is used to indicate that the compiler should report -## an error in the marked line (the line that contains the '#ERROR' -## directive.) -## The format for '#ERROR_IN' is: -## #ERROR_IN filename linenumber -## One can omit the extension of the filename ('.nim' is then used). -## Tests which contain none of the two directives should compile. Thus they -## are executed after successful compilation and their output is verified -## against the results specified with the '#OUT' directive. -## (Tests which require user interaction are not possible.) -## Tests can have an #ERROR_MSG directive specifiying the error message -## Nimrod shall produce. import - strutils, pegs, os, osproc, streams, parsecsv + strutils, pegs, os, osproc, streams, parsecsv, browsers const - cmdTemplate = r"nimrod cc --hints:on $options $filename" + cmdTemplate = r"nimrod cc --hints:on $# $#" type - TSpec = object of TObject ## specification object - line: int ## line number where compiler should throw an error - file: string ## file where compiler should throw an error - err: bool ## true if the specification says there should be an error - outp: string ## output that should be produced - puremsg: string ## pure message of compiler + TMsg = tuple[ + file: string, + line: int, + msg: string, + err: bool] + TOutp = tuple[file, outp: string] + TResult = tuple[test, expected, given: string, success: bool] + TResults = object + total, passed: int + data: seq[TResult] proc myExec(cmd: string): string = #echo("Executing: " & cmd) result = osproc.execProcess(cmd) #echo("Received: " & result) -proc parseTest(filename: string): TSpec = - var i = 0 # the line counter - var matches: array [0..2, string] - result.outp = "" - result.puremsg = "" - result.file = filename - for s in lines(filename): - inc(i) - if contains(s, peg"'#OUT' \s+ {.*}", matches): - result.outp = matches[0] - break - if contains(s, peg"'#ERROR_IN' \s* {\S*} \s* {\d+}", matches): - result.file = matches[0] - result.line = parseInt(matches[1]) - result.err = true - break - if contains(s, peg"'#ERROR_MSG' \s* {.*}", matches): - result.line = i - result.outp = matches[0] - result.err = True - break - if contains(s, peg"'#ERROR' \s* !.", matches): - result.line = i - result.err = true - break - var pegLineError = peg"{[^(]*} '(' {\d+} ', ' \d+ ') Error:' \s* {.*}" pegOtherError = peg"'Error:' \s* {.*}" pegSuccess = peg"'Hint: operation successful'.*" pegOfInterest = pegLineError / pegOtherError / pegSuccess -proc callCompiler(filename, options: string): TSpec = - var c = parseCmdLine(cmdTemplate % ["filename", filename, "options", options]) +proc callCompiler(filename, options: string): TMsg = + var c = parseCmdLine(cmdTemplate % [options, filename]) var a: seq[string] = @[] # slicing is not yet implemented :-( for i in 1 .. c.len-1: add(a, c[i]) var p = startProcess(command=c[0], args=a, @@ -86,112 +51,122 @@ proc callCompiler(filename, options: string): TSpec = if x =~ pegOfInterest: # `s` should contain the last error message s = x - result.outp = "" - result.puremsg = "" + result.msg = "" result.file = "" - result.err = true + result.err = true + result.line = -1 if s =~ pegLineError: - result.file = matches[0] + result.file = extractFilename(matches[0]) result.line = parseInt(matches[1]) - result.outp = s - result.puremsg = matches[2] + result.msg = matches[2] elif s =~ pegOtherError: - result.puremsg = matches[0] - result.outp = s - result.line = 1 + result.msg = matches[0] elif s =~ pegSuccess: - result.outp = s result.err = false -proc sameResults(filename: string, spec, comp: TSpec): bool = - # short filename for messages (better readability): - var shortfile = os.extractFilename(filename) - - if comp.err and comp.outp == "": - # the compiler did not say "[Error]" nor "operation sucessful" - Echo("[Tester] $# -- FAILED; COMPILER BROKEN" % shortfile) - elif spec.err != comp.err: - Echo(("[Tester] $# -- FAILED\n" & - "Compiler says: $#\n" & - "But specification says: $#") % - [shortfile, comp.outp, spec.outp]) - elif spec.err: - if extractFilename(comp.file) != extractFilename(spec.file): - Echo(("[Tester] $# -- FAILED: file names do not match:\n" & - "Compiler: $#\nSpec: $#") % [shortfile, comp.file, spec.file]) - elif strip(spec.outp) notin strip(comp.puremsg): - Echo(("[Tester] $# -- FAILED: error messages do not match:\n" & - "Compiler: $#\nSpec: $#") % [shortfile, comp.pureMsg, spec.outp]) - elif comp.line != spec.line: - Echo(("[Tester] $# -- FAILED: line numbers do not match:\n" & - "Compiler: $#\nSpec: $#") % [shortfile, $comp.line, $spec.line]) - else: - Echo("[Tester] $# -- OK" % shortfile) - result = true - else: - # we have to run the executable and check its output: - var exeFile = changeFileExt(filename, ExeExt) - if ExistsFile(exeFile): - if len(spec.outp) == 0: - # we have no output to validate against, but compilation succeeded, - # so it's okay: - Echo("[Tester] $# -- OK" % shortfile) - result = true - else: - var buf = myExec(exeFile) - result = strip(buf) == strip(spec.outp) - if result: - Echo("[Tester] $# -- compiled program OK" % shortfile) - else: - Echo("[Tester] $# -- compiled program FAILED" % shortfile) - else: - Echo("[Tester] $# -- FAILED; executable not found" % shortfile) - -proc main(options: string) = - # runs the complete testsuite - var total = 0 - var passed = 0 - for filename in os.walkFiles("tests/t*.nim"): - if extractFilename(filename) == "tester.nim": continue - var spec = parseTest(filename) - var comp = callCompiler(filename, options) - if sameResults(filename, spec, comp): inc(passed) - inc(total) - # ensure that the examples at least compile - for filename in os.walkFiles("examples/*.nim"): - var comp = callCompiler(filename, options) - var shortfile = os.extractFilename(filename) - if comp.err: - Echo("[Tester] Example '$#' -- FAILED" % shortfile) - else: - Echo("[Tester] Example $# -- OK" % shortfile) - inc(passed) - inc(total) - Echo("[Tester] $#/$# tests passed\n" % [$passed, $total]) - -proc reject(options: string) = - ## handle all the tests that the compiler should reject - const csvFile = "tests/reject/spec.csv" - var p: TCsvParser - +proc setupCvsParser(cvsFile: string): TCsvParser = var s = newFileStream(csvFile, fmRead) if s == nil: quit("cannot open the file" & csvFile) - p.open(s, csvFile, separator=';', skipInitialSpace=true) - while readRow(p): - for val in items(x.row): - Echo "##", val, "##" + result.open(s, csvFile, separator=';', skipInitialSpace=true) + +proc parseRejectData(dir: string): seq[TMsg] = + var p = setupCvsParser(dir / "spec.csv") + result = @[] + while readRow(p, 3): + result.add((p.row[0], parseInt(p.row[1]), p.row[2], true)) close(p) + +proc parseRunData(dir: string): seq[TOutp] = + var p = setupCvsParser(dir / "spec.csv") + result = @[] + while readRow(p, 2): + result.add((p.row[0], p.row[1])) + close(p) + +proc findSpec[T](specs: seq[T], filename: string): int = + for s in items(specs): + if s.file == filename: return + inc(result) + quit("cannot find spec for file: " & filename) + +proc initResults: TResults = + result.total = 0 + result.passed = 0 + result.data = @[] + +proc colorBool(b: bool): string = + if b: result = "<span color:\"green\">yes</span>" + else: result = "<span color:\"red\">no</span>" + +proc `$`(r: TResults): string = + result = "<table><tr><td>Test</td><td>Expected</td>" & + "<td>Given</td><td>Success</td></tr>\n" + for test, expected, given, success in items(r.data): + result.add("<tr><td>$#</td><td>$#</td><td>$#</td><td>$#</td></tr>\n" % [ + test, expected, given, success.colorBool]) + result.add("</table>\n") + +proc cmpMsgs(r: var TResults, expected, given: TMsg, test: string) = + inc(r.total) + if strip(expected.msg) notin strip(given.msg): + r.data.add((test, expected.msg, given.msg, false)) + elif expected.file != given.file: + r.data.add((test, expected.file, given.file, false)) + elif expected.line != given.line: + r.data.add((test, $expected.line, $given.line, false)) + else: + r.data.add((test, expected.msg, given.msg, true)) + inc(r.passed) + +proc reject(r: var TResults, dir, options: string) = + ## handle all the tests that the compiler should reject + var specs = parseRejectData(dir) + for test in os.walkFiles(dir / "t*.nim"): + var t = extractFilename(test) + var expected = findSpec(specs, t) + var given = callCompiler(test, options) + cmpMsgs(r, specs[expected], given, t) -proc accept(options: string) = - nil - +proc compile(r: var TResults, pattern, options: string) = + for test in os.walkFiles(pattern): + var t = extractFilename(test) + inc(r.total) + var given = callCompiler(test, options) + r.data.add((t, "", given.msg, not given.err)) + if not given.err: inc(r.passed) +proc run(r: var TResults, dir, options: string) = + var specs = parseRunData(dir) + for test in os.walkFiles(dir / "t*.nim"): + var t = extractFilename(test) + inc(r.total) + var given = callCompiler(test, options) + if given.err: + r.data.add((t, "", given.msg, not given.err)) + else: + var exeFile = changeFileExt(test, ExeExt) + var expected = specs[findSpec(specs, t)] + if existsFile(exeFile): + var buf = myExec(exeFile) + var success = strip(buf) == strip(spec.outp) + if success: inc(r.passed) + r.data.add((t, spec.outp, buf, success)) + else: + r.data.add((t, spec.outp, "executable not found", false)) - -var - options = "" +var options = "" +var rejectRes = initResults() +var compileRes = initResults() +var runRes = initResults() + for i in 1.. paramCount(): add(options, " ") add(options, paramStr(i)) -main(options) + +reject(rejectRes, "tests/reject", options) +compile(compileRes, "tests/accept/compile/t*.nim", options) +compile(compileRes, "examples/*.nim", options) +compile(compileRes, "examples/gtk/*.nim", options) +run(runRes, "tests/accept/run", options) + |