summary refs log tree commit diff stats
path: root/tests
diff options
context:
space:
mode:
authorAndreas Rumpf <andreas@andreas-desktop>2010-03-02 21:39:23 +0100
committerAndreas Rumpf <andreas@andreas-desktop>2010-03-02 21:39:23 +0100
commit6c08ba8290d8ddebbbf3b327401a6e3087c07188 (patch)
tree01302b6a55b4823f20f1a60ff83160f20fdfc1c7 /tests
parentd83b170e7fffae08b4914e00b74a86c16c645890 (diff)
downloadNim-6c08ba8290d8ddebbbf3b327401a6e3087c07188.tar.gz
fixed colors module
Diffstat (limited to 'tests')
-rwxr-xr-xtests/tester.nim257
1 files changed, 116 insertions, 141 deletions
diff --git a/tests/tester.nim b/tests/tester.nim
index f15f015b5..75ad5f27e 100755
--- a/tests/tester.nim
+++ b/tests/tester.nim
@@ -8,73 +8,38 @@
 #
 
 ## This program verifies Nimrod against the testcases.
-## The testcases may contain the directives '#ERROR' or '#ERROR_IN'.
-## '#ERROR' is used to indicate that the compiler should report
-## an error in the marked line (the line that contains the '#ERROR'
-## directive.)
-## The format for '#ERROR_IN' is:
-##      #ERROR_IN filename linenumber
-## One can omit the extension of the filename ('.nim' is then used).
-## Tests which contain none of the two directives should compile. Thus they
-## are executed after successful compilation and their output is verified
-## against the results specified with the '#OUT' directive.
-## (Tests which require user interaction are not possible.)
-## Tests can have an #ERROR_MSG directive specifiying the error message
-## Nimrod shall produce.
 
 import
-  strutils, pegs, os, osproc, streams, parsecsv
+  strutils, pegs, os, osproc, streams, parsecsv, browsers
 
 const
-  cmdTemplate = r"nimrod cc --hints:on $options $filename"
+  cmdTemplate = r"nimrod cc --hints:on $# $#"
 
 type
-  TSpec = object of TObject ## specification object
-    line: int    ## line number where compiler should throw an error
-    file: string ## file where compiler should throw an error
-    err: bool    ## true if the specification says there should be an error
-    outp: string ## output that should be produced
-    puremsg: string ## pure message of compiler
+  TMsg = tuple[
+    file: string,
+    line: int,       
+    msg: string,
+    err: bool]
+  TOutp = tuple[file, outp: string]
+  TResult = tuple[test, expected, given: string, success: bool]
+  TResults = object
+    total, passed: int
+    data: seq[TResult]
 
 proc myExec(cmd: string): string =
   #echo("Executing: " & cmd)
   result = osproc.execProcess(cmd)
   #echo("Received: " & result)
 
-proc parseTest(filename: string): TSpec =
-  var i = 0 # the line counter
-  var matches: array [0..2, string]
-  result.outp = ""
-  result.puremsg = ""
-  result.file = filename
-  for s in lines(filename):
-    inc(i)
-    if contains(s, peg"'#OUT' \s+ {.*}", matches):
-      result.outp = matches[0]
-      break
-    if contains(s, peg"'#ERROR_IN' \s* {\S*} \s* {\d+}", matches):
-      result.file = matches[0]
-      result.line = parseInt(matches[1])
-      result.err = true
-      break
-    if contains(s, peg"'#ERROR_MSG' \s* {.*}", matches):
-      result.line = i
-      result.outp = matches[0]
-      result.err = True
-      break
-    if contains(s, peg"'#ERROR' \s* !.", matches):
-      result.line = i
-      result.err = true
-      break
-
 var
   pegLineError = peg"{[^(]*} '(' {\d+} ', ' \d+ ') Error:' \s* {.*}"
   pegOtherError = peg"'Error:' \s* {.*}"
   pegSuccess = peg"'Hint: operation successful'.*"
   pegOfInterest = pegLineError / pegOtherError / pegSuccess
 
-proc callCompiler(filename, options: string): TSpec =
-  var c = parseCmdLine(cmdTemplate % ["filename", filename, "options", options])
+proc callCompiler(filename, options: string): TMsg =
+  var c = parseCmdLine(cmdTemplate % [options, filename])
   var a: seq[string] = @[] # slicing is not yet implemented :-(
   for i in 1 .. c.len-1: add(a, c[i])
   var p = startProcess(command=c[0], args=a,
@@ -86,112 +51,122 @@ proc callCompiler(filename, options: string): TSpec =
     if x =~ pegOfInterest:
       # `s` should contain the last error message
       s = x
-  result.outp = ""
-  result.puremsg = ""
+  result.msg = ""
   result.file = ""
-  result.err = true
+  result.err = true    
+  result.line = -1
   if s =~ pegLineError:
-    result.file = matches[0]
+    result.file = extractFilename(matches[0])
     result.line = parseInt(matches[1])
-    result.outp = s
-    result.puremsg = matches[2]
+    result.msg = matches[2]
   elif s =~ pegOtherError:
-    result.puremsg = matches[0]
-    result.outp = s
-    result.line = 1
+    result.msg = matches[0]
   elif s =~ pegSuccess:
-    result.outp = s
     result.err = false
 
-proc sameResults(filename: string, spec, comp: TSpec): bool =
-  # short filename for messages (better readability):
-  var shortfile = os.extractFilename(filename)
-
-  if comp.err and comp.outp == "":
-    # the compiler did not say "[Error]" nor "operation sucessful"
-    Echo("[Tester] $# -- FAILED; COMPILER BROKEN" % shortfile)
-  elif spec.err != comp.err:
-    Echo(("[Tester] $# -- FAILED\n" &
-         "Compiler says: $#\n" &
-         "But specification says: $#") %
-         [shortfile, comp.outp, spec.outp])
-  elif spec.err:
-    if extractFilename(comp.file) != extractFilename(spec.file):
-      Echo(("[Tester] $# -- FAILED: file names do not match:\n" &
-           "Compiler: $#\nSpec: $#") % [shortfile, comp.file, spec.file])
-    elif strip(spec.outp) notin strip(comp.puremsg):
-      Echo(("[Tester] $# -- FAILED: error messages do not match:\n" &
-           "Compiler: $#\nSpec: $#") % [shortfile, comp.pureMsg, spec.outp])
-    elif comp.line != spec.line:
-      Echo(("[Tester] $# -- FAILED: line numbers do not match:\n" &
-           "Compiler: $#\nSpec: $#") % [shortfile, $comp.line, $spec.line])
-    else:
-      Echo("[Tester] $# -- OK" % shortfile)
-      result = true
-  else:
-    # we have to run the executable and check its output:
-    var exeFile = changeFileExt(filename, ExeExt)
-    if ExistsFile(exeFile):
-      if len(spec.outp) == 0:
-        # we have no output to validate against, but compilation succeeded,
-        # so it's okay:
-        Echo("[Tester] $# -- OK" % shortfile)
-        result = true
-      else:
-        var buf = myExec(exeFile)
-        result = strip(buf) == strip(spec.outp)
-        if result:
-          Echo("[Tester] $# -- compiled program OK" % shortfile)
-        else:
-          Echo("[Tester] $# -- compiled program FAILED" % shortfile)
-    else:
-      Echo("[Tester] $# -- FAILED; executable not found" % shortfile)
-
-proc main(options: string) =
-  # runs the complete testsuite
-  var total = 0
-  var passed = 0
-  for filename in os.walkFiles("tests/t*.nim"):
-    if extractFilename(filename) == "tester.nim": continue
-    var spec = parseTest(filename)
-    var comp = callCompiler(filename, options)
-    if sameResults(filename, spec, comp): inc(passed)
-    inc(total)
-  # ensure that the examples at least compile
-  for filename in os.walkFiles("examples/*.nim"):
-    var comp = callCompiler(filename, options)
-    var shortfile = os.extractFilename(filename)
-    if comp.err:
-      Echo("[Tester] Example '$#' -- FAILED" % shortfile)
-    else:
-      Echo("[Tester] Example $# -- OK" % shortfile)
-      inc(passed)
-    inc(total)
-  Echo("[Tester] $#/$# tests passed\n" % [$passed, $total])
-
-proc reject(options: string) =  
-  ## handle all the tests that the compiler should reject
-  const csvFile = "tests/reject/spec.csv"
-  var p: TCsvParser
-  
+proc setupCvsParser(cvsFile: string): TCsvParser = 
   var s = newFileStream(csvFile, fmRead)
   if s == nil: quit("cannot open the file" & csvFile)
-  p.open(s, csvFile, separator=';', skipInitialSpace=true)
-  while readRow(p):
-    for val in items(x.row):
-      Echo "##", val, "##"
+  result.open(s, csvFile, separator=';', skipInitialSpace=true)
+
+proc parseRejectData(dir: string): seq[TMsg] = 
+  var p = setupCvsParser(dir / "spec.csv")
+  result = @[]
+  while readRow(p, 3):
+    result.add((p.row[0], parseInt(p.row[1]), p.row[2], true))
   close(p)
+
+proc parseRunData(dir: string): seq[TOutp] = 
+  var p = setupCvsParser(dir / "spec.csv")
+  result = @[]
+  while readRow(p, 2):
+    result.add((p.row[0], p.row[1]))
+  close(p)
+
+proc findSpec[T](specs: seq[T], filename: string): int = 
+  for s in items(specs): 
+    if s.file == filename: return
+    inc(result)
+  quit("cannot find spec for file: " & filename)
+
+proc initResults: TResults = 
+  result.total = 0
+  result.passed = 0
+  result.data = @[]
+
+proc colorBool(b: bool): string =
+  if b: result = "<span color:\"green\">yes</span>" 
+  else: result = "<span color:\"red\">no</span>"
+
+proc `$`(r: TResults): string = 
+  result = "<table><tr><td>Test</td><td>Expected</td>" &
+           "<td>Given</td><td>Success</td></tr>\n"
+  for test, expected, given, success in items(r.data):
+    result.add("<tr><td>$#</td><td>$#</td><td>$#</td><td>$#</td></tr>\n" % [
+      test, expected, given, success.colorBool])
+  result.add("</table>\n")
+
+proc cmpMsgs(r: var TResults, expected, given: TMsg, test: string) = 
+  inc(r.total)
+  if strip(expected.msg) notin strip(given.msg): 
+    r.data.add((test, expected.msg, given.msg, false))
+  elif expected.file != given.file:
+    r.data.add((test, expected.file, given.file, false))
+  elif expected.line != given.line: 
+    r.data.add((test, $expected.line, $given.line, false))
+  else:
+    r.data.add((test, expected.msg, given.msg, true))
+    inc(r.passed)
+
+proc reject(r: var TResults, dir, options: string) =  
+  ## handle all the tests that the compiler should reject
+  var specs = parseRejectData(dir)
   
+  for test in os.walkFiles(dir / "t*.nim"):
+    var t = extractFilename(test)
+    var expected = findSpec(specs, t)
+    var given = callCompiler(test, options)
+    cmpMsgs(r, specs[expected], given, t)
   
-proc accept(options: string) = 
-  nil
-  
+proc compile(r: var TResults, pattern, options: string) = 
+  for test in os.walkFiles(pattern): 
+    var t = extractFilename(test)
+    inc(r.total)
+    var given = callCompiler(test, options)
+    r.data.add((t, "", given.msg, not given.err))
+    if not given.err: inc(r.passed)
   
+proc run(r: var TResults, dir, options: string) = 
+  var specs = parseRunData(dir)
+  for test in os.walkFiles(dir / "t*.nim"): 
+    var t = extractFilename(test)
+    inc(r.total)
+    var given = callCompiler(test, options)
+    if given.err:
+      r.data.add((t, "", given.msg, not given.err))
+    else:
+      var exeFile = changeFileExt(test, ExeExt)
+      var expected = specs[findSpec(specs, t)]
+      if existsFile(exeFile):
+        var buf = myExec(exeFile)
+        var success = strip(buf) == strip(spec.outp)
+        if success: inc(r.passed)
+        r.data.add((t, spec.outp, buf, success))
+      else:
+        r.data.add((t, spec.outp, "executable not found", false))
 
-
-var
-  options = ""
+var options = ""
+var rejectRes = initResults()
+var compileRes = initResults()
+var runRes = initResults()
+  
 for i in 1.. paramCount():
   add(options, " ")
   add(options, paramStr(i))
-main(options)
+
+reject(rejectRes, "tests/reject", options)
+compile(compileRes, "tests/accept/compile/t*.nim", options)
+compile(compileRes, "examples/*.nim", options)
+compile(compileRes, "examples/gtk/*.nim", options)
+run(runRes, "tests/accept/run", options)
+