diff options
Diffstat (limited to 'tests/testament')
| -rw-r--r-- | tests/testament/backend.nim | 2 | ||||
| -rw-r--r-- | tests/testament/caasdriver.nim | 15 | ||||
| -rw-r--r-- | tests/testament/categories.nim | 114 | ||||
| -rw-r--r-- | tests/testament/htmlgen.nim | 41 | ||||
| -rw-r--r-- | tests/testament/specs.nim | 28 | ||||
| -rw-r--r-- | tests/testament/tester.nim | 119 | ||||
| -rw-r--r-- | tests/testament/tester.nim.cfg | 1 |
7 files changed, 191 insertions, 129 deletions
diff --git a/tests/testament/backend.nim b/tests/testament/backend.nim index c7122e1b2..11743c337 100644 --- a/tests/testament/backend.nim +++ b/tests/testament/backend.nim @@ -1,7 +1,7 @@ # # # The Nim Tester -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # Look at license.txt for more info. # All rights reserved. diff --git a/tests/testament/caasdriver.nim b/tests/testament/caasdriver.nim index 8f2eec33b..c61a9f108 100644 --- a/tests/testament/caasdriver.nim +++ b/tests/testament/caasdriver.nim @@ -1,4 +1,5 @@ import osproc, streams, os, strutils, re +{.experimental.} ## Compiler as a service tester. ## @@ -10,7 +11,7 @@ type ProcRun, CaasRun, SymbolProcRun NimSession* = object - nim: PProcess # Holds the open process for CaasRun sessions, nil otherwise. + nim: Process # Holds the open process for CaasRun sessions, nil otherwise. mode: TRunMode # Stores the type of run mode the session was started with. lastOutput: string # Preserves the last output, needed for ProcRun mode. filename: string # Appended to each command starting with '>'. Also a var. @@ -70,8 +71,10 @@ proc doCaasCommand(session: var NimSession, command: string): string = break proc doProcCommand(session: var NimSession, command: string): string = - assert session.mode == ProcRun or session.mode == SymbolProcRun - except: result = "FAILED TO EXECUTE: " & command & "\n" & result + try: + assert session.mode == ProcRun or session.mode == SymbolProcRun + except: + result = "FAILED TO EXECUTE: " & command & "\n" & result var process = startProcess(NimBin, args = session.replaceVars(command).split) stream = outputStream(process) @@ -102,11 +105,11 @@ proc doCommand(session: var NimSession, command: string) = session.lastOutput = doProcCommand(session, command & " " & session.filename) -proc close(session: var NimSession) {.destructor.} = +proc destroy(session: var NimSession) {.destructor.} = if session.mode == CaasRun: session.nim.close -proc doScenario(script: string, output: PStream, mode: TRunMode, verbose: bool): bool = +proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): bool = result = true var f = open(script) @@ -171,7 +174,7 @@ when isMainModule: failures = 0 verbose = false - for i in 0..ParamCount() - 1: + for i in 0..paramCount() - 1: let param = string(paramStr(i + 1)) case param of "verbose": verbose = true diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index ae9905cde..07a2421a6 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -1,7 +1,7 @@ # # # Nim Tester -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -22,35 +22,35 @@ proc delNimCache() = removeDir(nimcacheDir) except OSError: echo "[Warning] could not delete: ", nimcacheDir - + proc runRodFiles(r: var TResults, cat: Category, options: string) = template test(filename: expr): stmt = testSpec r, makeTest(rodfilesDir / filename, options, cat, actionRun) - + delNimCache() - + # test basic recompilation scheme: test "hallo" test "hallo" # test incremental type information: test "hallo2" delNimCache() - + # test type converters: test "aconv" test "bconv" delNimCache() - + # test G, A, B example from the documentation; test init sections: test "deada" test "deada2" delNimCache() - + # test method generation: test "bmethods" test "bmethods2" delNimCache() - + # test generics: test "tgeneric1" test "tgeneric2" @@ -79,26 +79,26 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = options & " --app:lib -d:createNimRtl", cat) testSpec c, makeTest("tests/dll/server.nim", options & " --app:lib -d:useNimRtl", cat) - - when defined(Windows): + + when defined(Windows): # windows looks in the dir of the exe (yay!): var nimrtlDll = DynlibFormat % "nimrtl" safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll) else: # posix relies on crappy LD_LIBRARY_PATH (ugh!): - var libpath = getenv"LD_LIBRARY_PATH".string - if peg"\i '/nim' (!'/')* '/lib'" notin libpath: - echo "[Warning] insufficient LD_LIBRARY_PATH" + var libpath = getEnv"LD_LIBRARY_PATH".string + # Temporarily add the lib directory to LD_LIBRARY_PATH: + putEnv("LD_LIBRARY_PATH", "lib:" & libpath) var serverDll = DynlibFormat % "server" safeCopyFile("tests/dll" / serverDll, "lib" / serverDll) - - testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl", + + testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl", cat, actionRun) proc dllTests(r: var TResults, cat: Category, options: string) = # dummy compile result: var c = initResults() - + runBasicDLLTest c, r, cat, options runBasicDLLTest c, r, cat, options & " -d:release" runBasicDLLTest c, r, cat, options & " --gc:boehm" @@ -107,30 +107,36 @@ proc dllTests(r: var TResults, cat: Category, options: string) = # ------------------------------ GC tests ------------------------------------- proc gcTests(r: var TResults, cat: Category, options: string) = - template test(filename: expr): stmt = + template testWithoutMs(filename: expr): stmt = testSpec r, makeTest("tests/gc" / filename, options, cat, actionRun) testSpec r, makeTest("tests/gc" / filename, options & " -d:release", cat, actionRun) testSpec r, makeTest("tests/gc" / filename, options & " -d:release -d:useRealtimeGC", cat, actionRun) + + template test(filename: expr): stmt = + testWithoutMs filename testSpec r, makeTest("tests/gc" / filename, options & " --gc:markAndSweep", cat, actionRun) testSpec r, makeTest("tests/gc" / filename, options & " -d:release --gc:markAndSweep", cat, actionRun) - + + test "growobjcrash" test "gcbench" test "gcleak" test "gcleak2" test "gctest" test "gcleak3" test "gcleak4" - test "gcleak5" + # Disabled because it works and takes too long to run: + #test "gcleak5" test "weakrefs" test "cycleleak" test "closureleak" - test "refarrayleak" + testWithoutMs "refarrayleak" + test "stackrefleak" - + test "cyclecollector" # ------------------------- threading tests ----------------------------------- @@ -141,7 +147,7 @@ proc threadTests(r: var TResults, cat: Category, options: string) = " -d:release", cat, actionRun) testSpec r, makeTest("tests/threads" / filename, options & " --tlsEmulation:on", cat, actionRun) - + test "tactors" test "tactors2" test "threadex" @@ -176,7 +182,7 @@ proc jsTests(r: var TResults, cat: Category, options: string) = actionRun, targetJS) testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat, actionRun, targetJS) - + for t in os.walkFiles("tests/js/t*.nim"): test(t) for testfile in ["exception/texceptions", "exception/texcpt1", @@ -193,13 +199,13 @@ proc jsTests(r: var TResults, cat: Category, options: string) = proc findMainFile(dir: string): string = # finds the file belonging to ".nim.cfg"; if there is no such file - # it returns the some ".nim" file if there is only one: + # it returns the some ".nim" file if there is only one: const cfgExt = ".nim.cfg" result = "" var nimFiles = 0 for kind, file in os.walkDir(dir): if kind == pcFile: - if file.endsWith(cfgExt): return file[.. -(cfgExt.len+1)] & ".nim" + if file.endsWith(cfgExt): return file[.. ^(cfgExt.len+1)] & ".nim" elif file.endsWith(".nim"): if result.len == 0: result = file inc nimFiles @@ -209,7 +215,7 @@ proc manyLoc(r: var TResults, cat: Category, options: string) = for kind, dir in os.walkDir("tests/manyloc"): if kind == pcDir: let mainfile = findMainFile(dir) - if mainfile != ".nim": + if mainfile != "": testNoSpec r, makeTest(mainfile, options, cat) proc compileExample(r: var TResults, pattern, options: string, cat: Category) = @@ -220,21 +226,21 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = for test in os.walkFiles(pattern): let contents = readFile(test).string if contents.contains("when isMainModule"): - testSpec r, makeTest(test, options, cat, actionRun) + testSpec r, makeTest(test, options, cat, actionRunNoSpec) else: testNoSpec r, makeTest(test, options, cat, actionCompile) -# ----------------------------- babel ---------------------------------------- +# ----------------------------- nimble ---------------------------------------- type PackageFilter = enum pfCoreOnly pfExtraOnly pfAll -let - babelExe = findExe("babel") - babelDir = getHomeDir() / ".babel" - packageDir = babelDir / "pkgs" - packageIndex = babelDir / "packages.json" +let + nimbleExe = findExe("nimble") + nimbleDir = getHomeDir() / ".nimble" + packageDir = nimbleDir / "pkgs" + packageIndex = nimbleDir / "packages.json" proc waitForExitEx(p: Process): int = var outp = outputStream(p) @@ -249,7 +255,7 @@ proc waitForExitEx(p: Process): int = proc getPackageDir(package: string): string = ## TODO - Replace this with dom's version comparison magic. - var commandOutput = execCmdEx("babel path $#" % package) + var commandOutput = execCmdEx("nimble path $#" % package) if commandOutput.exitCode != QuitSuccess: return "" else: @@ -262,7 +268,7 @@ iterator listPackages(filter: PackageFilter): tuple[name, url: string] = let name = package["name"].str url = package["url"].str - isCorePackage = "nimrod-code" in normalize(url) + isCorePackage = "nim-lang" in normalize(url) case filter: of pfCoreOnly: if isCorePackage: @@ -273,13 +279,13 @@ iterator listPackages(filter: PackageFilter): tuple[name, url: string] = of pfAll: yield (name, url) -proc testBabelPackages(r: var TResults, cat: Category, filter: PackageFilter) = - if babelExe == "": - echo("[Warning] - Cannot run babel tests: Babel binary not found.") +proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) = + if nimbleExe == "": + echo("[Warning] - Cannot run nimble tests: Nimble binary not found.") return - if execCmd("$# update" % babelExe) == QuitFailure: - echo("[Warning] - Cannot run babel tests: Babel update failed.") + if execCmd("$# update" % nimbleExe) == QuitFailure: + echo("[Warning] - Cannot run nimble tests: Nimble update failed.") return let packageFileTest = makeTest("PackageFileParsed", "", cat) @@ -288,7 +294,7 @@ proc testBabelPackages(r: var TResults, cat: Category, filter: PackageFilter) = var test = makeTest(name, "", cat) echo(url) let - installProcess = startProcess(babelExe, "", ["install", "-y", name]) + installProcess = startProcess(nimbleExe, "", ["install", "-y", name]) installStatus = waitForExitEx(installProcess) installProcess.close if installStatus != QuitSuccess: @@ -296,9 +302,8 @@ proc testBabelPackages(r: var TResults, cat: Category, filter: PackageFilter) = continue let - buildPath = getPackageDir(name)[0.. -3] - let - buildProcess = startProcess(babelExe, buildPath, ["build"]) + buildPath = getPackageDir(name).strip + buildProcess = startProcess(nimbleExe, buildPath, ["build"]) buildStatus = waitForExitEx(buildProcess) buildProcess.close if buildStatus != QuitSuccess: @@ -306,13 +311,13 @@ proc testBabelPackages(r: var TResults, cat: Category, filter: PackageFilter) = r.addResult(test, "", "", reSuccess) r.addResult(packageFileTest, "", "", reSuccess) except JsonParsingError: - echo("[Warning] - Cannot run babel tests: Invalid package file.") + echo("[Warning] - Cannot run nimble tests: Invalid package file.") r.addResult(packageFileTest, "", "", reBuildFailed) # ---------------------------------------------------------------------------- -const AdditionalCategories = ["debugger", "examples", "lib", "babel-core"] +const AdditionalCategories = ["debugger", "examples", "lib"] proc `&.?`(a, b: string): string = # candidate for the stdlib? @@ -325,8 +330,9 @@ proc `&?.`(a, b: string): string = proc processCategory(r: var TResults, cat: Category, options: string) = case cat.string.normalize of "rodfiles": - compileRodFiles(r, cat, options) - runRodFiles(r, cat, options) + discard # Disabled for now + #compileRodFiles(r, cat, options) + #runRodFiles(r, cat, options) of "js": # XXX JS doesn't need to be special anymore jsTests(r, cat, options) @@ -349,12 +355,12 @@ proc processCategory(r: var TResults, cat: Category, options: string) = compileExample(r, "examples/*.nim", options, cat) compileExample(r, "examples/gtk/*.nim", options, cat) compileExample(r, "examples/talk/*.nim", options, cat) - of "babel-core": - testBabelPackages(r, cat, pfCoreOnly) - of "babel-extra": - testBabelPackages(r, cat, pfExtraOnly) - of "babel-all": - testBabelPackages(r, cat, pfAll) + of "nimble-core": + testNimblePackages(r, cat, pfCoreOnly) + of "nimble-extra": + testNimblePackages(r, cat, pfExtraOnly) + of "nimble-all": + testNimblePackages(r, cat, pfAll) else: for name in os.walkFiles("tests" & DirSep &.? cat.string / "t*.nim"): testSpec r, makeTest(name, options, cat) diff --git a/tests/testament/htmlgen.nim b/tests/testament/htmlgen.nim index b9eda5383..a9f739995 100644 --- a/tests/testament/htmlgen.nim +++ b/tests/testament/htmlgen.nim @@ -1,7 +1,7 @@ # # # Nim Tester -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -20,7 +20,7 @@ const <td>Success</td></tr>""" TableFooter = "</table>" HtmlBegin = """<html> - <head> + <head> <title>Test results</title> <style type="text/css"> <!--""" & slurp("css/boilerplate.css") & "\n" & @@ -28,13 +28,13 @@ const """ ul#tabs { list-style-type: none; margin: 30px 0 0 0; padding: 0 0 0.3em 0; } ul#tabs li { display: inline; } -ul#tabs li a { color: #42454a; background-color: #dedbde; - border: 1px solid #c9c3ba; border-bottom: none; +ul#tabs li a { color: #42454a; background-color: #dedbde; + border: 1px solid #c9c3ba; border-bottom: none; padding: 0.3em; text-decoration: none; } ul#tabs li a:hover { background-color: #f1f0ee; } -ul#tabs li a.selected { color: #000; background-color: #f1f0ee; +ul#tabs li a.selected { color: #000; background-color: #f1f0ee; font-weight: bold; padding: 0.7em 0.3em 0.38em 0.3em; } -div.tabContent { border: 1px solid #c9c3ba; +div.tabContent { border: 1px solid #c9c3ba; padding: 0.5em; background-color: #f1f0ee; } div.tabContent.hide { display: none; } --> @@ -43,7 +43,7 @@ div.tabContent.hide { display: none; } var tabLinks = new Array(); var contentDivs = new Array(); - + function init() { // Grab the tab links and content divs from the page var tabListItems = document.getElementById('tabs').childNodes; @@ -103,7 +103,7 @@ div.tabContent.hide { display: none; } </head> <body onload="init()">""" - + HtmlEnd = "</body></html>" proc td(s: string): string = @@ -115,8 +115,8 @@ proc getCommit(db: TDbConn, c: int): string = if commit == 0: result = thisCommit[0] inc commit -proc generateHtml*(filename: string, commit: int) = - const selRow = """select name, category, target, action, +proc generateHtml*(filename: string, commit: int; onlyFailing: bool) = + const selRow = """select name, category, target, action, expected, given, result from TestResult where [commit] = ? and machine = ? @@ -140,17 +140,20 @@ proc generateHtml*(filename: string, commit: int) = for m in db.rows(sql"select id, name, os, cpu from Machine order by id"): outfile.writeln """<li><a href="#$#">$#: $#, $#</a></li>""" % m outfile.write("</ul>") - + for currentMachine in db.rows(sql"select id from Machine order by id"): let m = currentMachine[0] outfile.write("""<div class="tabContent" id="$#">""" % m) outfile.write(TableHeader) for row in db.rows(sql(selRow), lastCommit, m): - outfile.write("<tr>") - for x in row: - outfile.write(x.td) - outfile.write("</tr>") + if onlyFailing and row.len > 0 and row[row.high] == "reSuccess": + discard + else: + outfile.write("<tr>") + for x in row: + outfile.write(x.td) + outfile.write("</tr>") outfile.write(TableFooter) outfile.write("</div>") @@ -161,7 +164,7 @@ proc generateHtml*(filename: string, commit: int) = proc generateJson*(filename: string, commit: int) = const selRow = """select count(*), - sum(result = 'reSuccess'), + sum(result = 'reSuccess'), sum(result = 'reIgnored') from TestResult where [commit] = ? and machine = ? @@ -174,9 +177,9 @@ proc generateJson*(filename: string, commit: int) = on A.name = B.name and A.category = B.category where A.[commit] = ? and B.[commit] = ? and A.machine = ? and A.result != B.result""" - selResults = """select - category || '/' || target || '/' || name, - category, target, action, result, expected, given + selResults = """select + category || '/' || target || '/' || name, + category, target, action, result, expected, given from TestResult where [commit] = ?""" var db = open(connection="testament.db", user="testament", password="", diff --git a/tests/testament/specs.nim b/tests/testament/specs.nim index 37fe8cfee..9306bf025 100644 --- a/tests/testament/specs.nim +++ b/tests/testament/specs.nim @@ -1,7 +1,7 @@ # # # Nim Tester -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -10,13 +10,14 @@ import parseutils, strutils, os, osproc, streams, parsecfg const - cmdTemplate* = r"nim $target --hints:on $options $file" + cmdTemplate* = r"nim $target --hints:on -d:testing $options $file" type TTestAction* = enum actionCompile = "compile" actionRun = "run" actionReject = "reject" + actionRunNoSpec = "runNoSpec" TResultEnum* = enum reNimcCrash, # nim compiler seems to have crashed reMsgsDiffer, # error messages differ @@ -42,12 +43,14 @@ type action*: TTestAction file*, cmd*: string outp*: string - line*, exitCode*: int + line*, column*: int + exitCode*: int msg*: string ccodeCheck*: string err*: TResultEnum substr*, sortoutput*: bool targets*: set[TTarget] + nimout*: string const targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"] @@ -76,7 +79,7 @@ proc extractSpec(filename: string): string = when not defined(nimhygiene): {.pragma: inject.} -template parseSpecAux(fillResult: stmt) {.immediate.} = +template parseSpecAux(fillResult: untyped) = var ss = newStringStream(extractSpec(filename)) var p {.inject.}: CfgParser open(p, ss, filename, 1) @@ -90,12 +93,18 @@ template parseSpecAux(fillResult: stmt) {.immediate.} = fillResult close(p) -proc parseSpec*(filename: string): TSpec = - result.file = filename +proc specDefaults*(result: var TSpec) = result.msg = "" result.outp = "" + result.nimout = "" result.ccodeCheck = "" result.cmd = cmdTemplate + result.line = 0 + result.column = 0 + +proc parseSpec*(filename: string): TSpec = + specDefaults(result) + result.file = filename parseSpecAux: case normalize(e.key) of "action": @@ -106,7 +115,8 @@ proc parseSpec*(filename: string): TSpec = else: echo ignoreMsg(p, e) of "file": result.file = e.value of "line": discard parseInt(e.value, result.line) - of "output": + of "column": discard parseInt(e.value, result.column) + of "output": result.action = actionRun result.outp = e.value of "outputsub": @@ -115,7 +125,7 @@ proc parseSpec*(filename: string): TSpec = result.substr = true of "sortoutput": result.sortoutput = parseCfgBool(e.value) - of "exitcode": + of "exitcode": discard parseInt(e.value, result.exitCode) of "msg": result.msg = e.value @@ -124,6 +134,8 @@ proc parseSpec*(filename: string): TSpec = of "errormsg": result.msg = e.value result.action = actionReject + of "nimout": + result.nimout = e.value of "disabled": if parseCfgBool(e.value): result.err = reIgnored of "cmd": result.cmd = e.value diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index b74fa99c8..b3e65959a 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -1,7 +1,7 @@ # # # Nim Tester -# (c) Copyright 2014 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -12,7 +12,7 @@ import parseutils, strutils, pegs, os, osproc, streams, parsecfg, json, marshal, backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm + algorithm, compiler/nodejs const resultsFile = "testresults.html" @@ -30,6 +30,7 @@ Arguments: arguments are passed to the compiler Options: --print also print results to the console + --failing only show failing/ignored tests """ % resultsFile type @@ -48,8 +49,8 @@ type # ---------------------------------------------------------------------------- let - pegLineError = - peg"{[^(]*} '(' {\d+} ', ' \d+ ') ' ('Error') ':' \s* {.*}" + pegLineError = + peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" pegOtherError = peg"'Error:' \s* {.*}" pegSuccess = peg"'Hint: operation successful'.*" pegOfInterest = pegLineError / pegOtherError @@ -57,14 +58,16 @@ let proc callCompiler(cmdTemplate, filename, options: string, target: TTarget): TSpec = let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], - "options", options, "file", filename]) - var p = startProcess(command=c[0], args=c[1.. -1], + "options", options, "file", filename.quoteShell]) + var p = startProcess(command=c[0], args=c[1.. ^1], options={poStdErrToStdOut, poUseShell}) let outp = p.outputStream var suc = "" var err = "" var x = newStringOfCap(120) + result.nimout = "" while outp.readLine(x.TaintedString) or running(p): + result.nimout.add(x & "\n") if x =~ pegOfInterest: # `err` should contain the last error/warning message err = x @@ -74,11 +77,13 @@ proc callCompiler(cmdTemplate, filename, options: string, result.msg = "" result.file = "" result.outp = "" - result.line = -1 + result.line = 0 + result.column = 0 if err =~ pegLineError: result.file = extractFilename(matches[0]) result.line = parseInt(matches[1]) - result.msg = matches[2] + result.column = parseInt(matches[2]) + result.msg = matches[3] elif err =~ pegOtherError: result.msg = matches[0] elif suc =~ pegSuccess: @@ -105,14 +110,16 @@ proc addResult(r: var TResults, test: TTest, expected, given: string, success: TResultEnum) = let name = test.name.extractFilename & test.options backend.writeTestResult(name = name, - category = test.cat.string, + category = test.cat.string, target = $test.target, action = $test.action, result = $success, expected = expected, given = given) r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) - if success notin {reSuccess, reIgnored}: + if success == reIgnored: + styledEcho styleBright, name, fgYellow, " [", $success, "]" + elif success != reSuccess: styledEcho styleBright, name, fgRed, " [", $success, "]" echo"Expected:" styledEcho styleBright, expected @@ -125,8 +132,11 @@ proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) = elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: r.addResult(test, expected.file, given.file, reFilesDiffer) - elif expected.line != given.line and expected.line != 0: - r.addResult(test, $expected.line, $given.line, reLinesDiffer) + elif expected.line != given.line and expected.line != 0 or + expected.column != given.column and expected.column != 0: + r.addResult(test, $expected.line & ':' & $expected.column, + $given.line & ':' & $given.column, + reLinesDiffer) else: r.addResult(test, expected.msg, given.msg, reSuccess) inc(r.passed) @@ -134,47 +144,74 @@ proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) = proc generatedFile(path, name: string, target: TTarget): string = let ext = targetToExt[target] result = path / "nimcache" / - (if target == targetJS: path.splitPath.tail & "_" else: "") & + (if target == targetJS: path.splitPath.tail & "_" else: "compiler_") & name.changeFileExt(ext) proc codegenCheck(test: TTest, check: string, given: var TSpec) = - if check.len > 0: - try: - let (path, name, ext2) = test.name.splitFile - let genFile = generatedFile(path, name, test.target) - echo genFile - let contents = readFile(genFile).string - if contents.find(check.peg) < 0: + try: + let (path, name, ext2) = test.name.splitFile + let genFile = generatedFile(path, name, test.target) + let contents = readFile(genFile).string + if check[0] == '\\': + # little hack to get 'match' support: + if not contents.match(check.peg): given.err = reCodegenFailure - except ValueError: - given.err = reInvalidPeg - except IOError: - given.err = reCodeNotFound + elif contents.find(check.peg) < 0: + given.err = reCodegenFailure + except ValueError: + given.err = reInvalidPeg + echo getCurrentExceptionMsg() + except IOError: + given.err = reCodeNotFound + +proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) = + let exp = expectedNimout.strip.replace("\C\L", "\L") + let giv = given.nimout.strip.replace("\C\L", "\L") + if exp notin giv: + given.err = reMsgsDiffer proc makeDeterministic(s: string): string = var x = splitLines(s) sort(x, system.cmp) result = join(x, "\n") +proc compilerOutputTests(test: TTest, given: var TSpec, expected: TSpec; + r: var TResults) = + var expectedmsg: string = "" + var givenmsg: string = "" + if given.err == reSuccess: + if expected.ccodeCheck.len > 0: + codegenCheck(test, expected.ccodeCheck, given) + expectedmsg = expected.ccodeCheck + givenmsg = given.msg + if expected.nimout.len > 0: + expectedmsg = expected.nimout + givenmsg = given.nimout.strip + nimoutCheck(test, expectedmsg, given) + if given.err == reSuccess: inc(r.passed) + r.addResult(test, expectedmsg, givenmsg, given.err) + proc testSpec(r: var TResults, test: TTest) = # major entry point for a single test let tname = test.name.addFileExt(".nim") inc(r.total) styledEcho "Processing ", fgCyan, extractFilename(tname) - var expected = parseSpec(tname) + var expected: TSpec + if test.action != actionRunNoSpec: + expected = parseSpec(tname) + else: + specDefaults expected + expected.action = actionRunNoSpec if expected.err == reIgnored: r.addResult(test, "", "", reIgnored) inc(r.skipped) else: case expected.action of actionCompile: - var given = callCompiler(expected.cmd, test.name, test.options, - test.target) - if given.err == reSuccess: - codegenCheck(test, expected.ccodeCheck, given) - r.addResult(test, "", given.msg, given.err) - if given.err == reSuccess: inc(r.passed) - of actionRun: + var given = callCompiler(expected.cmd, test.name, + test.options & " --hint[Path]:off --hint[Processing]:off", test.target) + compilerOutputTests(test, given, expected, r) + of actionRun, actionRunNoSpec: var given = callCompiler(expected.cmd, test.name, test.options, test.target) if given.err != reSuccess: @@ -187,12 +224,13 @@ proc testSpec(r: var TResults, test: TTest) = else: exeFile = changeFileExt(tname, ExeExt) if existsFile(exeFile): - if test.target == targetJS and findExe("nodejs") == "": + let nodejs = findNodeJs() + if test.target == targetJS and nodejs == "": r.addResult(test, expected.outp, "nodejs binary not in PATH", reExeNotFound) return var (buf, exitCode) = execCmdEx( - (if test.target == targetJS: "nodejs " else: "") & exeFile) + (if test.target == targetJS: nodejs & " " else: "") & exeFile) if exitCode != expected.exitCode: r.addResult(test, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode, reExitCodesDiffer) @@ -202,10 +240,7 @@ proc testSpec(r: var TResults, test: TTest) = if bufB != strip(expected.outp): if not (expected.substr and expected.outp in bufB): given.err = reOutputsDiffer - if given.err == reSuccess: - codeGenCheck(test, expected.ccodeCheck, given) - if given.err == reSuccess: inc(r.passed) - r.addResult(test, expected.outp, buf.string, given.err) + compilerOutputTests(test, given, expected, r) else: r.addResult(test, expected.outp, "executable not found", reExeNotFound) of actionReject: @@ -241,11 +276,13 @@ proc main() = backend.open() var optPrintResults = false + var optFailing = false var p = initOptParser() p.next() - if p.kind == cmdLongoption: + while p.kind == cmdLongoption: case p.key.string.normalize of "print", "verbose": optPrintResults = true + of "failing": optFailing = true else: quit Usage p.next() if p.kind != cmdArgument: quit Usage @@ -257,7 +294,7 @@ proc main() = let testsDir = "tests" & DirSep for kind, dir in walkDir(testsDir): assert testsDir.startsWith(testsDir) - let cat = dir[testsDir.len .. -1] + let cat = dir[testsDir.len .. ^1] if kind == pcDir and cat notin ["testament", "testdata", "nimcache"]: processCategory(r, Category(cat), p.cmdLineRest.string) for a in AdditionalCategories: @@ -269,7 +306,7 @@ proc main() = of "html": var commit = 0 discard parseInt(p.cmdLineRest.string, commit) - generateHtml(resultsFile, commit) + generateHtml(resultsFile, commit, optFailing) generateJson(jsonFile, commit) else: quit Usage diff --git a/tests/testament/tester.nim.cfg b/tests/testament/tester.nim.cfg new file mode 100644 index 000000000..27fd67075 --- /dev/null +++ b/tests/testament/tester.nim.cfg @@ -0,0 +1 @@ +path = "$nim" # For compiler/nodejs |