#
#
# Nim Tester
# (c) Copyright 2015 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## This program verifies Nim against the testcases.
import
parseutils, strutils, pegs, os, osproc, streams, parsecfg, json,
marshal, backend, parseopt, specs, htmlgen, browsers, terminal,
algorithm, compiler/nodejs, re, times, sets
const
resultsFile = "testresults.html"
jsonFile = "testresults.json"
Usage = """Usage:
tester [options] command [arguments]
Command:
all run all tests
c|category <category> run all the tests of a certain category
r|run <test> run single test file
html [commit] generate $1 from the database; uses the latest
commit or a specific one (use -1 for the commit
before latest etc)
Arguments:
arguments are passed to the compiler
Options:
--print also print results to the console
--failing only show failing/ignored tests
--pedantic return non-zero status code if there are failures
--targets:"c c++ js objc" run tests for specified targets (default: all)
--nim:path use a particular nim executable (default: compiler/nim)
""" % resultsFile
type
Category = distinct string
TResults = object
total, passed, skipped: int
data: string
TTest = object
name: string
cat: Category
options: string
target: TTarget
action: TTestAction
startTime: float
# ----------------------------------------------------------------------------
let
pegLineError =
peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}"
pegLineTemplate =
peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' 'template/generic instantiation from here'.*"
pegOtherError = peg"'Error:' \s* {.*}"
pegSuccess = peg"'Hint: operation successful'.*"
pegOfInterest = pegLineError / pegOtherError
var targets = {low(TTarget)..high(TTarget)}
proc normalizeMsg(s: string): string = s.strip.replace("\C\L", "\L")
proc callCompiler(cmdTemplate, filename, options: string,
target: TTarget): TSpec =
let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
"options", options, "file", filename.quoteShell])
var p = startProcess(command=c[0], args=c[1.. ^1],
options={poStdErrToStdOut, poUsePath})
let outp = p.outputStream
var suc = ""
var err = ""
var tmpl = ""
var x = newStringOfCap(120)
result.nimout = ""
while outp.readLine(x.TaintedString) or running(p):
result.nimout.add(x & "\n")
if x =~ pegOfInterest:
# `err` should contain the last error/warning message
err = x
elif x =~ pegLineTemplate and err == "":
# `tmpl` contains the last template expansion before the error
tmpl = x
elif x =~ pegSuccess:
suc = x
close(p)
result.msg = ""
result.file = ""
result.outp = ""
result.line = 0
result.column = 0
result.tfile = ""
result.tline = 0
result.tcolumn = 0
if tmpl =~ pegLineTemplate:
result.tfile = extractFilename(matches[0])
result.tline = parseInt(matches[1])
result.tcolumn = parseInt(matches[2])
if err =~ pegLineError:
result.file = extractFilename(matches[0])
result.line = parseInt(matches[1])
result.column = parseInt(matches[2])
result.msg = matches[3]
elif err =~ pegOtherError:
result.msg = matches[0]
elif suc =~ pegSuccess:
result.err = reSuccess
proc callCCompiler(cmdTemplate, filename, options: string,
target: TTarget): TSpec =
let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
"options", options, "file", filename.quoteShell])
var p = startProcess(command="gcc", args=c[5.. ^1],
options={poStdErrToStdOut, poUsePath})
let outp = p.outputStream
var x = newStringOfCap(120)
result.nimout = ""
result.msg = ""
result.file = ""
result.outp = ""
result.line = -1
while outp.readLine(x.TaintedString) or running(p):
result.nimout.add(x & "\n")
close(p)
if p.peekExitCode == 0:
result.err = reSuccess
proc initResults: TResults =
result.total = 0
result.passed = 0
result.skipped = 0
result.data = ""
proc readResults(filename: string): TResults =
result = marshal.to[TResults](readFile(filename).string)
proc writeResults(filename: string, r: TResults) =
writeFile(filename, $$r)
proc `$`(x: TResults): string =
result = ("Tests passed: $1 / $3 <br />\n" &
"Tests skipped: $2 / $3 <br />\n") %
[$x.passed, $x.skipped, $x.total]
proc addResult(r: var TResults, test: TTest,
expected, given: string, success: TResultEnum) =
let name = test.name.extractFilename & test.options
let duration = epochTime() - test.startTime
backend.writeTestResult(name = name,
category = test.cat.string,
target = $test.target,
action = $test.action,
result = $success,
expected = expected,
given = given)
r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
if success == reSuccess:
styledEcho fgGreen, "PASS: ", fgCyan, name
elif success == reIgnored:
styledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name
else:
styledEcho styleBright, fgRed, "FAIL: ", fgCyan, name
styledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
styledEcho styleBright, fgRed, "Failure: ", $success
styledEcho fgYellow, "Expected:"
styledEcho styleBright, expected, "\n"
styledEcho fgYellow, "Gotten:"
styledEcho styleBright, given, "\n"
if existsEnv("APPVEYOR"):
let (outcome, msg) =
if success == reSuccess:
("Passed", "")
elif success == reIgnored:
("Skipped", "")
else:
("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options, "-Framework", "nim-testament", "-FileName", test.cat.string, "-Outcome", outcome, "-ErrorMessage", msg, "-Duration", $(duration*1000).int], options={poStdErrToStdOut, poUsePath, poParentStreams})
discard waitForExit(p)
close(p)
proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) =
if strip(expected.msg) notin strip(given.msg):
r.addResult(test, expected.msg, given.msg, reMsgsDiffer)
elif expected.nimout.len > 0 and expected.nimout.normalizeMsg notin given.nimout.normalizeMsg:
r.addResult(test, expected.nimout, given.nimout, reMsgsDiffer)
elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and
"internal error:" notin expected.msg:
r.addResult(test, expected.file, given.file, reFilesDiffer)
elif expected.line != given.line and expected.line != 0 or
expected.column != given.column and expected.column != 0:
r.addResult(test, $expected.line & ':' & $expected.column,
$given.line & ':' & $given.column,
reLinesDiffer)
elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and
"internal error:" notin expected.msg:
r.addResult(test, expected.tfile, given.tfile, reFilesDiffer)
elif expected.tline != given.tline and expected.tline != 0 or
expected.tcolumn != given.tcolumn and expected.tcolumn != 0:
r.addResult(test, $expected.tline & ':' & $expected.tcolumn,
$given.tline & ':' & $given.tcolumn,
reLinesDiffer)
else:
r.addResult(test, expected.msg, given.msg, reSuccess)
inc(r.passed)
proc generatedFile(path, name: string, target: TTarget): string =
let ext = targetToExt[target]
result = path / "nimcache" /
(if target == targetJS: "" else: "compiler_") &
name.changeFileExt(ext)
proc needsCodegenCheck(spec: TSpec): bool =
result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0
proc codegenCheck(test: TTest, spec: TSpec, expectedMsg: var string,
given: var TSpec) =
try:
let (path, name, _) = test.name.splitFile
let genFile = generatedFile(path, name, test.target)
let contents = readFile(genFile).string
let check = spec.ccodeCheck
if check.len > 0:
if check[0] == '\\':
# little hack to get 'match' support:
if not contents.match(check.peg):
given.err = reCodegenFailure
elif contents.find(check.peg) < 0:
given.err = reCodegenFailure
expectedMsg = check
if spec.maxCodeSize > 0 and contents.len > spec.maxCodeSize:
given.err = reCodegenFailure
given.msg = "generated code size: " & $contents.len
expectedMsg = "max allowed size: " & $spec.maxCodeSize
except ValueError:
given.err = reInvalidPeg
echo getCurrentExceptionMsg()
except IOError:
given.err = reCodeNotFound
proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) =
let exp = expectedNimout.strip.replace("\C\L", "\L")
let giv = given.nimout.strip.replace("\C\L", "\L")
if exp notin giv:
given.err = reMsgsDiffer
proc makeDeterministic(s: string): string =
var x = splitLines(s)
sort(x, system.cmp)
result = join(x, "\n")
proc compilerOutputTests(test: TTest, given: var TSpec, expected: TSpec;
r: var TResults) =
var expectedmsg: string = ""
var givenmsg: string = ""
if given.err == reSuccess:
if expected.needsCodegenCheck:
codegenCheck(test, expected, expectedmsg, given)
givenmsg = given.msg
if expected.nimout.len > 0:
expectedmsg = expected.nimout
givenmsg = given.nimout.strip
nimoutCheck(test, expectedmsg, given)
else:
givenmsg = given.nimout.strip
if given.err == reSuccess: inc(r.passed)
r.addResult(test, expectedmsg, givenmsg, given.err)
proc analyzeAndConsolidateOutput(s: string): string =
result = ""
let rows = s.splitLines
for i in 0 ..< rows.len:
if (let pos = find(rows[i], "Traceback (most recent call last)"); pos != -1):
result = substr(rows[i], pos) & "\n"
for i in i+1 ..< rows.len:
result.add rows[i] & "\n"
if not (rows[i] =~ re"^[^(]+\(\d+\)\s+"):
return
elif (let pos = find(rows[i], "SIGSEGV: Illegal storage access."); pos != -1):
result = substr(rows[i], pos)
return
proc testSpec(r: var TResults, test: TTest) =
# major entry point for a single test
if test.target notin targets:
r.addResult(test, "", "", reIgnored)
inc(r.skipped)
return
let tname = test.name.addFileExt(".nim")
#echo "TESTING ", tname
inc(r.total)
var expected: TSpec
if test.action != actionRunNoSpec:
expected = parseSpec(tname)
else:
specDefaults expected
expected.action = actionRunNoSpec
if expected.err == reIgnored:
r.addResult(test, "", "", reIgnored)
inc(r.skipped)
return
case expected.action
of actionCompile:
var given = callCompiler(expected.cmd, test.name,
test.options & " --stdout --hint[Path]:off --hint[Processing]:off",
test.target)
compilerOutputTests(test, given, expected, r)
of actionRun, actionRunNoSpec:
# In this branch of code "early return" pattern is clearer than deep
# nested conditionals - the empty rows in between to clarify the "danger"
var given = callCompiler(expected.cmd, test.name, test.options,
test.target)
if given.err != reSuccess:
r.addResult(test, "", given.msg, given.err)
return
let isJsTarget = test.target == targetJS
var exeFile: string
if isJsTarget:
let (dir, file, _) = splitFile(tname)
exeFile = dir / "nimcache" / file & ".js" # *TODO* hardcoded "nimcache"
else:
exeFile = changeFileExt(tname, ExeExt)
if not existsFile(exeFile):
r.addResult(test, expected.outp, "executable not found", reExeNotFound)
return
let nodejs = if isJsTarget: findNodeJs() else: ""
if isJsTarget and nodejs == "":
r.addResult(test, expected.outp, "nodejs binary not in PATH",
reExeNotFound)
return
let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile
var (buf, exitCode) = execCmdEx(exeCmd, options = {poStdErrToStdOut})
# Treat all failure codes from nodejs as 1. Older versions of nodejs used
# to return other codes, but for us it is sufficient to know that it's not 0.
if exitCode != 0: exitCode = 1
let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string))
else: strip(buf.string)
let expectedOut = strip(expected.outp)
if exitCode != expected.exitCode:
r.addResult(test, "exitcode: " & $expected.exitCode,
"exitcode: " & $exitCode & "\n\nOutput:\n" &
analyzeAndConsolidateOutput(bufB),
reExitCodesDiffer)
return
if bufB != expectedOut and expected.action != actionRunNoSpec:
if not (expected.substr and expectedOut in bufB):
given.err = reOutputsDiffer
r.addResult(test, expected.outp, bufB, reOutputsDiffer)
return
compilerOutputTests(test, given, expected, r)
return
of actionReject:
var given = callCompiler(expected.cmd, test.name, test.options,
test.target)
cmpMsgs(r, expected, given, test)
return
proc testNoSpec(r: var TResults, test: TTest) =
# does not extract the spec because the file is not supposed to have any
#let tname = test.name.addFileExt(".nim")
inc(r.total)
let given = callCompiler(cmdTemplate(), test.name, test.options, test.target)
r.addResult(test, "", given.msg, given.err)
if given.err == reSuccess: inc(r.passed)
proc testC(r: var TResults, test: TTest) =
# runs C code. Doesn't support any specs, just goes by exit code.
let tname = test.name.addFileExt(".c")
inc(r.total)
styledEcho "Processing ", fgCyan, extractFilename(tname)
var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, test.target)
if given.err != reSuccess:
r.addResult(test, "", given.msg, given.err)
elif test.action == actionRun:
let exeFile = changeFileExt(test.name, ExeExt)
var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath})
if exitCode != 0: given.err = reExitCodesDiffer
if given.err == reSuccess: inc(r.passed)
proc makeTest(test, options: string, cat: Category, action = actionCompile,
target = targetC, env: string = ""): TTest =
# start with 'actionCompile', will be overwritten in the spec:
result = TTest(cat: cat, name: test, options: options,
target: target, action: action, startTime: epochTime())
when defined(windows):
const
# array of modules disabled from compilation test of stdlib.
disabledFiles = ["coro.nim", "fsmonitor.nim"]
else:
const
# array of modules disabled from compilation test of stdlib.
disabledFiles = ["-"]
include categories
# proc runCaasTests(r: var TResults) =
# for test, output, status, mode in caasTestsRunner():
# r.addResult(test, "", output & "-> " & $mode,
# if status: reSuccess else: reOutputsDiffer)
proc main() =
os.putenv "NIMTEST_NO_COLOR", "1"
os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
backend.open()
var optPrintResults = false
var optFailing = false
var optPedantic = false
var p = initOptParser()
p.next()
while p.kind == cmdLongoption:
case p.key.string.normalize
of "print", "verbose": optPrintResults = true
of "failing": optFailing = true
of "pedantic": optPedantic = true
of "targets": targets = parseTargets(p.val.string)
of "nim": compilerPrefix = p.val.string
else: quit Usage
p.next()
if p.kind != cmdArgument: quit Usage
var action = p.key.string.normalize
p.next()
var r = initResults()
case action
of "all":
let testsDir = "tests" & DirSep
for kind, dir in walkDir(testsDir):
assert testsDir.startsWith(testsDir)
let cat = dir[testsDir.len .. ^1]
if kind == pcDir and cat notin ["testament", "testdata", "nimcache"]:
processCategory(r, Category(cat), p.cmdLineRest.string)
for a in AdditionalCategories:
processCategory(r, Category(a), p.cmdLineRest.string)
of "c", "cat", "category":
var cat = Category(p.key)
p.next
processCategory(r, cat, p.cmdLineRest.string)
of "r", "run":
let (dir, file) = splitPath(p.key.string)
let (_, subdir) = splitPath(dir)
var cat = Category(subdir)
processSingleTest(r, cat, p.cmdLineRest.string, file)
of "html":
var commit = 0
discard parseInt(p.cmdLineRest.string, commit)
generateHtml(resultsFile, commit, optFailing)
generateJson(jsonFile, commit)
else:
quit Usage
if optPrintResults:
if action == "html": openDefaultBrowser(resultsFile)
else: echo r, r.data
backend.close()
if optPedantic:
var failed = r.total - r.passed - r.skipped
if failed > 0:
echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ", r.skipped
quit(QuitFailure)
if paramCount() == 0:
quit Usage
main()