diff options
Diffstat (limited to 'testament/testament.nim')
| -rw-r--r-- | testament/testament.nim | 56 |
1 files changed, 47 insertions, 9 deletions
diff --git a/testament/testament.nim b/testament/testament.nim index ac99a2373..1e892e636 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -10,15 +10,20 @@ ## This program verifies Nim against the testcases. import - strutils, pegs, os, osproc, streams, json, std/exitprocs, - backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, times, md5, azure, intsets, macros + std/[strutils, pegs, os, osproc, streams, json, + parseopt, browsers, terminal, exitprocs, + algorithm, times, intsets, macros] + +import backend, specs, azure, htmlgen + from std/sugar import dup import compiler/nodejs import lib/stdtest/testutils from lib/stdtest/specialpaths import splitTestFile from std/private/gitutils import diffStrings +import ../dist/checksums/src/checksums/md5 + proc trimUnitSep(x: var string) = let L = x.len if L > 0 and x[^1] == '\31': @@ -29,6 +34,7 @@ var backendLogging = true var simulate = false var optVerbose = false var useMegatest = true +var valgrindEnabled = true proc verboseCmd(cmd: string) = if optVerbose: @@ -60,6 +66,7 @@ Options: --colors:on|off Turn messages coloring on|off. --backendLogging:on|off Disable or enable backend logging. By default turned on. --megatest:on|off Enable or disable megatest. Default is on. + --valgrind:on|off Enable or disable valgrind support. Default is on. --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API @@ -189,7 +196,6 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, foundSuccessMsg = true elif not running(p): break - close(p) result.msg = "" result.file = "" result.output = "" @@ -197,8 +203,9 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, result.column = 0 result.err = reNimcCrash - let exitCode = p.peekExitCode - case exitCode + result.exitCode = p.peekExitCode + close p + case result.exitCode of 0: if foundErrorMsg: result.debugInfo.add " compiler exit code was 0 but some Error's were found." @@ -210,7 +217,7 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, if foundSuccessMsg: result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true." else: - result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $exitCode + result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode if err =~ pegLineError: result.file = extractFilename(matches[0]) @@ -376,7 +383,7 @@ proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer) elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: - r.addResult(test, target, extraOptions, expected.filename, given.file, reFilesDiffer) + r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer) elif expected.line != given.line and expected.line != 0 or expected.column != given.column and expected.column != 0: r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column, @@ -487,7 +494,7 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, args = @["--unhandled-rejections=strict", exeFile] & args else: exeCmd = exeFile.dup(normalizeExe) - if expected.useValgrind != disabled: + if valgrindEnabled and expected.useValgrind != disabled: var valgrindOptions = @["--error-exitcode=1"] if expected.useValgrind != leaking: valgrindOptions.add "--leak-check=yes" @@ -517,7 +524,29 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer) compilerOutputTests(test, target, extraOptions, given, expected, r) of actionReject: + # Make sure its the compiler rejecting and not the system (e.g. segfault) cmpMsgs(r, expected, given, test, target, extraOptions) + if given.exitCode != QuitFailure: + r.addResult(test, target, extraOptions, "exitcode: " & $QuitFailure, + "exitcode: " & $given.exitCode & "\n\nOutput:\n" & + given.nimout, reExitcodesDiffer) + + + +proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget = + result = defaultTarget + var p = parseopt.initOptParser(extraOptions) + + while true: + parseopt.next(p) + case p.kind + of cmdEnd: break + of cmdLongOption, cmdShortOption: + if p.key == "b" or p.key == "backend": + result = parseEnum[TTarget](p.val.normalize) + # chooses the last one + else: + discard proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) = for target in expected.targets: @@ -531,6 +560,7 @@ proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: s else: let nimcache = nimcacheDir(test.name, test.options, target) var testClone = test + let target = changeTarget(extraOptions, target) testSpecHelper(r, testClone, expected, target, extraOptions, nimcache) proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = @@ -664,6 +694,14 @@ proc main() = useMegatest = false else: quit Usage + of "valgrind": + case p.val: + of "on": + valgrindEnabled = true + of "off": + valgrindEnabled = false + else: + quit Usage of "backendlogging": case p.val: of "on": |