diff options
Diffstat (limited to 'testament/testament.nim')
| -rw-r--r-- | testament/testament.nim | 543 |
1 files changed, 287 insertions, 256 deletions
diff --git a/testament/testament.nim b/testament/testament.nim index 109ce4ed9..1e892e636 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -10,88 +10,114 @@ ## This program verifies Nim against the testcases. import - strutils, pegs, os, osproc, streams, json, - backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, times, md5, sequtils, azure + std/[strutils, pegs, os, osproc, streams, json, + parseopt, browsers, terminal, exitprocs, + algorithm, times, intsets, macros] + +import backend, specs, azure, htmlgen + from std/sugar import dup import compiler/nodejs +import lib/stdtest/testutils +from lib/stdtest/specialpaths import splitTestFile +from std/private/gitutils import diffStrings + +import ../dist/checksums/src/checksums/md5 + +proc trimUnitSep(x: var string) = + let L = x.len + if L > 0 and x[^1] == '\31': + setLen x, L-1 var useColors = true var backendLogging = true var simulate = false +var optVerbose = false +var useMegatest = true +var valgrindEnabled = true + +proc verboseCmd(cmd: string) = + if optVerbose: + echo "executing: ", cmd const failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs testsDir = "tests" & DirSep resultsFile = "testresults.html" - #jsonFile = "testresults.json" # not used Usage = """Usage: testament [options] command [arguments] Command: p|pat|pattern <glob> run all the tests matching the given pattern - all run all tests + all run all tests in category folders c|cat|category <category> run all the tests of a certain category r|run <test> run single test file html generate $1 from the database - stats generate statistics about test cases Arguments: arguments are passed to the compiler Options: - --print also print results to the console + --print print results to the console + --verbose print commands (compiling and running tests) --simulate see what tests would be run but don't run them (for debugging) --failing only show failing/ignored tests - --targets:"c c++ js objc" run tests for specified targets (default: all) + --targets:"c cpp js objc" run tests for specified targets (default: c) --nim:path use a particular nim executable (default: $$PATH/nim) --directory:dir Change to directory dir before reading the tests or doing anything else. --colors:on|off Turn messages coloring on|off. --backendLogging:on|off Disable or enable backend logging. By default turned on. --megatest:on|off Enable or disable megatest. Default is on. + --valgrind:on|off Enable or disable valgrind support. Default is on. --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN. + +Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables +tests with remote networking (as in CI). """ % resultsFile +proc isNimRepoTests(): bool = + # this logic could either be specific to cwd, or to some file derived from + # the input file, eg testament r /pathto/tests/foo/tmain.nim; we choose + # the former since it's simpler and also works with `testament all`. + let file = "testament"/"testament.nim.cfg" + result = file.fileExists + type Category = distinct string TResults = object - total, passed, skipped: int + total, passed, failedButAllowed, skipped: int + ## xxx rename passed to passedOrAllowedFailure data: string - TTest = object name: string cat: Category options: string - args: seq[string] + testArgs: seq[string] spec: TSpec startTime: float + debugInfo: string # ---------------------------------------------------------------------------- let pegLineError = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" - pegLineTemplate = - peg""" - {[^(]*} '(' {\d+} ', ' {\d+} ') ' - 'template/generic instantiation' ( ' of `' [^`]+ '`' )? ' from here' .* - """ pegOtherError = peg"'Error:' \s* {.*}" pegOfInterest = pegLineError / pegOtherError var gTargets = {low(TTarget)..high(TTarget)} +var targetsSet = false proc isSuccess(input: string): bool = # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs + # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we + # could handle colors, there would be other issues such as handling other flags + # that may appear in user config (eg: `--filenames`). + # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config + # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory input.startsWith("Hint: ") and input.endsWith("[SuccessX]") -proc normalizeMsg(s: string): string = - result = newStringOfCap(s.len+1) - for x in splitLines(s): - if result.len > 0: result.add '\L' - result.add x.strip - proc getFileDir(filename: string): string = result = filename.splitFile().dir if not result.isAbsolute(): @@ -99,7 +125,7 @@ proc getFileDir(filename: string): string = proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: string = ""): tuple[ cmdLine: string, - output: TaintedString, + output: string, exitCode: int] {.tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = @@ -107,8 +133,9 @@ proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: str for arg in args: result.cmdLine.add ' ' result.cmdLine.add quoteShell(arg) - var p = startProcess(command, workingDir=workingDir, args=args, - options={poStdErrToStdOut, poUsePath}) + verboseCmd(result.cmdLine) + var p = startProcess(command, workingDir = workingDir, args = args, + options = {poStdErrToStdOut, poUsePath}) var outp = outputStream(p) # There is no way to provide input for the child process @@ -118,12 +145,12 @@ proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: str instream.write(input) close instream - result.exitCode = -1 - var line = newStringOfCap(120).TaintedString + result.exitCode = -1 + var line = newStringOfCap(120) while true: if outp.readLine(line): - result.output.string.add(line.string) - result.output.string.add("\n") + result.output.add line + result.output.add '\n' else: result.exitCode = peekExitCode(p) if result.exitCode != -1: break @@ -134,56 +161,64 @@ proc nimcacheDir(filename, options: string, target: TTarget): string = let hashInput = options & $target result = "nimcache" / (filename & '_' & hashInput.getMD5) -proc prepareTestArgs(cmdTemplate, filename, options, nimcache: string, - target: TTarget, extraOptions = ""): seq[string] = - var options = target.defaultOptions & " " & options - # improve pending https://github.com/nim-lang/Nim/issues/14343 - if nimcache.len > 0: options.add " " & ("--nimCache:" & nimcache).quoteShell - options.add " " & extraOptions - result = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], +proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): string = + var options = target.defaultOptions & ' ' & options + if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell) + options.add ' ' & extraOptions + # we avoid using `parseCmdLine` which is buggy, refs bug #14343 + result = cmdTemplate % ["target", targetToCmd[target], "options", options, "file", filename.quoteShell, - "filedir", filename.getFileDir()]) + "filedir", filename.getFileDir(), "nim", compilerPrefix] -proc callCompiler(cmdTemplate, filename, options, nimcache: string, - target: TTarget, extraOptions = ""): TSpec = - let c = prepareTestArgs(cmdTemplate, filename, options, nimcache, target, +proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): TSpec = + result.cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache, target, extraOptions) - result.cmd = quoteShellCommand(c) - var p = startProcess(command=c[0], args=c[1 .. ^1], - options={poStdErrToStdOut, poUsePath}) + verboseCmd(result.cmd) + var p = startProcess(command = result.cmd, + options = {poStdErrToStdOut, poUsePath, poEvalCommand}) let outp = p.outputStream - var suc = "" + var foundSuccessMsg = false + var foundErrorMsg = false var err = "" - var tmpl = "" var x = newStringOfCap(120) result.nimout = "" while true: - if outp.readLine(x.TaintedString): - result.nimout.add(x & "\n") + if outp.readLine(x): + trimUnitSep x + result.nimout.add(x & '\n') if x =~ pegOfInterest: - # `err` should contain the last error/warning message + # `err` should contain the last error message err = x - elif x =~ pegLineTemplate and err == "": - # `tmpl` contains the last template expansion before the error - tmpl = x + foundErrorMsg = true elif x.isSuccess: - suc = x + foundSuccessMsg = true elif not running(p): break - close(p) result.msg = "" result.file = "" result.output = "" result.line = 0 result.column = 0 - result.tfile = "" - result.tline = 0 - result.tcolumn = 0 + result.err = reNimcCrash - if tmpl =~ pegLineTemplate: - result.tfile = extractFilename(matches[0]) - result.tline = parseInt(matches[1]) - result.tcolumn = parseInt(matches[2]) + result.exitCode = p.peekExitCode + close p + case result.exitCode + of 0: + if foundErrorMsg: + result.debugInfo.add " compiler exit code was 0 but some Error's were found." + else: + result.err = reSuccess + of 1: + if not foundErrorMsg: + result.debugInfo.add " compiler exit code was 1 but no Error's were found." + if foundSuccessMsg: + result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true." + else: + result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode + if err =~ pegLineError: result.file = extractFilename(matches[0]) result.line = parseInt(matches[1]) @@ -191,38 +226,15 @@ proc callCompiler(cmdTemplate, filename, options, nimcache: string, result.msg = matches[3] elif err =~ pegOtherError: result.msg = matches[0] - elif suc.isSuccess: - result.err = reSuccess - -proc callCCompiler(cmdTemplate, filename, options: string, - target: TTarget): TSpec = - let c = prepareTestArgs(cmdTemplate, filename, options, nimcache = "", target) - var p = startProcess(command="gcc", args=c[5 .. ^1], - options={poStdErrToStdOut, poUsePath}) - let outp = p.outputStream - var x = newStringOfCap(120) - result.nimout = "" - result.msg = "" - result.file = "" - result.output = "" - result.line = -1 - while true: - if outp.readLine(x.TaintedString): - result.nimout.add(x & "\n") - elif not running(p): - break - close(p) - if p.peekExitCode == 0: - result.err = reSuccess + trimUnitSep result.msg proc initResults: TResults = result.total = 0 result.passed = 0 + result.failedButAllowed = 0 result.skipped = 0 result.data = "" -import macros - macro ignoreStyleEcho(args: varargs[typed]): untyped = let typForegroundColor = bindSym"ForegroundColor".getType let typBackgroundColor = bindSym"BackgroundColor".getType @@ -247,18 +259,29 @@ template maybeStyledEcho(args: varargs[untyped]): untyped = proc `$`(x: TResults): string = - result = ("Tests passed: $1 / $3 <br />\n" & - "Tests skipped: $2 / $3 <br />\n") % - [$x.passed, $x.skipped, $x.total] + result = """ +Tests passed or allowed to fail: $2 / $1 <br /> +Tests failed and allowed to fail: $3 / $1 <br /> +Tests skipped: $4 / $1 <br /> +""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] + +proc testName(test: TTest, target: TTarget, extraOptions: string, allowFailure: bool): string = + var name = test.name.replace(DirSep, '/') + name.add ' ' & $target + if allowFailure: + name.add " (allowed to fail) " + if test.options.len > 0: name.add ' ' & test.options + if extraOptions.len > 0: name.add ' ' & extraOptions + name.strip() proc addResult(r: var TResults, test: TTest, target: TTarget, - expected, given: string, successOrig: TResultEnum) = + extraOptions, expected, given: string, successOrig: TResultEnum, + allowFailure = false, givenSpec: ptr TSpec = nil) = + # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need + # instead of having to pass individual fields, or abusing existing ones like expected vs given. # test.name is easier to find than test.name.extractFilename # A bit hacky but simple and works with tests/testament/tshould_not_work.nim - var name = test.name.replace(DirSep, '/') - name.add " " & $target - if test.options.len > 0: name.add " " & test.options - + let name = testName(test, target, extraOptions, allowFailure) let duration = epochTime() - test.startTime let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout else: successOrig @@ -273,18 +296,22 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, expected = expected, given = given) r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) + template dispNonSkipped(color, outcome) = + maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)" template disp(msg) = - maybeStyledEcho styleDim, fgYellow, msg & " ", styleBright, fgCyan, name + maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name if success == reSuccess: - maybeStyledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)" + dispNonSkipped(fgGreen, "PASS: ") elif success == reDisabled: if test.spec.inCurrentBatch: disp("SKIP:") else: disp("NOTINBATCH:") elif success == reJoined: disp("JOINED:") else: - maybeStyledEcho styleBright, fgRed, failString, fgCyan, name + dispNonSkipped(fgRed, failString) maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" maybeStyledEcho styleBright, fgRed, "Failure: ", $success + if givenSpec != nil and givenSpec.debugInfo.len > 0: + echo "debugInfo: " & givenSpec.debugInfo if success in {reBuildFailed, reNimcCrash, reInstallFailed}: # expected is empty, no reason to print it. echo given @@ -293,7 +320,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, maybeStyledEcho styleBright, expected, "\n" maybeStyledEcho fgYellow, "Gotten:" maybeStyledEcho styleBright, given, "\n" - + echo diffStrings(expected, given).output if backendLogging and (isAppVeyor or isAzure): let (outcome, msg) = @@ -303,44 +330,66 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, of reDisabled, reJoined: ("Skipped", "") of reBuildFailed, reNimcCrash, reInstallFailed: - ("Failed", "Failure: " & $success & "\n" & given) + ("Failed", "Failure: " & $success & '\n' & given) else: ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given) if isAzure: azure.addTestResult(name, test.cat.string, int(duration * 1000), msg, success) else: - var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options, + var p = startProcess("appveyor", args = ["AddTest", test.name.replace("\\", "/") & test.options, "-Framework", "nim-testament", "-FileName", test.cat.string, "-Outcome", outcome, "-ErrorMessage", msg, - "-Duration", $(duration*1000).int], - options={poStdErrToStdOut, poUsePath, poParentStreams}) + "-Duration", $(duration * 1000).int], + options = {poStdErrToStdOut, poUsePath, poParentStreams}) discard waitForExit(p) close(p) -proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = - if strip(expected.msg) notin strip(given.msg): - r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer) - elif expected.nimout.len > 0 and expected.nimout.normalizeMsg notin given.nimout.normalizeMsg: - r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer) - elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and +proc toString(inlineError: InlineError, filename: string): string = + result.add "$file($line, $col) $kind: $msg" % [ + "file", filename, + "line", $inlineError.line, + "col", $inlineError.col, + "kind", $inlineError.kind, + "msg", $inlineError.msg + ] + +proc inlineErrorsMsgs(expected: TSpec): string = + for inlineError in expected.inlineErrors.items: + result.addLine inlineError.toString(expected.filename) + +proc checkForInlineErrors(expected, given: TSpec): bool = + for inlineError in expected.inlineErrors: + if inlineError.toString(expected.filename) notin given.nimout: + return false + true + +proc nimoutCheck(expected, given: TSpec): bool = + result = true + if expected.nimoutFull: + if expected.nimout != given.nimout: + result = false + elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): + result = false + +proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, + target: TTarget, extraOptions: string) = + if not checkForInlineErrors(expected, given) or + (not expected.nimoutFull and not nimoutCheck(expected, given)): + r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer) + elif strip(expected.msg) notin strip(given.msg): + r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer) + elif not nimoutCheck(expected, given): + r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer) + elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: - r.addResult(test, target, expected.file, given.file, reFilesDiffer) + r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer) elif expected.line != given.line and expected.line != 0 or expected.column != given.column and expected.column != 0: - r.addResult(test, target, $expected.line & ':' & $expected.column, - $given.line & ':' & $given.column, - reLinesDiffer) - elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and - "internal error:" notin expected.msg: - r.addResult(test, target, expected.tfile, given.tfile, reFilesDiffer) - elif expected.tline != given.tline and expected.tline != 0 or - expected.tcolumn != given.tcolumn and expected.tcolumn != 0: - r.addResult(test, target, $expected.tline & ':' & $expected.tcolumn, - $given.tline & ':' & $given.tcolumn, - reLinesDiffer) + r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column, + $given.line & ':' & $given.column, reLinesDiffer) else: - r.addResult(test, target, expected.msg, given.msg, reSuccess) + r.addResult(test, target, extraOptions, expected.msg, given.msg, reSuccess) inc(r.passed) proc generatedFile(test: TTest, target: TTarget): string = @@ -358,10 +407,9 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st given: var TSpec) = try: let genFile = generatedFile(test, target) - let contents = readFile(genFile).string - let check = spec.ccodeCheck - if check.len > 0: - if check[0] == '\\': + let contents = readFile(genFile) + for check in spec.ccodeCheck: + if check.len > 0 and check[0] == '\\': # little hack to get 'match' support: if not contents.match(check.peg): given.err = reCodegenFailure @@ -379,85 +427,78 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st given.err = reCodeNotFound echo getCurrentExceptionMsg() -proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) = - let giv = given.nimout.strip - var currentPos = 0 - # Only check that nimout contains all expected lines in that order. - # There may be more output in nimout. It is ignored here. - for line in expectedNimout.strip.splitLines: - currentPos = giv.find(line.strip, currentPos) - if currentPos < 0: - given.err = reMsgsDiffer - break - -proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, - expected: TSpec; r: var TResults) = +proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string, + given: var TSpec, expected: TSpec; r: var TResults) = var expectedmsg: string = "" var givenmsg: string = "" if given.err == reSuccess: if expected.needsCodegenCheck: codegenCheck(test, target, expected, expectedmsg, given) givenmsg = given.msg - if expected.nimout.len > 0: - expectedmsg = expected.nimout + if not nimoutCheck(expected, given) or + not checkForInlineErrors(expected, given): + given.err = reMsgsDiffer + expectedmsg = expected.nimout & inlineErrorsMsgs(expected) givenmsg = given.nimout.strip - nimoutCheck(test, expectedmsg, given) else: - givenmsg = "$ " & given.cmd & "\n" & given.nimout + givenmsg = "$ " & given.cmd & '\n' & given.nimout if given.err == reSuccess: inc(r.passed) - r.addResult(test, target, expectedmsg, givenmsg, given.err) + r.addResult(test, target, extraOptions, expectedmsg, givenmsg, given.err) proc getTestSpecTarget(): TTarget = - if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true": + if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": result = targetCpp else: result = targetC -proc checkDisabled(r: var TResults, test: TTest): bool = - if test.spec.err in {reDisabled, reJoined}: - # targetC is a lie, but parameter is required - r.addResult(test, targetC, "", "", test.spec.err) - inc(r.skipped) - inc(r.total) - result = false - else: - result = true - var count = 0 -proc testSpecHelper(r: var TResults, test: TTest, expected: TSpec, - target: TTarget, nimcache: string, extraOptions = "") = +proc equalModuloLastNewline(a, b: string): bool = + # allow lazy output spec that omits last newline, but really those should be fixed instead + result = a == b or b.endsWith("\n") and a == b[0 ..< ^1] + +proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, + target: TTarget, extraOptions: string, nimcache: string) = + test.startTime = epochTime() + if testName(test, target, extraOptions, false) in skips: + test.spec.err = reDisabled + + if test.spec.err in {reDisabled, reJoined}: + r.addResult(test, target, extraOptions, "", "", test.spec.err) + inc(r.skipped) + return + var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions) case expected.action of actionCompile: - var given = callCompiler(expected.getCmd, test.name, test.options, nimcache, target, - extraOptions = " --stdout --hint[Path]:off --hint[Processing]:off") - compilerOutputTests(test, target, given, expected, r) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionRun: - var given = callCompiler(expected.getCmd, test.name, test.options, - nimcache, target, extraOptions) if given.err != reSuccess: - r.addResult(test, target, "", "$ " & given.cmd & "\n" & given.nimout, given.err) + r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr) else: let isJsTarget = target == targetJS var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt) if not fileExists(exeFile): - r.addResult(test, target, expected.output, + r.addResult(test, target, extraOptions, expected.output, "executable not found: " & exeFile, reExeNotFound) else: let nodejs = if isJsTarget: findNodeJs() else: "" if isJsTarget and nodejs == "": - r.addResult(test, target, expected.output, "nodejs binary not in PATH", + r.addResult(test, target, extraOptions, expected.output, "nodejs binary not in PATH", reExeNotFound) else: var exeCmd: string - var args = test.args + var args = test.testArgs if isJsTarget: exeCmd = nodejs - args = concat(@[exeFile], args) + # see D20210217T215950 + args = @["--unhandled-rejections=strict", exeFile] & args else: exeCmd = exeFile.dup(normalizeExe) - if expected.useValgrind: - args = @["--error-exitcode=1"] & exeCmd & args + if valgrindEnabled and expected.useValgrind != disabled: + var valgrindOptions = @["--error-exitcode=1"] + if expected.useValgrind != leaking: + valgrindOptions.add "--leak-check=yes" + args = valgrindOptions & exeCmd & args exeCmd = "valgrind" var (_, buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input) # Treat all failure codes from nodejs as 1. Older versions of nodejs used @@ -465,47 +506,70 @@ proc testSpecHelper(r: var TResults, test: TTest, expected: TSpec, if exitCode != 0: exitCode = 1 let bufB = if expected.sortoutput: - var x = splitLines(strip(buf.string)) + var buf2 = buf + buf2.stripLineEnd + var x = splitLines(buf2) sort(x, system.cmp) - join(x, "\n") + join(x, "\n") & '\n' else: - strip(buf.string) + buf if exitCode != expected.exitCode: - r.addResult(test, target, "exitcode: " & $expected.exitCode, + given.err = reExitcodesDiffer + r.addResult(test, target, extraOptions, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode & "\n\nOutput:\n" & bufB, reExitcodesDiffer) - elif (expected.outputCheck == ocEqual and expected.output != bufB) or + elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or (expected.outputCheck == ocSubstr and expected.output notin bufB): given.err = reOutputsDiffer - r.addResult(test, target, expected.output, bufB, reOutputsDiffer) - else: - compilerOutputTests(test, target, given, expected, r) + r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionReject: - var given = callCompiler(expected.getCmd, test.name, test.options, - nimcache, target) - cmpMsgs(r, expected, given, test, target) + # Make sure its the compiler rejecting and not the system (e.g. segfault) + cmpMsgs(r, expected, given, test, target, extraOptions) + if given.exitCode != QuitFailure: + r.addResult(test, target, extraOptions, "exitcode: " & $QuitFailure, + "exitcode: " & $given.exitCode & "\n\nOutput:\n" & + given.nimout, reExitcodesDiffer) + + -proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions = "") = +proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget = + result = defaultTarget + var p = parseopt.initOptParser(extraOptions) + + while true: + parseopt.next(p) + case p.kind + of cmdEnd: break + of cmdLongOption, cmdShortOption: + if p.key == "b" or p.key == "backend": + result = parseEnum[TTarget](p.val.normalize) + # chooses the last one + else: + discard + +proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) = for target in expected.targets: inc(r.total) if target notin gTargets: - r.addResult(test, target, "", "", reDisabled) + r.addResult(test, target, extraOptions, "", "", reDisabled) inc(r.skipped) elif simulate: inc count echo "testSpec count: ", count, " expected: ", expected else: let nimcache = nimcacheDir(test.name, test.options, target) - testSpecHelper(r, test, expected, target, nimcache, extraOptions) + var testClone = test + let target = changeTarget(extraOptions, target) + testSpecHelper(r, testClone, expected, target, extraOptions, nimcache) proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = var expected = test.spec if expected.parseErrors.len > 0: # targetC is a lie, but a parameter is required - r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec) + r.addResult(test, targetC, "", "", expected.parseErrors, reInvalidSpec) inc(r.total) return - if not checkDisabled(r, test): return expected.targets.incl targets # still no target specified at all @@ -515,45 +579,13 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = for m in test.spec.matrix: targetHelper(r, test, expected, m) else: - targetHelper(r, test, expected) + targetHelper(r, test, expected, "") -proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) = - if not checkDisabled(r, test): return +proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} = for target in test.spec.targets: inc(r.total) - testSpecHelper(r, test, test.spec, target, nimcache) - -proc testC(r: var TResults, test: TTest, action: TTestAction) = - # runs C code. Doesn't support any specs, just goes by exit code. - if not checkDisabled(r, test): return - - let tname = test.name.addFileExt(".c") - inc(r.total) - maybeStyledEcho "Processing ", fgCyan, extractFilename(tname) - var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC) - if given.err != reSuccess: - r.addResult(test, targetC, "", given.msg, given.err) - elif action == actionRun: - let exeFile = changeFileExt(test.name, ExeExt) - var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath}) - if exitCode != 0: given.err = reExitcodesDiffer - if given.err == reSuccess: inc(r.passed) - -proc testExec(r: var TResults, test: TTest) = - # runs executable or script, just goes by exit code - if not checkDisabled(r, test): return - - inc(r.total) - let (outp, errC) = execCmdEx(test.options.strip()) - var given: TSpec - if errC == 0: - given.err = reSuccess - else: - given.err = reExitcodesDiffer - given.msg = outp.string - - if given.err == reSuccess: inc(r.passed) - r.addResult(test, targetC, "", given.msg, given.err) + var testClone = test + testSpecHelper(r, testClone, test.spec, target, "", nimcache) proc makeTest(test, options: string, cat: Category): TTest = result.cat = cat @@ -562,19 +594,17 @@ proc makeTest(test, options: string, cat: Category): TTest = result.spec = parseSpec(addFileExt(test, ".nim")) result.startTime = epochTime() -proc makeRawTest(test, options: string, cat: Category): TTest = +proc makeRawTest(test, options: string, cat: Category): TTest {.used.} = result.cat = cat result.name = test result.options = options result.spec = initSpec(addFileExt(test, ".nim")) - result.startTime = epochTime() result.spec.action = actionCompile result.spec.targets = {getTestSpecTarget()} + result.startTime = epochTime() # TODO: fix these files const disabledFilesDefault = @[ - "LockFreeHash.nim", - "sharedstrings.nim", "tableimpl.nim", "setimpl.nim", "hashcommon.nim", @@ -609,13 +639,10 @@ proc loadSkipFrom(name: string): seq[string] = # used by `nlvm` (at least) for line in lines(name): let sline = line.strip() - if sline.len > 0 and not sline.startsWith("#"): + if sline.len > 0 and not sline.startsWith('#'): result.add sline proc main() = - os.putEnv "NIMTEST_COLOR", "never" - os.putEnv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES" - azure.init() backend.open() var optPrintResults = false @@ -623,24 +650,25 @@ proc main() = var targetsStr = "" var isMainProcess = true var skipFrom = "" - var useMegatest = true var p = initOptParser() p.next() while p.kind in {cmdLongOption, cmdShortOption}: - case p.key.string.normalize - of "print", "verbose": optPrintResults = true + case p.key.normalize + of "print": optPrintResults = true + of "verbose": optVerbose = true of "failing": optFailing = true - of "pedantic": discard "now always enabled" + of "pedantic": discard # deadcode refs https://github.com/nim-lang/Nim/issues/16731 of "targets": - targetsStr = p.val.string + targetsStr = p.val gTargets = parseTargets(targetsStr) + targetsSet = true of "nim": - compilerPrefix = addFileExt(p.val.string, ExeExt) + compilerPrefix = addFileExt(p.val.absolutePath, ExeExt) of "directory": - setCurrentDir(p.val.string) + setCurrentDir(p.val) of "colors": - case p.val.string: + case p.val: of "on": useColors = true of "off": @@ -649,7 +677,7 @@ proc main() = quit Usage of "batch": testamentData0.batchArg = p.val - if p.val != "_": + if p.val != "_" and p.val.len > 0 and p.val[0] in {'0'..'9'}: let s = p.val.split("_") doAssert s.len == 2, $(p.val, s) testamentData0.testamentBatch = s[0].parseInt @@ -659,15 +687,23 @@ proc main() = of "simulate": simulate = true of "megatest": - case p.val.string: + case p.val: of "on": useMegatest = true of "off": useMegatest = false else: quit Usage + of "valgrind": + case p.val: + of "on": + valgrindEnabled = true + of "off": + valgrindEnabled = false + else: + quit Usage of "backendlogging": - case p.val.string: + case p.val: of "on": backendLogging = true of "off": @@ -675,18 +711,18 @@ proc main() = else: quit Usage of "skipfrom": - skipFrom = p.val.string + skipFrom = p.val else: quit Usage p.next() if p.kind != cmdArgument: quit Usage - var action = p.key.string.normalize + var action = p.key.normalize p.next() var r = initResults() case action of "all": - #processCategory(r, Category"megatest", p.cmdLineRest.string, testsDir, runJoinableTests = false) + #processCategory(r, Category"megatest", p.cmdLineRest, testsDir, runJoinableTests = false) var myself = quoteShell(getAppFilename()) if targetsStr.len > 0: @@ -700,13 +736,14 @@ proc main() = myself &= " " & quoteShell("--skipFrom:" & skipFrom) var cats: seq[string] - let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: "" + let rest = if p.cmdLineRest.len > 0: " " & p.cmdLineRest else: "" for kind, dir in walkDir(testsDir): assert testsDir.startsWith(testsDir) let cat = dir[testsDir.len .. ^1] if kind == pcDir and cat notin ["testdata", "nimcache"]: cats.add cat - cats.add AdditionalCategories + if isNimRepoTests(): + cats.add AdditionalCategories if useMegatest: cats.add MegaTestCat var cmds: seq[string] @@ -721,14 +758,14 @@ proc main() = skips = loadSkipFrom(skipFrom) for i, cati in cats: progressStatus(i) - processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, runJoinableTests = false) + processCategory(r, Category(cati), p.cmdLineRest, testsDir, runJoinableTests = false) else: - addQuitProc azure.finalize + addExitProc azure.finalize quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus) of "c", "cat", "category": skips = loadSkipFrom(skipFrom) var cat = Category(p.key) - processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true) + processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = true) of "pcat": skips = loadSkipFrom(skipFrom) # 'pcat' is used for running a category in parallel. Currently the only @@ -737,21 +774,15 @@ proc main() = isMainProcess = false var cat = Category(p.key) p.next - processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = false) + processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = false) of "p", "pat", "pattern": skips = loadSkipFrom(skipFrom) let pattern = p.key p.next - processPattern(r, pattern, p.cmdLineRest.string, simulate) + processPattern(r, pattern, p.cmdLineRest, simulate) of "r", "run": - var subPath = p.key.string - if subPath.isAbsolute: subPath = subPath.relativePath(getCurrentDir()) - # at least one directory is required in the path, to use as a category name - let pathParts = split(subPath, {DirSep, AltSep}) - # "stdlib/nre/captures.nim" -> "stdlib" + "nre/captures.nim" - let cat = Category(pathParts[0]) - subPath = joinPath(pathParts[1..^1]) - processSingleTest(r, cat, p.cmdLineRest.string, subPath) + let (cat, path) = splitTestFile(p.key) + processSingleTest(r, cat.Category, p.cmdLineRest, path, gTargets, targetsSet) of "html": generateHtml(resultsFile, optFailing) else: |