diff options
21 files changed, 64 insertions, 39 deletions
diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index 241ab1770..6d8620dc7 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -29,11 +29,17 @@ template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = when not defined(js): import std/strutils - proc greedyOrderedSubsetLines*(lhs, rhs: string): bool = + proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool = ## Returns true if each stripped line in `lhs` appears in rhs, using a greedy matching. + # xxx improve error reporting by showing the last matched pair iterator splitLinesClosure(): string {.closure.} = for line in splitLines(rhs.strip): yield line + template isMatch(lhsi, rhsi): bool = + if allowPrefixMatch: + startsWith(rhsi, lhsi): + else: + lhsi == rhsi var rhsIter = splitLinesClosure var currentLine = strip(rhsIter()) @@ -41,7 +47,7 @@ when not defined(js): for line in lhs.strip.splitLines: let line = line.strip if line.len != 0: - while line != currentLine: + while not isMatch(line, currentLine): currentLine = strip(rhsIter()) if rhsIter.finished: return false diff --git a/testament/testament.nim b/testament/testament.nim index 0607ac41d..11c541890 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -288,16 +288,18 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, expected = expected, given = given) r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) + template dispNonSkipped(color, outcome) = + maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)" template disp(msg) = maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name if success == reSuccess: - maybeStyledEcho fgGreen, "PASS: ", fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)" + dispNonSkipped(fgGreen, "PASS: ") elif success == reDisabled: if test.spec.inCurrentBatch: disp("SKIP:") else: disp("NOTINBATCH:") elif success == reJoined: disp("JOINED:") else: - maybeStyledEcho styleBright, fgRed, failString, fgCyan, name + dispNonSkipped(fgRed, failString) maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" maybeStyledEcho styleBright, fgRed, "Failure: ", $success if success in {reBuildFailed, reNimcCrash, reInstallFailed}: diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim index 7b5f0cce6..477da1e23 100644 --- a/testament/tests/shouldfail/tccodecheck.nim +++ b/testament/tests/shouldfail/tccodecheck.nim @@ -1,5 +1,4 @@ discard """ - targets: "c" ccodecheck: "baz" """ diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim index b79ec52a4..809ddec74 100644 --- a/testament/tests/shouldfail/tcolumn.nim +++ b/testament/tests/shouldfail/tcolumn.nim @@ -1,6 +1,5 @@ discard """ errormsg: "undeclared identifier: 'undeclared'" - targets: "c" line: 9 column: 7 """ diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim index e69035235..6c130d107 100644 --- a/testament/tests/shouldfail/terrormsg.nim +++ b/testament/tests/shouldfail/terrormsg.nim @@ -1,6 +1,5 @@ discard """ errormsg: "wrong error message" - targets: "c" line: 9 column: 6 """ diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim index e5e061578..605f046db 100644 --- a/testament/tests/shouldfail/texitcode1.nim +++ b/testament/tests/shouldfail/texitcode1.nim @@ -1,4 +1,3 @@ discard """ - targets: "c" exitcode: 1 """ diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim index 9463882f9..b40a4f44f 100644 --- a/testament/tests/shouldfail/tfile.nim +++ b/testament/tests/shouldfail/tfile.nim @@ -1,5 +1,4 @@ discard """ - targets: "c" errormsg: "undeclared identifier: 'undefined'" file: "notthisfile.nim" """ diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim index 7f7e90896..fe782eb03 100644 --- a/testament/tests/shouldfail/tline.nim +++ b/testament/tests/shouldfail/tline.nim @@ -1,5 +1,4 @@ discard """ - targets: "c" errormsg: "undeclared identifier: 'undeclared'" line: 10 column: 6 diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim index 9e2bd9cfb..92022ee97 100644 --- a/testament/tests/shouldfail/tmaxcodesize.nim +++ b/testament/tests/shouldfail/tmaxcodesize.nim @@ -1,5 +1,4 @@ discard """ - targets: "c" maxcodesize: 1 """ diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim index 832f134b0..0a65bfb70 100644 --- a/testament/tests/shouldfail/tnimout.nim +++ b/testament/tests/shouldfail/tnimout.nim @@ -1,5 +1,4 @@ discard """ - targets: "c" nimout: "Hello World!" action: compile """ diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim index 3349ceedf..4fc93f6d2 100644 --- a/testament/tests/shouldfail/tnimoutfull.nim +++ b/testament/tests/shouldfail/tnimoutfull.nim @@ -1,5 +1,4 @@ discard """ - targets: "c" nimout: ''' msg1 msg2 diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim index 0fa4d7278..eaf9e8652 100644 --- a/testament/tests/shouldfail/toutput.nim +++ b/testament/tests/shouldfail/toutput.nim @@ -1,5 +1,4 @@ discard """ - targets: "c" output: ''' done ''' diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim index b34f3a8f2..47324ecee 100644 --- a/testament/tests/shouldfail/toutputsub.nim +++ b/testament/tests/shouldfail/toutputsub.nim @@ -1,6 +1,5 @@ discard """ outputsub: "something else" - targets: "c" """ echo "Hello World!" diff --git a/testament/tests/shouldfail/treject.nim b/testament/tests/shouldfail/treject.nim index 395dc4251..1e7258f70 100644 --- a/testament/tests/shouldfail/treject.nim +++ b/testament/tests/shouldfail/treject.nim @@ -1,6 +1,5 @@ discard """ action: "reject" - targets: "c" """ # Because we set action="reject", we expect this line not to compile. But the diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim index 7a2845029..69dfbc0a0 100644 --- a/testament/tests/shouldfail/tsortoutput.nim +++ b/testament/tests/shouldfail/tsortoutput.nim @@ -1,6 +1,5 @@ discard """ sortoutput: true - targets: "c" output: ''' 2 1 diff --git a/testament/tests/shouldfail/ttimeout.nim b/testament/tests/shouldfail/ttimeout.nim index 8ffd71aaa..fd3e1a598 100644 --- a/testament/tests/shouldfail/ttimeout.nim +++ b/testament/tests/shouldfail/ttimeout.nim @@ -1,6 +1,5 @@ discard """ timeout: "0.1" - targets: "c" """ import os diff --git a/testament/tests/shouldfail/tvalgrind.nim b/testament/tests/shouldfail/tvalgrind.nim index 5502705b3..d551ff12e 100644 --- a/testament/tests/shouldfail/tvalgrind.nim +++ b/testament/tests/shouldfail/tvalgrind.nim @@ -1,6 +1,5 @@ discard """ valgrind: true - targets: "c" cmd: "nim $target --gc:arc -d:useMalloc $options $file" """ diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 014373dfb..1b679d92c 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -6,6 +6,8 @@ discard """ ## tests that don't quite fit the mold and are easier to handle via `execCmdEx` ## A few others could be added to here to simplify code. ## Note: this test is a bit slow but tests a lot of things; please don't disable. +## Note: if needed, we could use `matrix: "-d:case1; -d:case2"` to split this +## into several independent tests while retaining the common test helpers. import std/[strformat,os,osproc,unittest,compilesettings] from std/sequtils import toSeq,mapIt diff --git a/tests/stdlib/tchannels.nim b/tests/stdlib/tchannels.nim index 33108c50c..d609eec78 100644 --- a/tests/stdlib/tchannels.nim +++ b/tests/stdlib/tchannels.nim @@ -1,5 +1,5 @@ discard """ - timeout: 5.0 # but typically < 1s + timeout: 20.0 # but typically < 1s (in isolation but other tests running in parallel can affect this since based on epochTime) disabled: "freebsd" matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger" """ diff --git a/tests/stdlib/ttestutils.nim b/tests/stdlib/ttestutils.nim index 0d789cd76..d24c5b39a 100644 --- a/tests/stdlib/ttestutils.nim +++ b/tests/stdlib/ttestutils.nim @@ -22,3 +22,18 @@ block: # greedyOrderedSubsetLines not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4\na5:suffix") not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\nprefix:a5") not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\na5:suffix") + +block: # greedyOrderedSubsetLines with allowPrefixMatch = true + template fn(a, b): bool = + greedyOrderedSubsetLines(a, b, allowPrefixMatch = true) + assertAll: + fn("a1\na3", "a0\na1\na2\na3_suffix\na4") + not fn("a1\na3", "a0\na1\na2\nprefix_a3\na4") + # these are same as above, could be refactored + not fn("a3\na1", "a0\na1\na2\na3\na4") # out of order + not fn("a1\na5", "a0\na1\na2\na3\na4") # a5 not in lhs + + not fn("a1\na5", "a0\na1\na2\na3\na4\nprefix:a5") + fn("a1\na5", "a0\na1\na2\na3\na4\na5:suffix") + not fn("a5", "a0\na1\na2\na3\na4\nprefix:a5") + fn("a5", "a0\na1\na2\na3\na4\na5:suffix") diff --git a/tests/testament/tshould_not_work.nim b/tests/testament/tshould_not_work.nim index 2777bfe95..e11944e8f 100644 --- a/tests/testament/tshould_not_work.nim +++ b/tests/testament/tshould_not_work.nim @@ -1,39 +1,54 @@ discard """ -cmd: "testament/testament --directory:testament --colors:off --backendLogging:off --nim:$nim category shouldfail" -action: compile -nimout: ''' -FAIL: tests/shouldfail/tccodecheck.nim c + joinable: false +""" + +const expected = """ +FAIL: tests/shouldfail/tccodecheck.nim Failure: reCodegenFailure Expected: baz -FAIL: tests/shouldfail/tcolumn.nim c +FAIL: tests/shouldfail/tcolumn.nim Failure: reLinesDiffer -FAIL: tests/shouldfail/terrormsg.nim c +FAIL: tests/shouldfail/terrormsg.nim Failure: reMsgsDiffer -FAIL: tests/shouldfail/texitcode1.nim c +FAIL: tests/shouldfail/texitcode1.nim Failure: reExitcodesDiffer -FAIL: tests/shouldfail/tfile.nim c +FAIL: tests/shouldfail/tfile.nim Failure: reFilesDiffer -FAIL: tests/shouldfail/tline.nim c +FAIL: tests/shouldfail/tline.nim Failure: reLinesDiffer -FAIL: tests/shouldfail/tmaxcodesize.nim c +FAIL: tests/shouldfail/tmaxcodesize.nim Failure: reCodegenFailure max allowed size: 1 -FAIL: tests/shouldfail/tnimout.nim c +FAIL: tests/shouldfail/tnimout.nim Failure: reMsgsDiffer -FAIL: tests/shouldfail/tnimoutfull.nim c +FAIL: tests/shouldfail/tnimoutfull.nim Failure: reMsgsDiffer -FAIL: tests/shouldfail/toutput.nim c +FAIL: tests/shouldfail/toutput.nim Failure: reOutputsDiffer -FAIL: tests/shouldfail/toutputsub.nim c +FAIL: tests/shouldfail/toutputsub.nim Failure: reOutputsDiffer -FAIL: tests/shouldfail/treject.nim c +FAIL: tests/shouldfail/treject.nim Failure: reFilesDiffer -FAIL: tests/shouldfail/tsortoutput.nim c +FAIL: tests/shouldfail/tsortoutput.nim Failure: reOutputsDiffer -FAIL: tests/shouldfail/ttimeout.nim c +FAIL: tests/shouldfail/ttimeout.nim Failure: reTimeout -FAIL: tests/shouldfail/tvalgrind.nim c +FAIL: tests/shouldfail/tvalgrind.nim Failure: reExitcodesDiffer -''' """ + +import std/[os,strformat,osproc] +import stdtest/testutils + +proc main = + const nim = getCurrentCompilerExe() + # TODO: bin/testament instead? like other tools (eg bin/nim, bin/nimsuggest etc) + let testamentExe = "testament/testament" + let cmd = fmt"{testamentExe} --directory:testament --colors:off --backendLogging:off --nim:{nim} category shouldfail" + let (outp, status) = execCmdEx(cmd) + doAssert status == 1, $status + + let ok = greedyOrderedSubsetLines(expected, outp, allowPrefixMatch = true) + doAssert ok, &"\nexpected:\n{expected}\noutp:\n{outp}" +main() |