testament :show duration also for failed tests; improve `tshould_not_work`; mitigate #17946 tchannels timeouts (#17947)

* refs #17946; refactor testament test summary, show test duration for failures; increase timeout tchannels

* revert workarounds from https://github.com/nim-lang/Nim/pull/16698 and add allowPrefixMatch optional param to greedyOrderedSubsetLines

* add test

* workaround for yet another testament bug

21 files changed, 64 insertions, 39 deletions

diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim
index 241ab1770..6d8620dc7 100644
--- a/testament/lib/stdtest/testutils.nim
+++ b/testament/lib/stdtest/testutils.nim
@@ -29,11 +29,17 @@ template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) =
 when not defined(js):
   import std/strutils
 
-  proc greedyOrderedSubsetLines*(lhs, rhs: string): bool =
+  proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool =
     ## Returns true if each stripped line in `lhs` appears in rhs, using a greedy matching.
+    # xxx improve error reporting by showing the last matched pair
     iterator splitLinesClosure(): string {.closure.} =
       for line in splitLines(rhs.strip):
         yield line
+    template isMatch(lhsi, rhsi): bool =
+      if allowPrefixMatch:
+        startsWith(rhsi, lhsi):
+      else:
+        lhsi == rhsi
 
     var rhsIter = splitLinesClosure
     var currentLine = strip(rhsIter())
@@ -41,7 +47,7 @@ when not defined(js):
     for line in lhs.strip.splitLines:
       let line = line.strip
       if line.len != 0:
-        while line != currentLine:
+        while not isMatch(line, currentLine):
           currentLine = strip(rhsIter())
           if rhsIter.finished:
             return false
diff --git a/testament/testament.nim b/testament/testament.nim
index 0607ac41d..11c541890 100644
--- a/testament/testament.nim
+++ b/testament/testament.nim
@@ -288,16 +288,18 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
                             expected = expected,
                             given = given)
   r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
+  template dispNonSkipped(color, outcome) =
+    maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)"
   template disp(msg) =
     maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name
   if success == reSuccess:
-    maybeStyledEcho fgGreen, "PASS: ", fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)"
+    dispNonSkipped(fgGreen, "PASS: ")
   elif success == reDisabled:
     if test.spec.inCurrentBatch: disp("SKIP:")
     else: disp("NOTINBATCH:")
   elif success == reJoined: disp("JOINED:")
   else:
-    maybeStyledEcho styleBright, fgRed, failString, fgCyan, name
+    dispNonSkipped(fgRed, failString)
     maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
     maybeStyledEcho styleBright, fgRed, "Failure: ", $success
     if success in {reBuildFailed, reNimcCrash, reInstallFailed}:
diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim
index 7b5f0cce6..477da1e23 100644
--- a/testament/tests/shouldfail/tccodecheck.nim
+++ b/testament/tests/shouldfail/tccodecheck.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   ccodecheck: "baz"
 """
 
diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim
index b79ec52a4..809ddec74 100644
--- a/testament/tests/shouldfail/tcolumn.nim
+++ b/testament/tests/shouldfail/tcolumn.nim
@@ -1,6 +1,5 @@
 discard """
   errormsg: "undeclared identifier: 'undeclared'"
-  targets: "c"
   line: 9
   column: 7
 """
diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim
index e69035235..6c130d107 100644
--- a/testament/tests/shouldfail/terrormsg.nim
+++ b/testament/tests/shouldfail/terrormsg.nim
@@ -1,6 +1,5 @@
 discard """
   errormsg: "wrong error message"
-  targets: "c"
   line: 9
   column: 6
 """
diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim
index e5e061578..605f046db 100644
--- a/testament/tests/shouldfail/texitcode1.nim
+++ b/testament/tests/shouldfail/texitcode1.nim
@@ -1,4 +1,3 @@
 discard """
-  targets: "c"
   exitcode: 1
 """
diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim
index 9463882f9..b40a4f44f 100644
--- a/testament/tests/shouldfail/tfile.nim
+++ b/testament/tests/shouldfail/tfile.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   errormsg: "undeclared identifier: 'undefined'"
   file: "notthisfile.nim"
 """
diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim
index 7f7e90896..fe782eb03 100644
--- a/testament/tests/shouldfail/tline.nim
+++ b/testament/tests/shouldfail/tline.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   errormsg: "undeclared identifier: 'undeclared'"
   line: 10
   column: 6
diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim
index 9e2bd9cfb..92022ee97 100644
--- a/testament/tests/shouldfail/tmaxcodesize.nim
+++ b/testament/tests/shouldfail/tmaxcodesize.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   maxcodesize: 1
 """
 
diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim
index 832f134b0..0a65bfb70 100644
--- a/testament/tests/shouldfail/tnimout.nim
+++ b/testament/tests/shouldfail/tnimout.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   nimout: "Hello World!"
   action: compile
 """
diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim
index 3349ceedf..4fc93f6d2 100644
--- a/testament/tests/shouldfail/tnimoutfull.nim
+++ b/testament/tests/shouldfail/tnimoutfull.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   nimout: '''
 msg1
 msg2
diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim
index 0fa4d7278..eaf9e8652 100644
--- a/testament/tests/shouldfail/toutput.nim
+++ b/testament/tests/shouldfail/toutput.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   output: '''
   done
   '''
diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim
index b34f3a8f2..47324ecee 100644
--- a/testament/tests/shouldfail/toutputsub.nim
+++ b/testament/tests/shouldfail/toutputsub.nim
@@ -1,6 +1,5 @@
 discard """
   outputsub: "something else"
-  targets: "c"
 """
 
 echo "Hello World!"
diff --git a/testament/tests/shouldfail/treject.nim b/testament/tests/shouldfail/treject.nim
index 395dc4251..1e7258f70 100644
--- a/testament/tests/shouldfail/treject.nim
+++ b/testament/tests/shouldfail/treject.nim
@@ -1,6 +1,5 @@
 discard """
   action: "reject"
-  targets: "c"
 """
 
 # Because we set action="reject", we expect this line not to compile. But the
diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim
index 7a2845029..69dfbc0a0 100644
--- a/testament/tests/shouldfail/tsortoutput.nim
+++ b/testament/tests/shouldfail/tsortoutput.nim
@@ -1,6 +1,5 @@
 discard """
   sortoutput: true
-  targets: "c"
   output: '''
 2
 1
diff --git a/testament/tests/shouldfail/ttimeout.nim b/testament/tests/shouldfail/ttimeout.nim
index 8ffd71aaa..fd3e1a598 100644
--- a/testament/tests/shouldfail/ttimeout.nim
+++ b/testament/tests/shouldfail/ttimeout.nim
@@ -1,6 +1,5 @@
 discard """
   timeout: "0.1"
-  targets: "c"
 """
 
 import os
diff --git a/testament/tests/shouldfail/tvalgrind.nim b/testament/tests/shouldfail/tvalgrind.nim
index 5502705b3..d551ff12e 100644
--- a/testament/tests/shouldfail/tvalgrind.nim
+++ b/testament/tests/shouldfail/tvalgrind.nim
@@ -1,6 +1,5 @@
 discard """
   valgrind: true
-  targets: "c"
   cmd: "nim $target --gc:arc -d:useMalloc $options $file"
 """
 
diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim
index 014373dfb..1b679d92c 100644
--- a/tests/misc/trunner.nim
+++ b/tests/misc/trunner.nim
@@ -6,6 +6,8 @@ discard """
 ## tests that don't quite fit the mold and are easier to handle via `execCmdEx`
 ## A few others could be added to here to simplify code.
 ## Note: this test is a bit slow but tests a lot of things; please don't disable.
+## Note: if needed, we could use `matrix: "-d:case1; -d:case2"` to split this
+## into several independent tests while retaining the common test helpers.
 
 import std/[strformat,os,osproc,unittest,compilesettings]
 from std/sequtils import toSeq,mapIt
diff --git a/tests/stdlib/tchannels.nim b/tests/stdlib/tchannels.nim
index 33108c50c..d609eec78 100644
--- a/tests/stdlib/tchannels.nim
+++ b/tests/stdlib/tchannels.nim
@@ -1,5 +1,5 @@
 discard """
-  timeout:  5.0 # but typically < 1s
+  timeout:  20.0 # but typically < 1s (in isolation but other tests running in parallel can affect this since based on epochTime)
   disabled: "freebsd"
   matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger"
 """
diff --git a/tests/stdlib/ttestutils.nim b/tests/stdlib/ttestutils.nim
index 0d789cd76..d24c5b39a 100644
--- a/tests/stdlib/ttestutils.nim
+++ b/tests/stdlib/ttestutils.nim
@@ -22,3 +22,18 @@ block: # greedyOrderedSubsetLines
     not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4\na5:suffix")
     not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\nprefix:a5")
     not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\na5:suffix")
+
+block: # greedyOrderedSubsetLines with allowPrefixMatch = true
+  template fn(a, b): bool =
+    greedyOrderedSubsetLines(a, b, allowPrefixMatch = true)
+  assertAll:
+    fn("a1\na3", "a0\na1\na2\na3_suffix\na4")
+    not fn("a1\na3", "a0\na1\na2\nprefix_a3\na4")
+    # these are same as above, could be refactored
+    not fn("a3\na1", "a0\na1\na2\na3\na4") # out of order
+    not fn("a1\na5", "a0\na1\na2\na3\na4") # a5 not in lhs
+
+    not fn("a1\na5", "a0\na1\na2\na3\na4\nprefix:a5")
+    fn("a1\na5", "a0\na1\na2\na3\na4\na5:suffix")
+    not fn("a5", "a0\na1\na2\na3\na4\nprefix:a5")
+    fn("a5", "a0\na1\na2\na3\na4\na5:suffix")
diff --git a/tests/testament/tshould_not_work.nim b/tests/testament/tshould_not_work.nim
index 2777bfe95..e11944e8f 100644
--- a/tests/testament/tshould_not_work.nim
+++ b/tests/testament/tshould_not_work.nim
@@ -1,39 +1,54 @@
 discard """
-cmd: "testament/testament --directory:testament --colors:off --backendLogging:off --nim:$nim category shouldfail"
-action: compile
-nimout: '''
-FAIL: tests/shouldfail/tccodecheck.nim c
+  joinable: false
+"""
+
+const expected = """
+FAIL: tests/shouldfail/tccodecheck.nim
 Failure: reCodegenFailure
 Expected:
 baz
-FAIL: tests/shouldfail/tcolumn.nim c
+FAIL: tests/shouldfail/tcolumn.nim
 Failure: reLinesDiffer
-FAIL: tests/shouldfail/terrormsg.nim c
+FAIL: tests/shouldfail/terrormsg.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/texitcode1.nim c
+FAIL: tests/shouldfail/texitcode1.nim
 Failure: reExitcodesDiffer
-FAIL: tests/shouldfail/tfile.nim c
+FAIL: tests/shouldfail/tfile.nim
 Failure: reFilesDiffer
-FAIL: tests/shouldfail/tline.nim c
+FAIL: tests/shouldfail/tline.nim
 Failure: reLinesDiffer
-FAIL: tests/shouldfail/tmaxcodesize.nim c
+FAIL: tests/shouldfail/tmaxcodesize.nim
 Failure: reCodegenFailure
 max allowed size: 1
-FAIL: tests/shouldfail/tnimout.nim c
+FAIL: tests/shouldfail/tnimout.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/tnimoutfull.nim c
+FAIL: tests/shouldfail/tnimoutfull.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/toutput.nim c
+FAIL: tests/shouldfail/toutput.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/toutputsub.nim c
+FAIL: tests/shouldfail/toutputsub.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/treject.nim c
+FAIL: tests/shouldfail/treject.nim
 Failure: reFilesDiffer
-FAIL: tests/shouldfail/tsortoutput.nim c
+FAIL: tests/shouldfail/tsortoutput.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/ttimeout.nim c
+FAIL: tests/shouldfail/ttimeout.nim
 Failure: reTimeout
-FAIL: tests/shouldfail/tvalgrind.nim c
+FAIL: tests/shouldfail/tvalgrind.nim
 Failure: reExitcodesDiffer
-'''
 """
+
+import std/[os,strformat,osproc]
+import stdtest/testutils
+
+proc main =
+  const nim = getCurrentCompilerExe()
+  # TODO: bin/testament instead? like other tools (eg bin/nim, bin/nimsuggest etc)
+  let testamentExe = "testament/testament"
+  let cmd = fmt"{testamentExe} --directory:testament --colors:off --backendLogging:off --nim:{nim} category shouldfail"
+  let (outp, status) = execCmdEx(cmd)
+  doAssert status == 1, $status
+
+  let ok = greedyOrderedSubsetLines(expected, outp, allowPrefixMatch = true)
+  doAssert ok, &"\nexpected:\n{expected}\noutp:\n{outp}"
+main()