3 files changed, 43 insertions, 18 deletions
diff --git a/tests/testament/tinlinemsg.nim b/tests/testament/tinlinemsg.nim
new file mode 100644
index 000000000..199c263e9
--- /dev/null
+++ b/tests/testament/tinlinemsg.nim
@@ -0,0 +1,8 @@
+discard """
+  matrix: "--errorMax:0 --styleCheck:error"
+"""
+
+proc generic_proc*[T](a_a: int) = #[tt.Error
+     ^ 'generic_proc' should be: 'genericProc'; tt.Error
+                      ^ 'a_a' should be: 'aA' ]#
+  discard
diff --git a/tests/testament/tjoinable.nim b/tests/testament/tjoinable.nim
index 7a0ad7985..c651780de 100644
--- a/tests/testament/tjoinable.nim
+++ b/tests/testament/tjoinable.nim
@@ -3,6 +3,7 @@ discard """
 """
 
 # checks that this is joinable
+# checks that megatest allows duplicate names, see also `tests/misc/tjoinable.nim`
 doAssert defined(testing)
 doAssert defined(nimMegatest)
 echo "ok" # intentional to make sure this doesn't prevent `isJoinableSpec`
diff --git a/tests/testament/tshould_not_work.nim b/tests/testament/tshould_not_work.nim
index a0b4d6a36..8c99510a0 100644
--- a/tests/testament/tshould_not_work.nim
+++ b/tests/testament/tshould_not_work.nim
@@ -1,37 +1,53 @@
 discard """
-cmd: "testament/testament --directory:testament --colors:off --backendLogging:off --nim:$nim category shouldfail"
-action: compile
-nimout: '''
-FAIL: tests/shouldfail/tccodecheck.nim c
+  joinable: false
+"""
+
+const expected = """
+FAIL: tests/shouldfail/tccodecheck.nim
 Failure: reCodegenFailure
 Expected:
 baz
-FAIL: tests/shouldfail/tcolumn.nim c
+FAIL: tests/shouldfail/tcolumn.nim
 Failure: reLinesDiffer
-FAIL: tests/shouldfail/terrormsg.nim c
+FAIL: tests/shouldfail/terrormsg.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/texitcode1.nim c
+FAIL: tests/shouldfail/texitcode1.nim
 Failure: reExitcodesDiffer
-FAIL: tests/shouldfail/tfile.nim c
+FAIL: tests/shouldfail/tfile.nim
 Failure: reFilesDiffer
-FAIL: tests/shouldfail/tline.nim c
+FAIL: tests/shouldfail/tline.nim
 Failure: reLinesDiffer
-FAIL: tests/shouldfail/tmaxcodesize.nim c
+FAIL: tests/shouldfail/tmaxcodesize.nim
 Failure: reCodegenFailure
 max allowed size: 1
-FAIL: tests/shouldfail/tnimout.nim c
+FAIL: tests/shouldfail/tnimout.nim
+Failure: reMsgsDiffer
+FAIL: tests/shouldfail/tnimoutfull.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/toutput.nim c
+FAIL: tests/shouldfail/toutput.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/toutputsub.nim c
+FAIL: tests/shouldfail/toutputsub.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/treject.nim c
+FAIL: tests/shouldfail/treject.nim
 Failure: reFilesDiffer
-FAIL: tests/shouldfail/tsortoutput.nim c
+FAIL: tests/shouldfail/tsortoutput.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/ttimeout.nim c
+FAIL: tests/shouldfail/ttimeout.nim
 Failure: reTimeout
-FAIL: tests/shouldfail/tvalgrind.nim c
+FAIL: tests/shouldfail/tvalgrind.nim
 Failure: reExitcodesDiffer
-'''
 """
+
+import std/[os,strformat,osproc]
+import stdtest/testutils
+
+proc main =
+  const nim = getCurrentCompilerExe()
+  let testamentExe = "bin/testament"
+  let cmd = fmt"{testamentExe} --directory:testament --colors:off --backendLogging:off --nim:{nim} category shouldfail"
+  let (outp, status) = execCmdEx(cmd)
+  doAssert status == 1, $status
+
+  let ok = greedyOrderedSubsetLines(expected, outp, allowPrefixMatch = true)
+  doAssert ok, &"\nexpected:\n{expected}\noutp:\n{outp}"
+main()