testament: add `nimoutFull: bool` spec (#17867)

* testament: add `nimoutFull: bool` spec
* PRTEMP
* works
* cleanup
* add test for #12741
* PRTEMP failing test
* remove unrelated changes
* changelog

4 files changed, 32 insertions, 7 deletions

diff --git a/testament/categories.nim b/testament/categories.nim
index 19d9e4507..ffee5eeb3 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -576,6 +576,9 @@ proc isJoinableSpec(spec: TSpec): bool =
     spec.exitCode == 0 and
     spec.input.len == 0 and
     spec.nimout.len == 0 and
+    spec.nimoutFull == false and
+      # so that tests can have `nimoutFull: true` with `nimout.len == 0` with
+      # the meaning that they expect empty output.
     spec.matrix.len == 0 and
     spec.outputCheck != ocSubstr and
     spec.ccodeCheck.len == 0 and
diff --git a/testament/specs.nim b/testament/specs.nim
index 38e4ed0db..3774778b3 100644
--- a/testament/specs.nim
+++ b/testament/specs.nim
@@ -88,6 +88,7 @@ type
     targets*: set[TTarget]
     matrix*: seq[string]
     nimout*: string
+    nimoutFull*: bool # whether nimout is all compiler output or a subset
     parseErrors*: string            # when the spec definition is invalid, this is not empty.
     unjoinable*: bool
     unbatchable*: bool
@@ -297,6 +298,8 @@ proc parseSpec*(filename: string): TSpec =
         result.action = actionReject
       of "nimout":
         result.nimout = e.value
+      of "nimoutfull":
+        result.nimoutFull = parseCfgBool(e.value)
       of "batchable":
         result.unbatchable = not parseCfgBool(e.value)
       of "joinable":
diff --git a/testament/testament.nim b/testament/testament.nim
index f259c7b96..97f1f01b8 100644
--- a/testament/testament.nim
+++ b/testament/testament.nim
@@ -371,12 +371,20 @@ proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest,
     r.addResult(test, target, "", given.msg, reSuccess)
     inc(r.passed)
 
+proc nimoutCheck(expected, given: TSpec): bool =
+  result = true
+  if expected.nimoutFull:
+    if expected.nimout != given.nimout:
+      result = false
+  elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout):
+    result = false
+
 proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) =
   if expected.inlineErrors.len > 0:
     checkForInlineErrors(r, expected, given, test, target)
   elif strip(expected.msg) notin strip(given.msg):
     r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer)
-  elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout):
+  elif not nimoutCheck(expected, given):
     r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer)
   elif extractFilename(expected.file) != extractFilename(given.file) and
       "internal error:" notin expected.msg:
@@ -424,10 +432,6 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st
     given.err = reCodeNotFound
     echo getCurrentExceptionMsg()
 
-proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) =
-  if not greedyOrderedSubsetLines(expectedNimout, given.nimout):
-    given.err = reMsgsDiffer
-
 proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
                          expected: TSpec; r: var TResults) =
   var expectedmsg: string = ""
@@ -436,10 +440,10 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
     if expected.needsCodegenCheck:
       codegenCheck(test, target, expected, expectedmsg, given)
       givenmsg = given.msg
-    if expected.nimout.len > 0:
+    if not nimoutCheck(expected, given):
+      given.err = reMsgsDiffer
       expectedmsg = expected.nimout
       givenmsg = given.nimout.strip
-      nimoutCheck(test, expectedmsg, given)
   else:
     givenmsg = "$ " & given.cmd & '\n' & given.nimout
   if given.err == reSuccess: inc(r.passed)
diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim
new file mode 100644
index 000000000..3349ceedf
--- /dev/null
+++ b/testament/tests/shouldfail/tnimoutfull.nim
@@ -0,0 +1,15 @@
+discard """
+  targets: "c"
+  nimout: '''
+msg1
+msg2
+'''
+  action: compile
+  nimoutFull: true
+"""
+
+# should fail because `msg3` is not in nimout and `nimoutFill: true` was given
+static:
+  echo "msg1"
+  echo "msg2"
+  echo "msg3"