testament: add `nimoutFull: bool` spec (#17867)

* testament: add `nimoutFull: bool` spec
* PRTEMP
* works
* cleanup
* add test for #12741
* PRTEMP failing test
* remove unrelated changes
* changelog

7 files changed, 43 insertions, 9 deletions

diff --git a/changelog.md b/changelog.md
index e4b5c39f1..75d086450 100644
--- a/changelog.md
+++ b/changelog.md
@@ -399,3 +399,6 @@
 
 - `fusion` is now un-bundled from nim, `./koch fusion` will
   install it via nimble at a fixed hash.
+
+- testament: added `nimoutFull: bool` spec to compare full output of compiler
+  instead of a subset.
diff --git a/testament/categories.nim b/testament/categories.nim
index 19d9e4507..ffee5eeb3 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -576,6 +576,9 @@ proc isJoinableSpec(spec: TSpec): bool =
     spec.exitCode == 0 and
     spec.input.len == 0 and
     spec.nimout.len == 0 and
+    spec.nimoutFull == false and
+      # so that tests can have `nimoutFull: true` with `nimout.len == 0` with
+      # the meaning that they expect empty output.
     spec.matrix.len == 0 and
     spec.outputCheck != ocSubstr and
     spec.ccodeCheck.len == 0 and
diff --git a/testament/specs.nim b/testament/specs.nim
index 38e4ed0db..3774778b3 100644
--- a/testament/specs.nim
+++ b/testament/specs.nim
@@ -88,6 +88,7 @@ type
     targets*: set[TTarget]
     matrix*: seq[string]
     nimout*: string
+    nimoutFull*: bool # whether nimout is all compiler output or a subset
     parseErrors*: string            # when the spec definition is invalid, this is not empty.
     unjoinable*: bool
     unbatchable*: bool
@@ -297,6 +298,8 @@ proc parseSpec*(filename: string): TSpec =
         result.action = actionReject
       of "nimout":
         result.nimout = e.value
+      of "nimoutfull":
+        result.nimoutFull = parseCfgBool(e.value)
       of "batchable":
         result.unbatchable = not parseCfgBool(e.value)
       of "joinable":
diff --git a/testament/testament.nim b/testament/testament.nim
index f259c7b96..97f1f01b8 100644
--- a/testament/testament.nim
+++ b/testament/testament.nim
@@ -371,12 +371,20 @@ proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest,
     r.addResult(test, target, "", given.msg, reSuccess)
     inc(r.passed)
 
+proc nimoutCheck(expected, given: TSpec): bool =
+  result = true
+  if expected.nimoutFull:
+    if expected.nimout != given.nimout:
+      result = false
+  elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout):
+    result = false
+
 proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) =
   if expected.inlineErrors.len > 0:
     checkForInlineErrors(r, expected, given, test, target)
   elif strip(expected.msg) notin strip(given.msg):
     r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer)
-  elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout):
+  elif not nimoutCheck(expected, given):
     r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer)
   elif extractFilename(expected.file) != extractFilename(given.file) and
       "internal error:" notin expected.msg:
@@ -424,10 +432,6 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st
     given.err = reCodeNotFound
     echo getCurrentExceptionMsg()
 
-proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) =
-  if not greedyOrderedSubsetLines(expectedNimout, given.nimout):
-    given.err = reMsgsDiffer
-
 proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
                          expected: TSpec; r: var TResults) =
   var expectedmsg: string = ""
@@ -436,10 +440,10 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
     if expected.needsCodegenCheck:
       codegenCheck(test, target, expected, expectedmsg, given)
       givenmsg = given.msg
-    if expected.nimout.len > 0:
+    if not nimoutCheck(expected, given):
+      given.err = reMsgsDiffer
       expectedmsg = expected.nimout
       givenmsg = given.nimout.strip
-      nimoutCheck(test, expectedmsg, given)
   else:
     givenmsg = "$ " & given.cmd & '\n' & given.nimout
   if given.err == reSuccess: inc(r.passed)
diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim
new file mode 100644
index 000000000..3349ceedf
--- /dev/null
+++ b/testament/tests/shouldfail/tnimoutfull.nim
@@ -0,0 +1,15 @@
+discard """
+  targets: "c"
+  nimout: '''
+msg1
+msg2
+'''
+  action: compile
+  nimoutFull: true
+"""
+
+# should fail because `msg3` is not in nimout and `nimoutFill: true` was given
+static:
+  echo "msg1"
+  echo "msg2"
+  echo "msg3"
diff --git a/tests/config.nims b/tests/config.nims
index e5d6545f4..578b69e76 100644
--- a/tests/config.nims
+++ b/tests/config.nims
@@ -6,9 +6,13 @@ switch("path", "$lib/../testament/lib")
 ## prevent common user config settings to interfere with testament expectations
 ## Indifidual tests can override this if needed to test for these options.
 switch("colors", "off")
-switch("filenames", "legacyRelProj")
+
 switch("excessiveStackTrace", "off")
-switch("spellSuggest", "0")
+
+when (NimMajor, NimMinor, NimPatch) >= (1,5,1):
+  # to make it easier to test against older nim versions, (best effort only)
+  switch("filenames", "legacyRelProj")
+  switch("spellSuggest", "0")
 
 # for std/unittest
 switch("define", "nimUnittestOutputLevel:PRINT_FAILURES")
diff --git a/tests/testament/tshould_not_work.nim b/tests/testament/tshould_not_work.nim
index a0b4d6a36..2777bfe95 100644
--- a/tests/testament/tshould_not_work.nim
+++ b/tests/testament/tshould_not_work.nim
@@ -21,6 +21,8 @@ Failure: reCodegenFailure
 max allowed size: 1
 FAIL: tests/shouldfail/tnimout.nim c
 Failure: reMsgsDiffer
+FAIL: tests/shouldfail/tnimoutfull.nim c
+Failure: reMsgsDiffer
 FAIL: tests/shouldfail/toutput.nim c
 Failure: reOutputsDiffer
 FAIL: tests/shouldfail/toutputsub.nim c