fix #16576: honor matrix in testament by making such tests non joinable (#16577)

* refs #16576: honor matrix in testament by making such tests non joinable

* add tests + misc fixes

* fix test for i386 with -d:danger

6 files changed, 37 insertions, 12 deletions

diff --git a/testament/categories.nim b/testament/categories.nim
index c894bc9f9..39f0aaa27 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -49,6 +49,7 @@ proc isTestFile*(file: string): bool =
 
 # ---------------- IC tests ---------------------------------------------
 
+# xxx deadcode
 proc icTests(r: var TResults; testsDir: string, cat: Category, options: string) =
   const
     tooltests = ["compiler/nim.nim", "tools/nimgrep.nim"]
@@ -583,6 +584,8 @@ proc processSingleTest(r: var TResults, cat: Category, options, test: string, ta
   testSpec r, makeTest(test, options, cat), targets
 
 proc isJoinableSpec(spec: TSpec): bool =
+  # xxx simplify implementation using a whitelist of fields that are allowed to be
+  # set to non-default values (use `fieldPairs`), to avoid issues like bug #16576.
   result = not spec.sortoutput and
     spec.action == actionRun and
     not fileExists(spec.file.changeFileExt("cfg")) and
@@ -595,6 +598,7 @@ proc isJoinableSpec(spec: TSpec): bool =
     spec.exitCode == 0 and
     spec.input.len == 0 and
     spec.nimout.len == 0 and
+    spec.matrix.len == 0 and
     spec.outputCheck != ocSubstr and
     spec.ccodeCheck.len == 0 and
     (spec.targets == {} or spec.targets == {targetC})
@@ -656,7 +660,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
   writeFile(megatestFile, megatest)
 
   let root = getCurrentDir()
-  let args = ["c", "--nimCache:" & outDir, "-d:testing", "--listCmd", "--path:" & root, megatestFile]
+  let args = ["c", "--nimCache:" & outDir, "-d:testing", "-d:nimMegatest", "--listCmd", "--path:" & root, megatestFile]
   var (cmdLine, buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, input = "")
   if exitCode != 0:
     echo "$ " & cmdLine & "\n" & buf.string
@@ -759,7 +763,9 @@ proc processCategory(r: var TResults, cat: Category,
       testSpec r, test
       inc testsRun
     if testsRun == 0:
-      const whiteListedDirs = ["deps"]
+      const whiteListedDirs = ["deps", "htmldocs", "pkgs"]
+        # `pkgs` because bug #16556 creates `pkgs` dirs and this can affect some users
+        # that try an old version of choosenim.
       doAssert cat.string in whiteListedDirs,
         "Invalid category specified: '$#' not in whilelist: $#" % [cat.string, $whiteListedDirs]
 
diff --git a/testament/specs.nim b/testament/specs.nim
index a7f0fd4bb..58fe7bf4f 100644
--- a/testament/specs.nim
+++ b/testament/specs.nim
@@ -71,6 +71,7 @@ type
     disabled, enabled, leaking
   
   TSpec* = object
+    # xxx make sure `isJoinableSpec` takes into account each field here.
     action*: TTestAction
     file*, cmd*: string
     input*: string
diff --git a/testament/testament.nim b/testament/testament.nim
index 7b1a25bf0..5686da81c 100644
--- a/testament/testament.nim
+++ b/testament/testament.nim
@@ -88,11 +88,12 @@ proc isSuccess(input: string): bool =
   # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs
   input.startsWith("Hint: ") and input.endsWith("[SuccessX]")
 
-proc normalizeMsg(s: string): string =
-  result = newStringOfCap(s.len+1)
-  for x in splitLines(s):
-    if result.len > 0: result.add '\L'
-    result.add x.strip
+when false: # deadcode
+  proc normalizeMsg(s: string): string =
+    result = newStringOfCap(s.len+1)
+    for x in splitLines(s):
+      if result.len > 0: result.add '\L'
+      result.add x.strip
 
 proc getFileDir(filename: string): string =
   result = filename.splitFile().dir
@@ -794,7 +795,7 @@ proc main() =
     var subPath = p.key.string
     let nimRoot = currentSourcePath / "../.."
       # makes sure points to this regardless of cwd or which nim is used to compile this.
-    doAssert existsDir(nimRoot/testsDir) # sanity check
+    doAssert dirExists(nimRoot/testsDir) # sanity check
     if subPath.isAbsolute: subPath = subPath.relativePath(nimRoot)
     # at least one directory is required in the path, to use as a category name
     let pathParts = subPath.relativePath(testsDir).split({DirSep, AltSep})
diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim
index 4176f70e8..16c274032 100644
--- a/tests/stdlib/tmath.nim
+++ b/tests/stdlib/tmath.nim
@@ -1,8 +1,10 @@
 discard """
   targets: "c cpp js"
+  matrix:"; -d:danger"
 """
 
-## xxx enable matrix:"; -d:nimTmathCase2 -d:danger --passc:-ffast-math"
+# xxx: there should be a test with `-d:nimTmathCase2 -d:danger --passc:-ffast-math`,
+# but it requires disabling certain lines with `when not defined(nimTmathCase2)`
 
 import std/[math, random, os]
 import std/[unittest]
@@ -156,13 +158,13 @@ block:
     doAssert(erf(6.0) > erf(5.0))
     doAssert(erfc(6.0) < erfc(5.0))
 
-
-    # Function for approximate comparison of floats
     proc `==~`(x, y: float): bool = (abs(x-y) < 1e-9)
+      # Function for approximate comparison of floats
+      # xxx use `almostEqual`
 
     block: # prod
       doAssert prod([1, 2, 3, 4]) == 24
-      doAssert prod([1.5, 3.4]) == 5.1
+      doAssert prod([1.5, 3.4]).almostEqual 5.1
       let x: seq[float] = @[]
       doAssert prod(x) == 1.0
 
diff --git a/tests/testament/t16576.nim b/tests/testament/t16576.nim
new file mode 100644
index 000000000..8d0dd57e3
--- /dev/null
+++ b/tests/testament/t16576.nim
@@ -0,0 +1,7 @@
+discard """
+  matrix:"-d:nimTest_t16576"
+"""
+
+# bug #16576
+doAssert defined(nimTest_t16576)
+doAssert not defined(nimMegatest)
diff --git a/tests/testament/tjoinable.nim b/tests/testament/tjoinable.nim
new file mode 100644
index 000000000..7a0ad7985
--- /dev/null
+++ b/tests/testament/tjoinable.nim
@@ -0,0 +1,8 @@
+discard """
+  output: "ok"
+"""
+
+# checks that this is joinable
+doAssert defined(testing)
+doAssert defined(nimMegatest)
+echo "ok" # intentional to make sure this doesn't prevent `isJoinableSpec`