added 'sortoutput' option to make output deterministic for threading tests

2 files changed, 13 insertions, 3 deletions

diff --git a/tests/testament/specs.nim b/tests/testament/specs.nim
index 225ea1891..6e72f4b5e 100644
--- a/tests/testament/specs.nim
+++ b/tests/testament/specs.nim
@@ -46,7 +46,7 @@ type
     msg*: string
     ccodeCheck*: string
     err*: TResultEnum
-    substr*: bool
+    substr*, sortoutput*: bool
     targets*: set[TTarget]
 
 const
@@ -113,6 +113,8 @@ proc parseSpec*(filename: string): TSpec =
       result.action = actionRun
       result.outp = e.value
       result.substr = true
+    of "sortoutput":
+      result.sortoutput = parseCfgBool(e.value)
     of "exitcode": 
       discard parseInt(e.value, result.exitCode)
     of "msg":
diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim
index 50d0e6eac..adf9785e0 100644
--- a/tests/testament/tester.nim
+++ b/tests/testament/tester.nim
@@ -11,7 +11,8 @@
 
 import
   parseutils, strutils, pegs, os, osproc, streams, parsecfg, json,
-  marshal, backend, parseopt, specs, htmlgen, browsers, terminal
+  marshal, backend, parseopt, specs, htmlgen, browsers, terminal, sequtils,
+  algorithm
 
 const
   resultsFile = "testresults.html"
@@ -150,6 +151,11 @@ proc codegenCheck(test: TTest, check: string, given: var TSpec) =
     except EIO:
       given.err = reCodeNotFound
 
+proc makeDeterministic(s: string): string =
+  var x = toSeq(s.lines)
+  sort(x, system.cmp)
+  result = join(x, "\n")
+
 proc testSpec(r: var TResults, test: TTest) =
   # major entry point for a single test
   let tname = test.name.addFileExt(".nim")
@@ -191,7 +197,9 @@ proc testSpec(r: var TResults, test: TTest) =
             r.addResult(test, "exitcode: " & $expected.exitCode,
                               "exitcode: " & $exitCode, reExitCodesDiffer)
           else:
-            if strip(buf.string) != strip(expected.outp):
+            var bufB = strip(buf.string)
+            if expected.sortoutput: bufB = makeDeterministic(bufB)
+            if bufB != strip(expected.outp):
               if not (expected.substr and expected.outp in buf.string):
                 given.err = reOutputsDiffer
             if given.err == reSuccess: