testament: joinable is now an explicit concept of a test spec

17 files changed, 55 insertions, 51 deletions

diff --git a/testament/categories.nim b/testament/categories.nim
index 4c34eb1d1..7273b6a4f 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -35,31 +35,9 @@ const
     "testdata",
     "nimcache",
     "coroutines",
-    "osproc"
-  ]
-
-
-# these tests still have bugs. At some point when the bugs are fixd
-# this should become empty.
-
-# exclude for various reasons
-const
-  specialDisabedTests = [
-    "tests/dir with space/tspace.nim", # can't import dir with spaces.
-    "tests/method/tmultim.nim",        # (77, 8) Error: method is not a base
-    "tests/system/talloc2.nim",        # too much memory
-    "tests/collections/ttables.nim",   # takes too long
-    "tests/system/tparams.nim",        # executes itself with parameters
-    "tests/stdlib/tquit.nim",          # not testing for obvious reasons
-    "tests/system/trealloc.nim",       # out of memory
-    "tests/system/t7894.nim",          # causes out of memory in later tests
-    "tests/types/tissues_types.nim",   # causes out of memory with --gc:boehm
-    "tests/pragmas/tused.nim",         # paths in nimout differ when imported
-    "tests/generics/trtree.nim",       # very very ugly test
-    "tests/array/tarray.nim",          #
-    "tests/destructor/turn_destroy_into_finalizer.nim", # fails when imported
-    "tests/misc/tnew.nim",
-    "tests/misc/tcmdline.nim"
+    "osproc",
+    "shouldfail",
+    "dir with space"
   ]
 
 # included from tester.nim
@@ -536,11 +514,11 @@ proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
 proc isJoinableSpec(spec: TSpec): bool =
   result = not spec.sortoutput and
     spec.action == actionRun and
-    spec.file.replace('\\', '/') notin specialDisabedTests and
     not fileExists(spec.file.changeFileExt("cfg")) and
     not fileExists(parentDir(spec.file) / "nim.cfg") and
     spec.cmd.len == 0 and
-    spec.err != reIgnored and
+    spec.err != reDisabled and
+    not spec.unjoinable and
     spec.exitCode == 0 and
     spec.input.len == 0 and
     spec.nimout.len == 0 and
@@ -600,11 +578,12 @@ proc processCategory(r: var TResults, cat: Category, options: string, runJoinabl
   else:
     var testsRun = 0
     for name in os.walkFiles("tests" & DirSep &.? cat.string / "t*.nim"):
-      let test = makeTest(name, options, cat)
+      var test = makeTest(name, options, cat)
       if runJoinableTests or not isJoinableSpec(test.spec):
-        testSpec r, test
+        discard "run the test"
       else:
-        echo "filter out: ", test.name
+        test.spec.err = reJoined
+      testSpec r, test
       inc testsRun
     if testsRun == 0:
       echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run"
diff --git a/testament/specs.nim b/testament/specs.nim
index 6283d18d7..df12db543 100644
--- a/testament/specs.nim
+++ b/testament/specs.nim
@@ -38,7 +38,8 @@ type
     reExeNotFound,
     reInstallFailed     # package installation failed
     reBuildFailed       # package building failed
-    reIgnored,          # test is ignored
+    reDisabled,         # test is disabled
+    reJoined,           # test is disabled because it was joined into the megatest
     reSuccess           # test was successful
     reInvalidSpec       # test had problems to parse the spec
 
@@ -66,6 +67,7 @@ type
     targets*: set[TTarget]
     nimout*: string
     parseErrors*: string # when the spec definition is invalid, this is not empty.
+    unjoinable*: bool
 
 proc getCmd*(s: TSpec): string =
   if s.cmd.len == 0:
@@ -182,26 +184,28 @@ proc parseSpec*(filename: string): TSpec =
         result.action = actionReject
       of "nimout":
         result.nimout = e.value
+      of "joinable":
+        result.unjoinable = not parseCfgBool(e.value)
       of "disabled":
         case e.value.normalize
-        of "y", "yes", "true", "1", "on": result.err = reIgnored
+        of "y", "yes", "true", "1", "on": result.err = reDisabled
         of "n", "no", "false", "0", "off": discard
         of "win", "windows":
-          when defined(windows): result.err = reIgnored
+          when defined(windows): result.err = reDisabled
         of "linux":
-          when defined(linux): result.err = reIgnored
+          when defined(linux): result.err = reDisabled
         of "bsd":
-          when defined(bsd): result.err = reIgnored
+          when defined(bsd): result.err = reDisabled
         of "macosx":
-          when defined(macosx): result.err = reIgnored
+          when defined(macosx): result.err = reDisabled
         of "unix":
-          when defined(unix): result.err = reIgnored
+          when defined(unix): result.err = reDisabled
         of "posix":
-          when defined(posix): result.err = reIgnored
+          when defined(posix): result.err = reDisabled
         of "travis":
-          if isTravis: result.err = reIgnored
+          if isTravis: result.err = reDisabled
         of "appveyor":
-          if isAppVeyor: result.err = reIgnored
+          if isAppVeyor: result.err = reDisabled
         else:
           result.parseErrors.addLine "cannot interpret as a bool: ", e.value
       of "cmd":
@@ -237,6 +241,4 @@ proc parseSpec*(filename: string): TSpec =
       result.parseErrors.addLine e.msg
     of cfgEof:
       break
-
-
   close(p)
diff --git a/testament/tester.nim b/testament/tester.nim
index b290bb3b0..f8a3c0336 100644
--- a/testament/tester.nim
+++ b/testament/tester.nim
@@ -233,8 +233,10 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
   r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
   if success == reSuccess:
     maybeStyledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)"
-  elif success == reIgnored:
+  elif success == reDisabled:
     maybeStyledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name
+  elif success == reJoined:
+    maybeStyledEcho styleDim, fgYellow, "JOINED: ", styleBright, fgCyan, name
   else:
     maybeStyledEcho styleBright, fgRed, "FAIL: ", fgCyan, name
     maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
@@ -248,7 +250,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
     let (outcome, msg) =
       if success == reSuccess:
         ("Passed", "")
-      elif success == reIgnored:
+      elif success in {reDisabled, reJoined}:
         ("Skipped", "")
       else:
         ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
@@ -357,9 +359,9 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
     inc(r.total)
     return
 
-  if expected.err == reIgnored:
+  if expected.err in {reDisabled, reJoined}:
     # targetC is a lie, but parameter is required
-    r.addResult(test, targetC, "", "", reIgnored)
+    r.addResult(test, targetC, "", "", expected.err)
     inc(r.skipped)
     inc(r.total)
     return
@@ -571,7 +573,7 @@ proc main() =
   of "c", "cat", "category":
     var cat = Category(p.key)
     p.next
-    processCategory(r, cat, p.cmdLineRest.string, runJoinableTests = false)
+    processCategory(r, cat, p.cmdLineRest.string, runJoinableTests = true)
   of "r", "run":
     let (dir, file) = splitPath(p.key.string)
     let (_, subdir) = splitPath(dir)
@@ -579,7 +581,7 @@ proc main() =
     processSingleTest(r, cat, p.cmdLineRest.string, file)
   of "html":
     generateHtml(resultsFile, optFailing)
-  of "stats":
+  of "megatest":
     discard runJoinedTest(testsDir)
   else:
     quit Usage
diff --git a/tests/array/tarray.nim b/tests/array/tarray.nim
index 96e90c97a..2a371b788 100644
--- a/tests/array/tarray.nim
+++ b/tests/array/tarray.nim
@@ -28,6 +28,7 @@ dflfdjkl__abcdefgasfsgdfgsgdfggsdfasdfsafewfkljdsfajsdf
 kgdchlfniambejop
 fjpmholcibdgeakn
 '''
+joinable: false
 """
 
 block tarray:
diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim
index b22b2b237..6798e5731 100644
--- a/tests/collections/ttables.nim
+++ b/tests/collections/ttables.nim
@@ -6,10 +6,11 @@ And we get here
 2
 3
 '''
+joinable: false
 """
 import hashes, sequtils, tables
 
-
+# test should not be joined because it takes too long.
 block tableadds:
   proc main =
     var tab = newTable[string, string]()
diff --git a/tests/destructor/turn_destroy_into_finalizer.nim b/tests/destructor/turn_destroy_into_finalizer.nim
index b3f34003e..1409c1c57 100644
--- a/tests/destructor/turn_destroy_into_finalizer.nim
+++ b/tests/destructor/turn_destroy_into_finalizer.nim
@@ -1,5 +1,6 @@
 discard """
   output: "turn_destroy_into_finalizer works"
+  joinable: false
 """
 
 type
diff --git a/tests/generics/trtree.nim b/tests/generics/trtree.nim
index 3fb080335..6ec1c8f6f 100644
--- a/tests/generics/trtree.nim
+++ b/tests/generics/trtree.nim
@@ -2,8 +2,11 @@ discard """
   output: '''1 [2, 3, 4, 7]
 [0, 0]'''
   target: "c"
+  joinable: false
 """
 
+# don't join because the code is too messy.
+
 # Nim RTree and R*Tree implementation
 # S. Salewski, 06-JAN-2018
 
diff --git a/tests/method/tmultim.nim b/tests/method/tmultim.nim
index 310502c56..0e68b660f 100644
--- a/tests/method/tmultim.nim
+++ b/tests/method/tmultim.nim
@@ -4,9 +4,10 @@ collide: unit, thing
 collide: unit, thing
 collide: thing, unit
 collide: thing, thing
-collide: unit, thing | collide: unit, thing | collide: thing, unit | 
+collide: unit, thing | collide: unit, thing | collide: thing, unit |
 do nothing
 '''
+  joinable: false
 """
 
 
diff --git a/tests/misc/tcmdline.nim b/tests/misc/tcmdline.nim
index 2c4768716..71e1301ca 100644
--- a/tests/misc/tcmdline.nim
+++ b/tests/misc/tcmdline.nim
@@ -1,5 +1,6 @@
 discard """
 outputsub: "Number of parameters: 0"
+joinable: false
 """
 # Test the command line
 
diff --git a/tests/misc/tnew.nim b/tests/misc/tnew.nim
index 02282dd4a..2d9a64461 100644
--- a/tests/misc/tnew.nim
+++ b/tests/misc/tnew.nim
@@ -3,6 +3,7 @@ outputsub: '''
 Simple tree node allocation worked!
 Simple cycle allocation worked!
 '''
+joinable: false
 """
 
 # Test the implementation of the new operator
diff --git a/tests/pragmas/tused.nim b/tests/pragmas/tused.nim
index dce854146..67a039d8a 100644
--- a/tests/pragmas/tused.nim
+++ b/tests/pragmas/tused.nim
@@ -4,8 +4,10 @@ compile start
 tused.nim(15, 8) Hint: 'tused.echoSub(a: int, b: int)[declared in tused.nim(15, 7)]' is declared but not used [XDeclaredButNotUsed]
 compile end'''
   output: "8\n8"
+  joinable: false
 """
 
+# not joinable because paths in nimout differ when imported
 static:
   echo "compile start"
 
diff --git a/tests/stdlib/tquit.nim b/tests/stdlib/tquit.nim
index 4f8d5fb20..1f9283ec4 100644
--- a/tests/stdlib/tquit.nim
+++ b/tests/stdlib/tquit.nim
@@ -2,6 +2,7 @@ discard """
 output: '''
 just exiting...
 '''
+joinable: false
 """
 
 # Test the new beforeQuit variable:
diff --git a/tests/system/t7894.nim b/tests/system/t7894.nim
index 27ee3f220..b7ca1eec8 100644
--- a/tests/system/t7894.nim
+++ b/tests/system/t7894.nim
@@ -1,6 +1,7 @@
 discard """
 disabled: "travis"
 disabled: "appveyor"
+joinable: false
 """
 
 # CI integration servers are out of memory for this test
diff --git a/tests/system/talloc2.nim b/tests/system/talloc2.nim
index e40c3f93c..7e0dec9d3 100644
--- a/tests/system/talloc2.nim
+++ b/tests/system/talloc2.nim
@@ -1,5 +1,6 @@
 discard """
 disabled: "windows"
+joinable: false
 """
 
 # appveyor is "out of memory"
diff --git a/tests/system/tparams.nim b/tests/system/tparams.nim
index 015530043..dcd620b20 100644
--- a/tests/system/tparams.nim
+++ b/tests/system/tparams.nim
@@ -1,3 +1,8 @@
+discard """
+joinable: false
+"""
+
+# not joinable because it executes itself with parameters
 import os
 import osproc
 import parseopt2
diff --git a/tests/system/trealloc.nim b/tests/system/trealloc.nim
index dc5f712d6..7180b8eda 100644
--- a/tests/system/trealloc.nim
+++ b/tests/system/trealloc.nim
@@ -1,5 +1,6 @@
 discard """
   output: '''success'''
+  joinable: false
 """
 
 # bug #4818
diff --git a/tests/types/tissues_types.nim b/tests/types/tissues_types.nim
index 97c796302..d3643842e 100644
--- a/tests/types/tissues_types.nim
+++ b/tests/types/tissues_types.nim
@@ -9,8 +9,9 @@ ptr Foo
 (member: "hello world", x: ...)
 (member: 123.456, x: ...)
 '''
+joinable: false
 """
-
+# not joinable because it causes out of memory with --gc:boehm
 import typetraits
 
 block t1252: