summary refs log tree commit diff stats
path: root/tests
diff options
context:
space:
mode:
authorSergey Avseyev <sergey.avseyev@gmail.com>2015-05-25 21:16:35 +0300
committerSergey Avseyev <sergey.avseyev@gmail.com>2015-05-25 22:37:30 +0300
commit411e602d134aba05a8d876da41e8abe1b04c374c (patch)
tree2036ba5635bd51e9951195092083c8852f12dd46 /tests
parent71561bef5846fbbfea1f77ecba1f9d6d7426ca43 (diff)
downloadNim-411e602d134aba05a8d876da41e8abe1b04c374c.tar.gz
Introduce pedantic mode for tester
Motivation
----------
External tools need to know whether or not any test in suite failed. For
example buildbot, or packaging tool would like to stop the execution and
mark it as failed if some tests does not pass.

Modification
------------
Add `--pedantic` switch to tester program which will force it to quit
with non-zero exit code if at least one failure detected. Also update
`tests()` proc in koch to inspect result from tester and propagate it to
user.

Result
------
Nothing has changed in default behaviour. But following invocations will
exit with non-zero code if there failed tests:

    ./koch tests --pedantic all
    ./tests/testament/tester --pedantic all
Diffstat (limited to 'tests')
-rw-r--r--tests/testament/tester.nim7
1 files changed, 6 insertions, 1 deletions
diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim
index 0308ce940..62dafee80 100644
--- a/tests/testament/tester.nim
+++ b/tests/testament/tester.nim
@@ -31,6 +31,7 @@ Arguments:
 Options:
   --print                   also print results to the console
   --failing                 only show failing/ignored tests
+  --pedantic                return non-zero status code if there are failures
 """ % resultsFile
 
 type
@@ -310,12 +311,14 @@ proc main() =
   backend.open()
   var optPrintResults = false
   var optFailing = false
+  var optPedantic = false
   var p = initOptParser()
   p.next()
   while p.kind == cmdLongoption:
     case p.key.string.normalize
     of "print", "verbose": optPrintResults = true
     of "failing": optFailing = true
+    of "pedantic": optPedantic = true
     else: quit Usage
     p.next()
   if p.kind != cmdArgument: quit Usage
@@ -348,8 +351,10 @@ proc main() =
     if action == "html": openDefaultBrowser(resultsFile)
     else: echo r, r.data
   backend.close()
+  if optPedantic:
+    var failed = r.total - r.passed - r.skipped
+    if failed > 0 : quit(QuitFailure)
 
 if paramCount() == 0:
   quit Usage
 main()
-