15 files changed, 121 insertions, 0 deletions
diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim
new file mode 100644
index 000000000..477da1e23
--- /dev/null
+++ b/testament/tests/shouldfail/tccodecheck.nim
@@ -0,0 +1,8 @@
+discard """
+  ccodecheck: "baz"
+"""
+
+proc foo(): void {.exportc: "bar".}=
+  echo "Hello World"
+
+foo()
diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim
new file mode 100644
index 000000000..809ddec74
--- /dev/null
+++ b/testament/tests/shouldfail/tcolumn.nim
@@ -0,0 +1,8 @@
+discard """
+  errormsg: "undeclared identifier: 'undeclared'"
+  line: 9
+  column: 7
+"""
+
+# test should fail because the line directive is wrong
+echo undeclared
diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim
new file mode 100644
index 000000000..6c130d107
--- /dev/null
+++ b/testament/tests/shouldfail/terrormsg.nim
@@ -0,0 +1,8 @@
+discard """
+  errormsg: "wrong error message"
+  line: 9
+  column: 6
+"""
+
+# test should fail because the line directive is wrong
+echo undeclared
diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim
new file mode 100644
index 000000000..605f046db
--- /dev/null
+++ b/testament/tests/shouldfail/texitcode1.nim
@@ -0,0 +1,3 @@
+discard """
+  exitcode: 1
+"""
diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim
new file mode 100644
index 000000000..b40a4f44f
--- /dev/null
+++ b/testament/tests/shouldfail/tfile.nim
@@ -0,0 +1,6 @@
+discard """
+  errormsg: "undeclared identifier: 'undefined'"
+  file: "notthisfile.nim"
+"""
+
+echo undefined
diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim
new file mode 100644
index 000000000..fe782eb03
--- /dev/null
+++ b/testament/tests/shouldfail/tline.nim
@@ -0,0 +1,8 @@
+discard """
+  errormsg: "undeclared identifier: 'undeclared'"
+  line: 10
+  column: 6
+"""
+
+# test should fail because the line directive is wrong
+echo undeclared
diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim
new file mode 100644
index 000000000..92022ee97
--- /dev/null
+++ b/testament/tests/shouldfail/tmaxcodesize.nim
@@ -0,0 +1,5 @@
+discard """
+  maxcodesize: 1
+"""
+
+echo "Hello World"
diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim
new file mode 100644
index 000000000..0a65bfb70
--- /dev/null
+++ b/testament/tests/shouldfail/tnimout.nim
@@ -0,0 +1,7 @@
+discard """
+  nimout: "Hello World!"
+  action: compile
+"""
+
+static:
+  echo "something else"
diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim
new file mode 100644
index 000000000..4fc93f6d2
--- /dev/null
+++ b/testament/tests/shouldfail/tnimoutfull.nim
@@ -0,0 +1,14 @@
+discard """
+  nimout: '''
+msg1
+msg2
+'''
+  action: compile
+  nimoutFull: true
+"""
+
+# should fail because `msg3` is not in nimout and `nimoutFill: true` was given
+static:
+  echo "msg1"
+  echo "msg2"
+  echo "msg3"
diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim
new file mode 100644
index 000000000..eaf9e8652
--- /dev/null
+++ b/testament/tests/shouldfail/toutput.nim
@@ -0,0 +1,7 @@
+discard """
+  output: '''
+  done
+  '''
+"""
+
+echo "broken"
diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim
new file mode 100644
index 000000000..47324ecee
--- /dev/null
+++ b/testament/tests/shouldfail/toutputsub.nim
@@ -0,0 +1,5 @@
+discard """
+  outputsub: "something else"
+"""
+
+echo "Hello World!"
diff --git a/testament/tests/shouldfail/treject.nim b/testament/tests/shouldfail/treject.nim
new file mode 100644
index 000000000..1e7258f70
--- /dev/null
+++ b/testament/tests/shouldfail/treject.nim
@@ -0,0 +1,7 @@
+discard """
+  action: "reject"
+"""
+
+# Because we set action="reject", we expect this line not to compile. But the
+# line does compile, therefore the test fails.
+assert true
diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim
new file mode 100644
index 000000000..69dfbc0a0
--- /dev/null
+++ b/testament/tests/shouldfail/tsortoutput.nim
@@ -0,0 +1,11 @@
+discard """
+  sortoutput: true
+  output: '''
+2
+1
+'''
+"""
+
+# this test should ensure that the output is actually sorted
+echo "2"
+echo "1"
diff --git a/testament/tests/shouldfail/ttimeout.nim b/testament/tests/shouldfail/ttimeout.nim
new file mode 100644
index 000000000..fd3e1a598
--- /dev/null
+++ b/testament/tests/shouldfail/ttimeout.nim
@@ -0,0 +1,7 @@
+discard """
+  timeout: "0.1"
+"""
+
+import os
+
+os.sleep(1000)
diff --git a/testament/tests/shouldfail/tvalgrind.nim b/testament/tests/shouldfail/tvalgrind.nim
new file mode 100644
index 000000000..d551ff12e
--- /dev/null
+++ b/testament/tests/shouldfail/tvalgrind.nim
@@ -0,0 +1,17 @@
+discard """
+  valgrind: true
+  cmd: "nim $target --gc:arc -d:useMalloc $options $file"
+"""
+
+# this is the same check used by testament/specs.nim whether or not valgrind
+# tests are supported
+when defined(linux) and sizeof(int) == 8:
+  # discarding this allocation will cause valgrind to fail (which is what we
+  # want), but valgrind only runs on 64-bit Linux machines...
+  discard alloc(1)
+else:
+  # ...so on all other OS/architectures, simulate any non-zero exit code to
+  # mimic how valgrind would have failed on this test. We cannot use things like
+  # `disabled: "freebsd"` in the Testament configs above or else the tests will
+  # be SKIP-ed rather than FAIL-ed
+  quit(1) # choose 1 to match valgrind's `--error-exit=1`, but could be anything