diff options
Diffstat (limited to 'testament')
| -rw-r--r-- | testament/categories.nim | 7 | ||||
| -rw-r--r-- | testament/important_packages.nim | 48 | ||||
| -rw-r--r-- | testament/lib/stdtest/specialpaths.nim | 2 | ||||
| -rw-r--r-- | testament/lib/stdtest/testutils.nim | 5 | ||||
| -rw-r--r-- | testament/specs.nim | 15 | ||||
| -rw-r--r-- | testament/testament.nim | 40 |
6 files changed, 82 insertions, 35 deletions
diff --git a/testament/categories.nim b/testament/categories.nim index c449cf3e0..843bef3f9 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -81,10 +81,13 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) - if "boehm" notin options and not isOrc: + if "boehm" notin options: # hcr tests - testSpec r, makeTest("tests/dll/nimhcr_basic.nim", options & " --threads:off --forceBuild --hotCodeReloading:on " & rpath, cat) + var basicHcrTest = makeTest("tests/dll/nimhcr_basic.nim", options & " --threads:off --forceBuild --hotCodeReloading:on " & rpath, cat) + # test segfaults for now but compiles: + if isOrc: basicHcrTest.spec.action = actionCompile + testSpec r, basicHcrTest # force build required - see the comments in the .nim file for more details var hcri = makeTest("tests/dll/nimhcr_integration.nim", diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 9016a0d3d..efec04b3c 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -46,7 +46,7 @@ pkg "BipBuffer" pkg "blscurve", allowFailure = true pkg "bncurve" pkg "brainfuck", "nim c -d:release -r tests/compile.nim" -pkg "bump", "nim c --gc:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump", allowFailure = true +pkg "bump", "nim c --mm:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump", allowFailure = true pkg "c2nim", "nim c testsuite/tester.nim" pkg "cascade" pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true @@ -55,18 +55,22 @@ pkg "chroma" pkg "chronicles", "nim c -o:chr -r chronicles.nim" pkg "chronos", "nim c -r -d:release tests/testall" pkg "cligen", "nim c --path:. -r cligen.nim" -pkg "combparser", "nimble test --gc:orc" +pkg "combparser", "nimble test --mm:orc" pkg "compactdict" pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehension" +pkg "constantine", "nimble make_lib" pkg "cowstrings" pkg "criterion", allowFailure = true # needs testing binary pkg "datamancer" pkg "dashing", "nim c tests/functional.nim" pkg "delaunay" +pkg "dnsclient", allowFailure = true # super fragile pkg "docopt" +pkg "dotenv" # when defined(linux): pkg "drchaos" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" +pkg "faststreams" pkg "fidget" pkg "fragments", "nim c -r fragments/dsl.nim", allowFailure = true # pending https://github.com/nim-lang/packages/issues/2115 pkg "fusion" @@ -75,14 +79,17 @@ pkg "glob" pkg "ggplotnim", "nim c -d:noCairo -r tests/tests.nim" pkg "gittyup", "nimble test", "https://github.com/disruptek/gittyup", allowFailure = true pkg "gnuplot", "nim c gnuplot.nim" -# pkg "gram", "nim c -r --gc:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram" +# pkg "gram", "nim c -r --mm:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram" # pending https://github.com/nim-lang/Nim/issues/16509 pkg "hts", "nim c -o:htss src/hts.nim" pkg "httpauth" +pkg "httputils" pkg "illwill", "nimble examples" pkg "inim" pkg "itertools", "nim doc src/itertools.nim" pkg "iterutils" +pkg "json_rpc" +pkg "json_serialization" pkg "jstin" pkg "karax", "nim c -r tests/tester.nim" pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree" @@ -91,14 +98,13 @@ pkg "lockfreequeues" pkg "macroutils" pkg "manu" pkg "markdown" -pkg "measuremancer", "nimble install -y unchained@#HEAD; nimble -y test" -# when unchained is version 0.3.7 or higher, use `nimble testDeps;` +pkg "measuremancer", "nimble testDeps; nimble -y test" pkg "memo" pkg "msgpack4nim", "nim c -r tests/test_spec.nim" pkg "nake", "nim c nakefile.nim" -pkg "neo", "nim c -d:blas=openblas --mm:refc tests/all.nim", "https://github.com/metagn/neo" -# remove custom url when https://github.com/andreaferretti/neo/pull/53 is merged -pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true +pkg "neo", "nim c -d:blas=openblas --mm:refc tests/all.nim" +pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true, allowFailure = true + # inactive, tests not adapted to #23096 pkg "netty" pkg "nico", allowFailure = true pkg "nicy", "nim c -r src/nicy.nim" @@ -120,10 +126,11 @@ pkg "nimquery" pkg "nimsl" pkg "nimsvg" pkg "nimterop", "nimble minitest", url = "https://github.com/nim-lang/nimterop" -pkg "nimwc", "nim c nimwc.nim", allowFailure = true -pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true +pkg "nimwc", "nim c nimwc.nim" +pkg "nimx", "nim c test/main.nim", allowFailure = true pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" pkg "norm", "testament r tests/common/tmodel.nim" +pkg "normalize" pkg "npeg", "nimble testarc" pkg "numericalnim", "nimble nimCI" pkg "optionsutils" @@ -134,9 +141,10 @@ pkg "pixie" pkg "plotly", "nim c examples/all.nim" pkg "pnm" pkg "polypbren" +pkg "presto" pkg "prologue", "nimble tcompile" -pkg "protobuf", "nim c -o:protobuff -r src/protobuf.nim" -pkg "pylib" +# remove fork after https://github.com/PMunch/combparser/pull/7 is merged: +pkg "protobuf", "nimble install -y https://github.com/metagn/combparser@#HEAD; nim c -o:protobuff -r src/protobuf.nim" pkg "rbtree" pkg "react", "nimble example" pkg "regex", "nim c src/regex" @@ -145,13 +153,14 @@ pkg "RollingHash", "nim c -r tests/test_cyclichash.nim" pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim" pkg "sdl1", "nim c -r src/sdl.nim" pkg "sdl2_nim", "nim c -r sdl2/sdl.nim" -pkg "sigv4", "nim c --gc:arc -r sigv4.nim", "https://github.com/disruptek/sigv4" +pkg "serialization" +pkg "sigv4", "nim c --mm:arc -r sigv4.nim", "https://github.com/disruptek/sigv4" pkg "sim" pkg "smtp", "nimble compileExample" pkg "snip", "nimble test", "https://github.com/genotrance/snip" pkg "ssostrings" pkg "stew" -pkg "stint", "nim r stint.nim" +pkg "stint", "nim c stint.nim" pkg "strslice" pkg "strunicode", "nim c -r --mm:refc src/strunicode.nim" pkg "supersnappy" @@ -163,17 +172,22 @@ pkg "templates" pkg "tensordsl", "nim c -r --mm:refc tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" pkg "terminaltables", "nim c src/terminaltables.nim" pkg "termstyle", "nim c -r termstyle.nim" +pkg "testutils" pkg "timeit" pkg "timezones" pkg "tiny_sqlite" pkg "unicodedb", "nim c -d:release -r tests/tests.nim" pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" +pkg "union", "nim c -r tests/treadme.nim", url = "https://github.com/alaviss/union" +pkg "unittest2" pkg "unpack" -pkg "weave", "nimble test_gc_arc", useHead = true +pkg "weave", "nimble install -y cligen@#HEAD; nimble test_gc_arc", useHead = true +pkg "websock" pkg "websocket", "nim c websocket.nim" -pkg "winim", "nim c winim.nim" +# pkg "winim", allowFailure = true pkg "with" pkg "ws", allowFailure = true -pkg "yaml", "nim c -r test/tserialization.nim" +pkg "yaml" pkg "zero_functional", "nim c -r test.nim" pkg "zippy" +pkg "zxcvbn" diff --git a/testament/lib/stdtest/specialpaths.nim b/testament/lib/stdtest/specialpaths.nim index 7df63666f..e214d113d 100644 --- a/testament/lib/stdtest/specialpaths.nim +++ b/testament/lib/stdtest/specialpaths.nim @@ -48,7 +48,7 @@ proc splitTestFile*(file: string): tuple[cat: string, path: string] = else: result.path = file return result - doAssert false, "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'" + raiseAssert "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'" static: # sanity check diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index 66afd62af..a490b17c8 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -66,6 +66,11 @@ template enableRemoteNetworking*: bool = ## a `nim` invocation (possibly via additional intermediate processes). getEnv("NIM_TESTAMENT_REMOTE_NETWORKING") == "1" +template disableSSLTesting*: bool = + ## TODO: workaround for GitHub Action gcc 14 matrix; remove this + ## matrix and the flag after Azure agent supports ubuntu 24.04 + getEnv("NIM_TESTAMENT_DISABLE_SSL") == "1" + template whenRuntimeJs*(bodyIf, bodyElse) = ##[ Behaves as `when defined(js) and not nimvm` (which isn't legal yet). diff --git a/testament/specs.nim b/testament/specs.nim index 744d1f75a..c3040c1d8 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -143,27 +143,26 @@ proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var T ## ## Can parse a single message for a line: ## - ## .. code-block:: nim - ## + ## ```nim ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error ## ^ 'generic_proc' should be: 'genericProc' [Name] ]# + ## ``` ## ## Can parse multiple messages for a line when they are separated by ';': ## - ## .. code-block:: nim - ## + ## ```nim ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error ## ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error ## ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error ## ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` ## - ## .. code-block:: nim - ## + ## ```nim ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error ## ^ 'generic_proc' should be: 'genericProc' [Name]; ## tt.Error ^ 'no_destroy' should be: 'nodestroy' [Name]; ## tt.Error ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# - ## + ## ``` result = i + len(inlineErrorMarker) inc col, len(inlineErrorMarker) let msgLine = line @@ -514,7 +513,7 @@ proc parseSpec*(filename: string): TSpec = try: msg % ["/", $DirSep, "file", result.filename] except ValueError: - result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unitests-output-message-variable-interpolation')" + result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unit-tests-output-message-variable-interpolation')" msg result.nimout = result.nimout.varSub result.msg = result.msg.varSub diff --git a/testament/testament.nim b/testament/testament.nim index 5f443ffba..1e892e636 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -10,9 +10,12 @@ ## This program verifies Nim against the testcases. import - strutils, pegs, os, osproc, streams, json, std/exitprocs, - backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, times, azure, intsets, macros + std/[strutils, pegs, os, osproc, streams, json, + parseopt, browsers, terminal, exitprocs, + algorithm, times, intsets, macros] + +import backend, specs, azure, htmlgen + from std/sugar import dup import compiler/nodejs import lib/stdtest/testutils @@ -193,7 +196,6 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, foundSuccessMsg = true elif not running(p): break - close(p) result.msg = "" result.file = "" result.output = "" @@ -201,8 +203,9 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, result.column = 0 result.err = reNimcCrash - let exitCode = p.peekExitCode - case exitCode + result.exitCode = p.peekExitCode + close p + case result.exitCode of 0: if foundErrorMsg: result.debugInfo.add " compiler exit code was 0 but some Error's were found." @@ -214,7 +217,7 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, if foundSuccessMsg: result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true." else: - result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $exitCode + result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode if err =~ pegLineError: result.file = extractFilename(matches[0]) @@ -521,7 +524,29 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer) compilerOutputTests(test, target, extraOptions, given, expected, r) of actionReject: + # Make sure its the compiler rejecting and not the system (e.g. segfault) cmpMsgs(r, expected, given, test, target, extraOptions) + if given.exitCode != QuitFailure: + r.addResult(test, target, extraOptions, "exitcode: " & $QuitFailure, + "exitcode: " & $given.exitCode & "\n\nOutput:\n" & + given.nimout, reExitcodesDiffer) + + + +proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget = + result = defaultTarget + var p = parseopt.initOptParser(extraOptions) + + while true: + parseopt.next(p) + case p.kind + of cmdEnd: break + of cmdLongOption, cmdShortOption: + if p.key == "b" or p.key == "backend": + result = parseEnum[TTarget](p.val.normalize) + # chooses the last one + else: + discard proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) = for target in expected.targets: @@ -535,6 +560,7 @@ proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: s else: let nimcache = nimcacheDir(test.name, test.options, target) var testClone = test + let target = changeTarget(extraOptions, target) testSpecHelper(r, testClone, expected, target, extraOptions, nimcache) proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = |