diff options
Diffstat (limited to 'testament')
| -rw-r--r-- | testament/categories.nim | 20 | ||||
| -rw-r--r-- | testament/important_packages.nim | 62 | ||||
| -rw-r--r-- | testament/lib/stdtest/specialpaths.nim | 2 | ||||
| -rw-r--r-- | testament/lib/stdtest/testutils.nim | 7 | ||||
| -rw-r--r-- | testament/specs.nim | 22 | ||||
| -rw-r--r-- | testament/testament.nim | 56 |
6 files changed, 123 insertions, 46 deletions
diff --git a/testament/categories.nim b/testament/categories.nim index e8b13746a..843bef3f9 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -47,7 +47,7 @@ proc isTestFile*(file: string): bool = # --------------------- DLL generation tests ---------------------------------- -proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = +proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = false) = const rpath = when defined(macosx): " --passL:-rpath --passL:@loader_path" else: @@ -59,6 +59,7 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) test2.spec.action = actionCompile testSpec c, test2 + var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) test3.spec.action = actionCompile testSpec c, test3 @@ -81,6 +82,13 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) if "boehm" notin options: + # hcr tests + + var basicHcrTest = makeTest("tests/dll/nimhcr_basic.nim", options & " --threads:off --forceBuild --hotCodeReloading:on " & rpath, cat) + # test segfaults for now but compiles: + if isOrc: basicHcrTest.spec.action = actionCompile + testSpec r, basicHcrTest + # force build required - see the comments in the .nim file for more details var hcri = makeTest("tests/dll/nimhcr_integration.nim", options & " --threads:off --forceBuild --hotCodeReloading:on" & rpath, cat) @@ -96,6 +104,8 @@ proc dllTests(r: var TResults, cat: Category, options: string) = runBasicDLLTest c, r, cat, options & " --mm:refc" runBasicDLLTest c, r, cat, options & " -d:release --mm:refc" + runBasicDLLTest c, r, cat, options, isOrc = true + runBasicDLLTest c, r, cat, options & " -d:release", isOrc = true when not defined(windows): # still cannot find a recent Windows version of boehm.dll: runBasicDLLTest c, r, cat, options & " --gc:boehm" @@ -208,9 +218,9 @@ proc jsTests(r: var TResults, cat: Category, options: string) = for testfile in ["exception/texceptions", "exception/texcpt1", "exception/texcsub", "exception/tfinally", "exception/tfinally2", "exception/tfinally3", - "actiontable/tactiontable", "method/tmultimjs", + "collections/tactiontable", "method/tmultimjs", "varres/tvarres0", "varres/tvarres3", "varres/tvarres4", - "varres/tvartup", "misc/tints", "misc/tunsignedinc", + "varres/tvartup", "int/tints", "int/tunsignedinc", "async/tjsandnativeasync"]: test "tests/" & testfile & ".nim" @@ -680,7 +690,7 @@ proc processCategory(r: var TResults, cat: Category, else: jsTests(r, cat, options) of "dll": - dllTests(r, cat, options) + dllTests(r, cat, options & " -d:nimDebugDlOpen") of "gc": gcTests(r, cat, options) of "debugger": @@ -718,6 +728,8 @@ proc processCategory(r: var TResults, cat: Category, case cat2 of "megatest": runJoinedTest(r, cat, testsDir, options) + if isNimRepoTests(): + runJoinedTest(r, cat, testsDir, options & " --mm:refc") else: var testsRun = 0 var files: seq[string] diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 3531e59a8..efec04b3c 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -43,27 +43,34 @@ pkg "awk" pkg "bigints" pkg "binaryheap", "nim c -r binaryheap.nim" pkg "BipBuffer" -pkg "blscurve", allowFailure = true # pending https://github.com/status-im/nim-blscurve/issues/39 +pkg "blscurve", allowFailure = true pkg "bncurve" pkg "brainfuck", "nim c -d:release -r tests/compile.nim" -pkg "bump", "nim c --gc:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump" +pkg "bump", "nim c --mm:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump", allowFailure = true pkg "c2nim", "nim c testsuite/tester.nim" pkg "cascade" pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true +pkg "checksums" pkg "chroma" pkg "chronicles", "nim c -o:chr -r chronicles.nim" pkg "chronos", "nim c -r -d:release tests/testall" pkg "cligen", "nim c --path:. -r cligen.nim" -pkg "combparser", "nimble test --gc:orc" +pkg "combparser", "nimble test --mm:orc" pkg "compactdict" pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehension" -pkg "criterion", allowFailure = true # pending https://github.com/disruptek/criterion/issues/3 (wrongly closed) +pkg "constantine", "nimble make_lib" +pkg "cowstrings" +pkg "criterion", allowFailure = true # needs testing binary pkg "datamancer" pkg "dashing", "nim c tests/functional.nim" -pkg "delaunay", url = "https://github.com/nim-lang/DelaunayNim", useHead = true +pkg "delaunay" +pkg "dnsclient", allowFailure = true # super fragile pkg "docopt" +pkg "dotenv" +# when defined(linux): pkg "drchaos" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" +pkg "faststreams" pkg "fidget" pkg "fragments", "nim c -r fragments/dsl.nim", allowFailure = true # pending https://github.com/nim-lang/packages/issues/2115 pkg "fusion" @@ -72,14 +79,17 @@ pkg "glob" pkg "ggplotnim", "nim c -d:noCairo -r tests/tests.nim" pkg "gittyup", "nimble test", "https://github.com/disruptek/gittyup", allowFailure = true pkg "gnuplot", "nim c gnuplot.nim" -# pkg "gram", "nim c -r --gc:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram" +# pkg "gram", "nim c -r --mm:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram" # pending https://github.com/nim-lang/Nim/issues/16509 pkg "hts", "nim c -o:htss src/hts.nim" pkg "httpauth" +pkg "httputils" pkg "illwill", "nimble examples" pkg "inim" pkg "itertools", "nim doc src/itertools.nim" pkg "iterutils" +pkg "json_rpc" +pkg "json_serialization" pkg "jstin" pkg "karax", "nim c -r tests/tester.nim" pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree" @@ -93,7 +103,8 @@ pkg "memo" pkg "msgpack4nim", "nim c -r tests/test_spec.nim" pkg "nake", "nim c nakefile.nim" pkg "neo", "nim c -d:blas=openblas --mm:refc tests/all.nim" -pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true +pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true, allowFailure = true + # inactive, tests not adapted to #23096 pkg "netty" pkg "nico", allowFailure = true pkg "nicy", "nim c -r src/nicy.nim" @@ -103,22 +114,23 @@ pkg "NimData", "nim c -o:nimdataa src/nimdata.nim" pkg "nimes", "nim c src/nimes.nim" pkg "nimfp", "nim c -o:nfp -r src/fp.nim" pkg "nimgame2", "nim c --mm:refc nimgame2/nimgame.nim" - # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" +pkg "nimib" pkg "nimlsp" pkg "nimly", "nim c -r tests/test_readme_example.nim" pkg "nimongo", "nimble test_ci", allowFailure = true pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true pkg "nimPNG", useHead = true pkg "nimpy", "nim c -r tests/nimfrompy.nim" -pkg "nimquery", url = "https://github.com/nim-lang/nimquery", useHead = true +pkg "nimquery" pkg "nimsl" pkg "nimsvg" -pkg "nimterop", "nimble minitest" +pkg "nimterop", "nimble minitest", url = "https://github.com/nim-lang/nimterop" pkg "nimwc", "nim c nimwc.nim" -pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true +pkg "nimx", "nim c test/main.nim", allowFailure = true pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" -pkg "norm", "testament r tests/sqlite/trows.nim" +pkg "norm", "testament r tests/common/tmodel.nim" +pkg "normalize" pkg "npeg", "nimble testarc" pkg "numericalnim", "nimble nimCI" pkg "optionsutils" @@ -129,23 +141,26 @@ pkg "pixie" pkg "plotly", "nim c examples/all.nim" pkg "pnm" pkg "polypbren" +pkg "presto" pkg "prologue", "nimble tcompile" -pkg "protobuf", "nim c -o:protobuff -r src/protobuf.nim" -pkg "pylib" +# remove fork after https://github.com/PMunch/combparser/pull/7 is merged: +pkg "protobuf", "nimble install -y https://github.com/metagn/combparser@#HEAD; nim c -o:protobuff -r src/protobuf.nim" pkg "rbtree" pkg "react", "nimble example" pkg "regex", "nim c src/regex" -pkg "result", "nim c -r result.nim" +pkg "results", "nim c -r results.nim" pkg "RollingHash", "nim c -r tests/test_cyclichash.nim" pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim" pkg "sdl1", "nim c -r src/sdl.nim" pkg "sdl2_nim", "nim c -r sdl2/sdl.nim" -pkg "sigv4", "nim c --gc:arc -r sigv4.nim", "https://github.com/disruptek/sigv4" +pkg "serialization" +pkg "sigv4", "nim c --mm:arc -r sigv4.nim", "https://github.com/disruptek/sigv4" pkg "sim" pkg "smtp", "nimble compileExample" pkg "snip", "nimble test", "https://github.com/genotrance/snip" +pkg "ssostrings" pkg "stew" -pkg "stint", "nim r stint.nim" +pkg "stint", "nim c stint.nim" pkg "strslice" pkg "strunicode", "nim c -r --mm:refc src/strunicode.nim" pkg "supersnappy" @@ -157,17 +172,22 @@ pkg "templates" pkg "tensordsl", "nim c -r --mm:refc tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" pkg "terminaltables", "nim c src/terminaltables.nim" pkg "termstyle", "nim c -r termstyle.nim" +pkg "testutils" pkg "timeit" pkg "timezones" pkg "tiny_sqlite" pkg "unicodedb", "nim c -d:release -r tests/tests.nim" pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" +pkg "union", "nim c -r tests/treadme.nim", url = "https://github.com/alaviss/union" +pkg "unittest2" pkg "unpack" -pkg "weave", "nimble test_gc_arc", useHead = true +pkg "weave", "nimble install -y cligen@#HEAD; nimble test_gc_arc", useHead = true +pkg "websock" pkg "websocket", "nim c websocket.nim" -pkg "winim", "nim c winim.nim" +# pkg "winim", allowFailure = true pkg "with" pkg "ws", allowFailure = true -pkg "yaml", "nim c -r test/tserialization.nim" -pkg "zero_functional", "nim c -r -d:nimNoLentIterators test.nim" +pkg "yaml" +pkg "zero_functional", "nim c -r test.nim" pkg "zippy" +pkg "zxcvbn" diff --git a/testament/lib/stdtest/specialpaths.nim b/testament/lib/stdtest/specialpaths.nim index 7df63666f..e214d113d 100644 --- a/testament/lib/stdtest/specialpaths.nim +++ b/testament/lib/stdtest/specialpaths.nim @@ -48,7 +48,7 @@ proc splitTestFile*(file: string): tuple[cat: string, path: string] = else: result.path = file return result - doAssert false, "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'" + raiseAssert "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'" static: # sanity check diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index 12e6a56ab..a490b17c8 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -40,7 +40,7 @@ when not defined(js) and not defined(nimscript): yield line template isMatch(lhsi, rhsi): bool = if allowPrefixMatch: - startsWith(rhsi, lhsi): + startsWith(rhsi, lhsi) else: lhsi == rhsi @@ -66,6 +66,11 @@ template enableRemoteNetworking*: bool = ## a `nim` invocation (possibly via additional intermediate processes). getEnv("NIM_TESTAMENT_REMOTE_NETWORKING") == "1" +template disableSSLTesting*: bool = + ## TODO: workaround for GitHub Action gcc 14 matrix; remove this + ## matrix and the flag after Azure agent supports ubuntu 24.04 + getEnv("NIM_TESTAMENT_DISABLE_SSL") == "1" + template whenRuntimeJs*(bodyIf, bodyElse) = ##[ Behaves as `when defined(js) and not nimvm` (which isn't legal yet). diff --git a/testament/specs.nim b/testament/specs.nim index cbb73a5f2..c3040c1d8 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -105,7 +105,7 @@ type proc getCmd*(s: TSpec): string = if s.cmd.len == 0: - result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:build/deps/pkgs $options $file" + result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:build/deps/pkgs2 $options $file" else: result = s.cmd @@ -143,27 +143,26 @@ proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var T ## ## Can parse a single message for a line: ## - ## .. code-block:: nim - ## + ## ```nim ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error ## ^ 'generic_proc' should be: 'genericProc' [Name] ]# + ## ``` ## ## Can parse multiple messages for a line when they are separated by ';': ## - ## .. code-block:: nim - ## + ## ```nim ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error ## ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error ## ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error ## ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` ## - ## .. code-block:: nim - ## + ## ```nim ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error ## ^ 'generic_proc' should be: 'genericProc' [Name]; ## tt.Error ^ 'no_destroy' should be: 'nodestroy' [Name]; ## tt.Error ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# - ## + ## ``` result = i + len(inlineErrorMarker) inc col, len(inlineErrorMarker) let msgLine = line @@ -222,7 +221,10 @@ proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var T while result < s.len-1: if s[result] == '\n': - msg.add '\n' + if result > 0 and s[result - 1] == '\r': + msg[^1] = '\n' + else: + msg.add '\n' inc result inc line col = 1 @@ -511,7 +513,7 @@ proc parseSpec*(filename: string): TSpec = try: msg % ["/", $DirSep, "file", result.filename] except ValueError: - result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unitests-output-message-variable-interpolation')" + result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unit-tests-output-message-variable-interpolation')" msg result.nimout = result.nimout.varSub result.msg = result.msg.varSub diff --git a/testament/testament.nim b/testament/testament.nim index ac99a2373..1e892e636 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -10,15 +10,20 @@ ## This program verifies Nim against the testcases. import - strutils, pegs, os, osproc, streams, json, std/exitprocs, - backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, times, md5, azure, intsets, macros + std/[strutils, pegs, os, osproc, streams, json, + parseopt, browsers, terminal, exitprocs, + algorithm, times, intsets, macros] + +import backend, specs, azure, htmlgen + from std/sugar import dup import compiler/nodejs import lib/stdtest/testutils from lib/stdtest/specialpaths import splitTestFile from std/private/gitutils import diffStrings +import ../dist/checksums/src/checksums/md5 + proc trimUnitSep(x: var string) = let L = x.len if L > 0 and x[^1] == '\31': @@ -29,6 +34,7 @@ var backendLogging = true var simulate = false var optVerbose = false var useMegatest = true +var valgrindEnabled = true proc verboseCmd(cmd: string) = if optVerbose: @@ -60,6 +66,7 @@ Options: --colors:on|off Turn messages coloring on|off. --backendLogging:on|off Disable or enable backend logging. By default turned on. --megatest:on|off Enable or disable megatest. Default is on. + --valgrind:on|off Enable or disable valgrind support. Default is on. --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API @@ -189,7 +196,6 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, foundSuccessMsg = true elif not running(p): break - close(p) result.msg = "" result.file = "" result.output = "" @@ -197,8 +203,9 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, result.column = 0 result.err = reNimcCrash - let exitCode = p.peekExitCode - case exitCode + result.exitCode = p.peekExitCode + close p + case result.exitCode of 0: if foundErrorMsg: result.debugInfo.add " compiler exit code was 0 but some Error's were found." @@ -210,7 +217,7 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, if foundSuccessMsg: result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true." else: - result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $exitCode + result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode if err =~ pegLineError: result.file = extractFilename(matches[0]) @@ -376,7 +383,7 @@ proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer) elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: - r.addResult(test, target, extraOptions, expected.filename, given.file, reFilesDiffer) + r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer) elif expected.line != given.line and expected.line != 0 or expected.column != given.column and expected.column != 0: r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column, @@ -487,7 +494,7 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, args = @["--unhandled-rejections=strict", exeFile] & args else: exeCmd = exeFile.dup(normalizeExe) - if expected.useValgrind != disabled: + if valgrindEnabled and expected.useValgrind != disabled: var valgrindOptions = @["--error-exitcode=1"] if expected.useValgrind != leaking: valgrindOptions.add "--leak-check=yes" @@ -517,7 +524,29 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer) compilerOutputTests(test, target, extraOptions, given, expected, r) of actionReject: + # Make sure its the compiler rejecting and not the system (e.g. segfault) cmpMsgs(r, expected, given, test, target, extraOptions) + if given.exitCode != QuitFailure: + r.addResult(test, target, extraOptions, "exitcode: " & $QuitFailure, + "exitcode: " & $given.exitCode & "\n\nOutput:\n" & + given.nimout, reExitcodesDiffer) + + + +proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget = + result = defaultTarget + var p = parseopt.initOptParser(extraOptions) + + while true: + parseopt.next(p) + case p.kind + of cmdEnd: break + of cmdLongOption, cmdShortOption: + if p.key == "b" or p.key == "backend": + result = parseEnum[TTarget](p.val.normalize) + # chooses the last one + else: + discard proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) = for target in expected.targets: @@ -531,6 +560,7 @@ proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: s else: let nimcache = nimcacheDir(test.name, test.options, target) var testClone = test + let target = changeTarget(extraOptions, target) testSpecHelper(r, testClone, expected, target, extraOptions, nimcache) proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = @@ -664,6 +694,14 @@ proc main() = useMegatest = false else: quit Usage + of "valgrind": + case p.val: + of "on": + valgrindEnabled = true + of "off": + valgrindEnabled = false + else: + quit Usage of "backendlogging": case p.val: of "on": |