diff options
Diffstat (limited to 'testament')
28 files changed, 1438 insertions, 785 deletions
diff --git a/testament/backend.nim b/testament/backend.nim index 0a6d7d8b9..1770c6657 100644 --- a/testament/backend.nim +++ b/testament/backend.nim @@ -13,7 +13,6 @@ type CommitId = distinct string proc `$`*(id: MachineId): string {.borrow.} -#proc `$`(id: CommitId): string {.borrow.} # not used var thisMachine: MachineId @@ -21,10 +20,10 @@ var thisBranch: string proc getMachine*(): MachineId = - var name = execProcess("hostname").string.strip + var name = execProcess("hostname").strip if name.len == 0: - name = when defined(posix): getEnv("HOSTNAME").string - else: getEnv("COMPUTERNAME").string + name = when defined(posix): getEnv("HOSTNAME") + else: getEnv("COMPUTERNAME") if name.len == 0: quit "cannot determine the machine name" @@ -32,8 +31,8 @@ proc getMachine*(): MachineId = proc getCommit(): CommitId = const commLen = "commit ".len - let hash = execProcess("git log -n 1").string.strip[commLen..commLen+10] - thisBranch = execProcess("git symbolic-ref --short HEAD").string.strip + let hash = execProcess("git log -n 1").strip[commLen..commLen+10] + thisBranch = execProcess("git symbolic-ref --short HEAD").strip if hash.len == 0 or thisBranch.len == 0: quit "cannot determine git HEAD" result = CommitId(hash) diff --git a/testament/caasdriver.nim b/testament/caasdriver.nim index 30383bddb..01e402e07 100644 --- a/testament/caasdriver.nim +++ b/testament/caasdriver.nim @@ -62,7 +62,7 @@ proc doCaasCommand(session: var NimSession, command: string): string = result = "" while true: - var line = TaintedString("") + var line = "" if session.nim.outputStream.readLine(line): if line.string == "": break result.add(line.string & "\n") @@ -78,7 +78,7 @@ proc doProcCommand(session: var NimSession, command: string): string = var process = startProcess(NimBin, args = session.replaceVars(command).split) stream = outputStream(process) - line = TaintedString("") + line = "" result = "" while stream.readLine(line): @@ -113,12 +113,12 @@ proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): result = true var f = open(script) - var project = TaintedString("") + var project = "" if f.readLine(project): var s = startNimSession(script.parentDir / project.string, script, mode) - tline = TaintedString("") + tline = "" ln = 1 while f.readLine(tline): @@ -175,7 +175,7 @@ when isMainModule: verbose = false for i in 0..paramCount() - 1: - let param = string(paramStr(i + 1)) + let param = paramStr(i + 1) case param of "verbose": verbose = true else: filter = param diff --git a/testament/categories.nim b/testament/categories.nim index f8cc625c4..843bef3f9 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -10,9 +10,11 @@ ## Include for the tester that contains test suites that test special features ## of the compiler. -# included from tester.nim +# included from testament.nim import important_packages +import std/[strformat, strutils] +from std/sequtils import filterIt const specialCategories = [ @@ -21,26 +23,21 @@ const "debugger", "dll", "examples", - "flags", "gc", "io", "js", "ic", "lib", - "longgc", "manyloc", "nimble-packages", "niminaction", - "rodfiles", "threads", - "untestable", - "stdlib", + "untestable", # see trunner_special "testdata", "nimcache", "coroutines", "osproc", "shouldfail", - "dir with space", "destructor" ] @@ -48,72 +45,9 @@ proc isTestFile*(file: string): bool = let (_, name, ext) = splitFile(file) result = ext == ".nim" and name.startsWith("t") -# ---------------- IC tests --------------------------------------------- - -proc icTests(r: var TResults; testsDir: string, cat: Category, options: string) = - const - tooltests = ["compiler/nim.nim", "tools/nimgrep.nim"] - writeOnly = " --incremental:writeonly " - readOnly = " --incremental:readonly " - incrementalOn = " --incremental:on " - - template test(x: untyped) = - testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) - - template editedTest(x: untyped) = - var test = makeTest(file, x & options, cat) - test.spec.targets = {getTestSpecTarget()} - testSpecWithNimcache(r, test, nimcache) - - const tempExt = "_temp.nim" - for it in walkDirRec(testsDir / "ic"): - if isTestFile(it) and not it.endsWith(tempExt): - let nimcache = nimcacheDir(it, options, getTestSpecTarget()) - removeDir(nimcache) - - let content = readFile(it) - for fragment in content.split("#!EDIT!#"): - let file = it.replace(".nim", tempExt) - writeFile(file, fragment) - let oldPassed = r.passed - editedTest incrementalOn - if r.passed != oldPassed+1: break - - for file in tooltests: - let nimcache = nimcacheDir(file, options, getTestSpecTarget()) - removeDir(nimcache) - - let oldPassed = r.passed - test writeOnly - - if r.passed == oldPassed+1: - test readOnly - if r.passed == oldPassed+2: - test readOnly & "-d:nimBackendAssumesChange " - -# --------------------- flags tests ------------------------------------------- - -proc flagTests(r: var TResults, cat: Category, options: string) = - # --genscript - const filename = testsDir/"flags"/"tgenscript" - const genopts = " --genscript" - let nimcache = nimcacheDir(filename, genopts, targetC) - testSpec r, makeTest(filename, genopts, cat) - - when defined(windows): - testExec r, makeTest(filename, " cmd /c cd " & nimcache & - " && compile_tgenscript.bat", cat) - - elif defined(posix): - testExec r, makeTest(filename, " sh -c \"cd " & nimcache & - " && sh compile_tgenscript.sh\"", cat) - - # Run - testExec r, makeTest(filename, " " & nimcache / "tgenscript", cat) - # --------------------- DLL generation tests ---------------------------------- -proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = +proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = false) = const rpath = when defined(macosx): " --passL:-rpath --passL:@loader_path" else: @@ -125,42 +59,53 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) test2.spec.action = actionCompile testSpec c, test2 - var test3 = makeTest("lib/nimhcr.nim", options & " --outdir:tests/dll" & rpath, cat) + + var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) test3.spec.action = actionCompile testSpec c, test3 - var test4 = makeTest("tests/dll/visibility.nim", options & " --app:lib" & rpath, cat) + var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat) test4.spec.action = actionCompile testSpec c, test4 # windows looks in the dir of the exe (yay!): - when not defined(Windows): + when not defined(windows): # posix relies on crappy LD_LIBRARY_PATH (ugh!): const libpathenv = when defined(haiku): "LIBRARY_PATH" else: "LD_LIBRARY_PATH" - var libpath = getEnv(libpathenv).string + var libpath = getEnv(libpathenv) # Temporarily add the lib directory to LD_LIBRARY_PATH: putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: "")) defer: putEnv(libpathenv, libpath) testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) - testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & rpath, cat) - testSpec r, makeTest("tests/dll/visibility.nim", options & rpath, cat) + testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) + testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) if "boehm" notin options: + # hcr tests + + var basicHcrTest = makeTest("tests/dll/nimhcr_basic.nim", options & " --threads:off --forceBuild --hotCodeReloading:on " & rpath, cat) + # test segfaults for now but compiles: + if isOrc: basicHcrTest.spec.action = actionCompile + testSpec r, basicHcrTest + # force build required - see the comments in the .nim file for more details var hcri = makeTest("tests/dll/nimhcr_integration.nim", - options & " --forceBuild --hotCodeReloading:on" & rpath, cat) + options & " --threads:off --forceBuild --hotCodeReloading:on" & rpath, cat) let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget()) - hcri.args = prepareTestArgs(hcri.spec.getCmd, hcri.name, + let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name, hcri.options, nimcache, getTestSpecTarget()) + hcri.testArgs = cmd.parseCmdLine testSpec r, hcri proc dllTests(r: var TResults, cat: Category, options: string) = # dummy compile result: var c = initResults() - runBasicDLLTest c, r, cat, options - runBasicDLLTest c, r, cat, options & " -d:release" + runBasicDLLTest c, r, cat, options & " --mm:refc" + runBasicDLLTest c, r, cat, options & " -d:release --mm:refc" + runBasicDLLTest c, r, cat, options, isOrc = true + runBasicDLLTest c, r, cat, options & " -d:release", isOrc = true when not defined(windows): # still cannot find a recent Windows version of boehm.dll: runBasicDLLTest c, r, cat, options & " --gc:boehm" @@ -169,18 +114,10 @@ proc dllTests(r: var TResults, cat: Category, options: string) = # ------------------------------ GC tests ------------------------------------- proc gcTests(r: var TResults, cat: Category, options: string) = - template testWithNone(filename: untyped) = - testSpec r, makeTest("tests/gc" / filename, options & - " --gc:none", cat) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release --gc:none", cat) - template testWithoutMs(filename: untyped) = - testSpec r, makeTest("tests/gc" / filename, options, cat) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release", cat) + testSpec r, makeTest("tests/gc" / filename, options & "--mm:refc", cat) testSpec r, makeTest("tests/gc" / filename, options & - " -d:release -d:useRealtimeGC", cat) + " -d:release -d:useRealtimeGC --mm:refc", cat) when filename != "gctest": testSpec r, makeTest("tests/gc" / filename, options & " --gc:orc", cat) @@ -193,6 +130,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) = " --gc:markAndSweep", cat) testSpec r, makeTest("tests/gc" / filename, options & " -d:release --gc:markAndSweep", cat) + template test(filename: untyped) = testWithoutBoehm filename when not defined(windows) and not defined(android): @@ -210,7 +148,6 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "gcleak" test "gcleak2" testWithoutBoehm "gctest" - testWithNone "gctest" test "gcleak3" test "gcleak4" # Disabled because it works and takes too long to run: @@ -225,19 +162,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "stackrefleak" test "cyclecollector" - -proc longGCTests(r: var TResults, cat: Category, options: string) = - when defined(windows): - let cOptions = "-ldl -DWIN" - else: - let cOptions = "-ldl" - - var c = initResults() - # According to ioTests, this should compile the file - testSpec c, makeTest("tests/realtimeGC/shared", options, cat) - # ^- why is this not appended to r? Should this be discarded? - testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun - testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat) + testWithoutBoehm "trace_globals" # ------------------------- threading tests ----------------------------------- @@ -256,6 +181,11 @@ proc ioTests(r: var TResults, cat: Category, options: string) = # dummy compile result: var c = initResults() testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat) + # ^- why is this not appended to r? Should this be discarded? + # EDIT: this should be replaced by something like in D20210524T180826, + # likewise in similar instances where `testSpec c` is used, or more generally + # when a test depends on another test, as it makes tests non-independent, + # creating complications for batching and megatest logic. testSpec r, makeTest("tests/system/tio", options, cat) # ------------------------- async tests --------------------------------------- @@ -268,25 +198,29 @@ proc asyncTests(r: var TResults, cat: Category, options: string) = # ------------------------- debugger tests ------------------------------------ proc debuggerTests(r: var TResults, cat: Category, options: string) = - var t = makeTest("tools/nimgrep", options & " --debugger:on", cat) - t.spec.action = actionCompile - testSpec r, t + if fileExists("tools/nimgrep.nim"): + var t = makeTest("tools/nimgrep", options & " --debugger:on", cat) + t.spec.action = actionCompile + # force target to C because of MacOS 10.15 SDK headers bug + # https://github.com/nim-lang/Nim/pull/15612#issuecomment-712471879 + t.spec.targets = {targetC} + testSpec r, t # ------------------------- JS tests ------------------------------------------ proc jsTests(r: var TResults, cat: Category, options: string) = template test(filename: untyped) = - testSpec r, makeTest(filename, options & " -d:nodejs", cat), {targetJS} - testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat), {targetJS} + testSpec r, makeTest(filename, options, cat), {targetJS} + testSpec r, makeTest(filename, options & " -d:release", cat), {targetJS} for t in os.walkFiles("tests/js/t*.nim"): test(t) for testfile in ["exception/texceptions", "exception/texcpt1", "exception/texcsub", "exception/tfinally", "exception/tfinally2", "exception/tfinally3", - "actiontable/tactiontable", "method/tmultimjs", + "collections/tactiontable", "method/tmultimjs", "varres/tvarres0", "varres/tvarres3", "varres/tvarres4", - "varres/tvartup", "misc/tints", "misc/tunsignedinc", + "varres/tvartup", "int/tints", "int/tunsignedinc", "async/tjsandnativeasync"]: test "tests/" & testfile & ".nim" @@ -296,8 +230,6 @@ proc jsTests(r: var TResults, cat: Category, options: string) = # ------------------------- nim in action ----------- proc testNimInAction(r: var TResults, cat: Category, options: string) = - let options = options & " --nilseqs:on" - template test(filename: untyped) = testSpec r, makeTest(filename, options, cat) @@ -388,7 +320,7 @@ proc findMainFile(dir: string): string = var nimFiles = 0 for kind, file in os.walkDir(dir): if kind == pcFile: - if file.endsWith(cfgExt): return file[.. ^(cfgExt.len+1)] & ".nim" + if file.endsWith(cfgExt): return file[0..^(cfgExt.len+1)] & ".nim" elif file.endsWith(".nim"): if result.len == 0: result = file inc nimFiles @@ -418,7 +350,7 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = proc isValid(file: string): bool = for dir in parentDirs(file, inclusive = false): if dir.lastPathPart in ["includes", "nimcache"]: - # eg: lib/pure/includes/osenv.nim gives: Error: This is an include file for os.nim! + # e.g.: lib/pure/includes/osenv.nim gives: Error: This is an include file for os.nim! return false let name = extractFilename(file) if name.splitFile.ext != ".nim": return false @@ -433,12 +365,12 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = files.sort # reproducible order for testFile in files: - let contents = readFile(testFile).string + let contents = readFile(testFile) var testObj = makeTest(testFile, options, cat) #[ todo: this logic is fragile: - false positives (if appears in a comment), or false negatives, eg + false positives (if appears in a comment), or false negatives, e.g. `when defined(osx) and isMainModule`. Instead of fixing this, see https://github.com/nim-lang/Nim/issues/10045 for a much better way. @@ -448,114 +380,183 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = testSpec r, testObj # ----------------------------- nimble ---------------------------------------- - -var nimbleDir = getEnv("NIMBLE_DIR").string -if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" -let - nimbleExe = findExe("nimble") - packageIndex = nimbleDir / "packages_official.json" - -iterator listPackages(): tuple[name, url, cmd: string, hasDeps: bool] = - let defaultCmd = "nimble test" +proc listPackagesAll(): seq[NimblePackage] = + var nimbleDir = getEnv("NIMBLE_DIR") + if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" + let packageIndex = nimbleDir / "packages_official.json" let packageList = parseFile(packageIndex) - for n, cmd, hasDeps, url in important_packages.packages.items: - let cmd = if cmd.len == 0: defaultCmd else: cmd - if url.len != 0: - yield (n, url, cmd, hasDeps) + proc findPackage(name: string): JsonNode = + for a in packageList: + if a["name"].str == name: return a + for pkg in important_packages.packages.items: + var pkg = pkg + if pkg.url.len == 0: + let pkg2 = findPackage(pkg.name) + if pkg2 == nil: + raise newException(ValueError, "Cannot find package '$#'." % pkg.name) + pkg.url = pkg2["url"].str + result.add pkg + +proc listPackages(packageFilter: string): seq[NimblePackage] = + let pkgs = listPackagesAll() + if packageFilter.len != 0: + # xxx document `packageFilter`, seems like a bad API, + # at least should be a regex; a substring match makes no sense. + result = pkgs.filterIt(packageFilter in it.name) + else: + if testamentData0.batchArg == "allowed_failures": + result = pkgs.filterIt(it.allowFailure) + elif testamentData0.testamentNumBatch == 0: + result = pkgs else: - var found = false - for package in packageList.items: - let name = package["name"].str - if name == n: - found = true - let pUrl = package["url"].str - yield (name, pUrl, cmd, hasDeps) - break - if not found: - raise newException(ValueError, "Cannot find package '$#'." % n) - -proc makeSupTest(test, options: string, cat: Category): TTest = + let pkgs2 = pkgs.filterIt(not it.allowFailure) + for i in 0..<pkgs2.len: + if i mod testamentData0.testamentNumBatch == testamentData0.testamentBatch: + result.add pkgs2[i] + +proc makeSupTest(test, options: string, cat: Category, debugInfo = ""): TTest = result.cat = cat result.name = test result.options = options + result.debugInfo = debugInfo result.startTime = epochTime() -proc testNimblePackages(r: var TResults, cat: Category) = - if nimbleExe == "": - echo "[Warning] - Cannot run nimble tests: Nimble binary not found." - return - if execCmd("$# update" % nimbleExe) == QuitFailure: - echo "[Warning] - Cannot run nimble tests: Nimble update failed." - return +import std/private/gitutils +proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = + let nimbleExe = findExe("nimble") + doAssert nimbleExe != "", "Cannot run nimble tests: Nimble binary not found." + doAssert execCmd("$# update" % nimbleExe) == 0, "Cannot run nimble tests: Nimble update failed." let packageFileTest = makeSupTest("PackageFileParsed", "", cat) let packagesDir = "pkgstemp" + createDir(packagesDir) var errors = 0 try: - for name, url, cmd, hasDep in listPackages(): + let pkgs = listPackages(packageFilter) + for i, pkg in pkgs: inc r.total - var test = makeSupTest(url, "", cat) - let buildPath = packagesDir / name - if not existsDir(buildPath): - if hasDep: - let installName = if url.len != 0: url else: name - let (nimbleCmdLine, nimbleOutput, nimbleStatus) = execCmdEx2("nimble", ["install", "-y", installName]) - if nimbleStatus != QuitSuccess: - let message = "nimble install failed:\n$ " & nimbleCmdLine & "\n" & nimbleOutput - r.addResult(test, targetC, "", message, reInstallFailed) - continue - - let (installCmdLine, installOutput, installStatus) = execCmdEx2("git", ["clone", url, buildPath]) - if installStatus != QuitSuccess: - let message = "git clone failed:\n$ " & installCmdLine & "\n" & installOutput - r.addResult(test, targetC, "", message, reInstallFailed) + var test = makeSupTest(pkg.name, "", cat, "[$#/$#] " % [$i, $pkgs.len]) + let buildPath = packagesDir / pkg.name + template tryCommand(cmd: string, workingDir2 = buildPath, reFailed = reInstallFailed, maxRetries = 1): string = + var outp: string + let ok = retryCall(maxRetry = maxRetries, backoffDuration = 10.0): + var status: int + (outp, status) = execCmdEx(cmd, workingDir = workingDir2) + status == QuitSuccess + if not ok: + if pkg.allowFailure: + inc r.passed + inc r.failedButAllowed + addResult(r, test, targetC, "", "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure) continue - - let cmdArgs = parseCmdLine(cmd) - - let (buildCmdLine, buildOutput, buildStatus) = execCmdEx2(cmdArgs[0], cmdArgs[1..^1], workingDir=buildPath) - if buildStatus != QuitSuccess: - let message = "package test failed\n$ " & buildCmdLine & "\n" & buildOutput - r.addResult(test, targetC, "", message, reBuildFailed) - else: - inc r.passed - r.addResult(test, targetC, "", "", reSuccess) + outp + + if not dirExists(buildPath): + discard tryCommand("git clone $# $#" % [pkg.url.quoteShell, buildPath.quoteShell], workingDir2 = ".", maxRetries = 3) + if not pkg.useHead: + discard tryCommand("git fetch --tags", maxRetries = 3) + let describeOutput = tryCommand("git describe --tags --abbrev=0") + discard tryCommand("git checkout $#" % [describeOutput.strip.quoteShell]) + discard tryCommand("nimble install --depsOnly -y", maxRetries = 3) + let cmds = pkg.cmd.split(';') + for i in 0 ..< cmds.len - 1: + discard tryCommand(cmds[i], maxRetries = 3) + discard tryCommand(cmds[^1], reFailed = reBuildFailed) + inc r.passed + r.addResult(test, targetC, "", "", "", reSuccess, allowFailure = pkg.allowFailure) errors = r.total - r.passed if errors == 0: - r.addResult(packageFileTest, targetC, "", "", reSuccess) + r.addResult(packageFileTest, targetC, "", "", "", reSuccess) else: - r.addResult(packageFileTest, targetC, "", "", reBuildFailed) + r.addResult(packageFileTest, targetC, "", "", "", reBuildFailed) except JsonParsingError: - echo "[Warning] - Cannot run nimble tests: Invalid package file." - r.addResult(packageFileTest, targetC, "", "Invalid package file", reBuildFailed) + errors = 1 + r.addResult(packageFileTest, targetC, "", "", "Invalid package file", reBuildFailed) + raise except ValueError: - echo "[Warning] - $#" % getCurrentExceptionMsg() - r.addResult(packageFileTest, targetC, "", "Unknown package", reBuildFailed) + errors = 1 + r.addResult(packageFileTest, targetC, "", "", "Unknown package", reBuildFailed) + raise # bug #18805 finally: if errors == 0: removeDir(packagesDir) +# ---------------- IC tests --------------------------------------------- + +proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; + isNavigatorTest: bool) = + const + tooltests = ["compiler/nim.nim"] + writeOnly = " --incremental:writeonly " + readOnly = " --incremental:readonly " + incrementalOn = " --incremental:on -d:nimIcIntegrityChecks " + navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint:Conf:off --warnings:off " + + template test(x: untyped) = + testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) + + template editedTest(x: untyped) = + var test = makeTest(file, x & options, cat) + if isNavigatorTest: + test.spec.action = actionCompile + test.spec.targets = {getTestSpecTarget()} + testSpecWithNimcache(r, test, nimcache) + + template checkTest() = + var test = makeRawTest(file, options, cat) + test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:on $options " & file + testSpecWithNimcache(r, test, nimcache) + + if not isNavigatorTest: + for file in tooltests: + let nimcache = nimcacheDir(file, options, getTestSpecTarget()) + removeDir(nimcache) + + let oldPassed = r.passed + checkTest() + + if r.passed == oldPassed+1: + checkTest() + if r.passed == oldPassed+2: + checkTest() + + const tempExt = "_temp.nim" + for it in walkDirRec(testsDir): + # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test + if isTestFile(it) and not it.endsWith(tempExt): + let nimcache = nimcacheDir(it, options, getTestSpecTarget()) + removeDir(nimcache) + + let content = readFile(it) + for fragment in content.split("#!EDIT!#"): + let file = it.replace(".nim", tempExt) + writeFile(file, fragment) + let oldPassed = r.passed + editedTest(if isNavigatorTest: navTestConfig else: incrementalOn) + if r.passed != oldPassed+1: break # ---------------------------------------------------------------------------- -const AdditionalCategories = ["debugger", "examples", "lib", "ic"] +const AdditionalCategories = ["debugger", "examples", "lib", "ic", "navigator"] const MegaTestCat = "megatest" proc `&.?`(a, b: string): string = # candidate for the stdlib? result = if b.startsWith(a): b else: a & b -proc processSingleTest(r: var TResults, cat: Category, options, test: string) = - let test = testsDir &.? cat.string / test - let target = if cat.string.normalize == "js": targetJS else: targetC - if existsFile(test): - testSpec r, makeTest(test, options, cat), {target} - else: - echo "[Warning] - ", test, " test does not exist" +proc processSingleTest(r: var TResults, cat: Category, options, test: string, targets: set[TTarget], targetsSet: bool) = + var targets = targets + if not targetsSet: + let target = if cat.string.normalize == "js": targetJS else: targetC + targets = {target} + doAssert fileExists(test), test & " test does not exist" + testSpec r, makeTest(test, options, cat), targets proc isJoinableSpec(spec: TSpec): bool = - result = not spec.sortoutput and + # xxx simplify implementation using a whitelist of fields that are allowed to be + # set to non-default values (use `fieldPairs`), to avoid issues like bug #16576. + result = useMegatest and not spec.sortoutput and spec.action == actionRun and not fileExists(spec.file.changeFileExt("cfg")) and not fileExists(spec.file.changeFileExt("nims")) and @@ -567,36 +568,48 @@ proc isJoinableSpec(spec: TSpec): bool = spec.exitCode == 0 and spec.input.len == 0 and spec.nimout.len == 0 and + spec.nimoutFull == false and + # so that tests can have `nimoutFull: true` with `nimout.len == 0` with + # the meaning that they expect empty output. + spec.matrix.len == 0 and spec.outputCheck != ocSubstr and spec.ccodeCheck.len == 0 and (spec.targets == {} or spec.targets == {targetC}) - -proc norm(s: var string) = - while true: - let tmp = s.replace("\n\n", "\n") - if tmp == s: break - s = tmp - s = s.strip + if result: + if spec.file.readFile.contains "when isMainModule": + result = false proc quoted(a: string): string = # todo: consider moving to system.nim result.addQuoted(a) -proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = +proc runJoinedTest(r: var TResults, cat: Category, testsDir: string, options: string) = ## returns a list of tests that have problems + #[ + xxx create a reusable megatest API after abstracting out testament specific code, + refs https://github.com/timotheecour/Nim/issues/655 + and https://github.com/nim-lang/gtk2/pull/28; it's useful in other contexts. + ]# var specs: seq[TSpec] = @[] for kind, dir in walkDir(testsDir): - assert testsDir.startsWith(testsDir) + assert dir.startsWith(testsDir) let cat = dir[testsDir.len .. ^1] if kind == pcDir and cat notin specialCategories: for file in walkDirRec(testsDir / cat): if isTestFile(file): - let spec = parseSpec(file) + var spec: TSpec + try: + spec = parseSpec(file) + except ValueError: + # e.g. for `tests/navigator/tincludefile.nim` which have multiple + # specs; this will be handled elsewhere + echo "parseSpec raised ValueError for: '$1', assuming this will be handled outside of megatest" % file + continue if isJoinableSpec(spec): specs.add spec - proc cmp(a: TSpec, b:TSpec): auto = cmp(a.file, b.file) - sort(specs, cmp=cmp) # reproducible order + proc cmp(a: TSpec, b: TSpec): auto = cmp(a.file, b.file) + sort(specs, cmp = cmp) # reproducible order echo "joinable specs: ", specs.len if simulate: @@ -606,142 +619,156 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = return var megatest: string - #[ - TODO(minor): - get from Nim cmd - put outputGotten.txt, outputGotten.txt, megatest.nim there too - delete upon completion, maybe - ]# + # xxx (minor) put outputExceptedFile, outputGottenFile, megatestFile under here or `buildDir` var outDir = nimcacheDir(testsDir / "megatest", "", targetC) - const marker = "megatest:processing: " - + template toMarker(file, i): string = + "megatest:processing: [$1] $2" % [$i, file] for i, runSpec in specs: let file = runSpec.file - let file2 = outDir / ("megatest_" & $i & ".nim") + let file2 = outDir / ("megatest_a_$1.nim" % $i) # `include` didn't work with `trecmod2.nim`, so using `import` - let code = "echo \"" & marker & "\", " & quoted(file) & "\n" + let code = "echo $1\nstatic: echo \"CT:\", $1\n" % [toMarker(file, i).quoted] createDir(file2.parentDir) writeFile(file2, code) - megatest.add "import " & quoted(file2) & "\n" - megatest.add "import " & quoted(file) & "\n" + megatest.add "import $1\nimport $2 as megatest_b_$3\n" % [file2.quoted, file.quoted, $i] - writeFile("megatest.nim", megatest) + let megatestFile = testsDir / "megatest.nim" # so it uses testsDir / "config.nims" + writeFile(megatestFile, megatest) - let args = ["c", "--nimCache:" & outDir, "-d:testing", "--listCmd", - "--listFullPaths:off", "--excessiveStackTrace:off", "megatest.nim"] + let root = getCurrentDir() + var args = @["c", "--nimCache:" & outDir, "-d:testing", "-d:nimMegatest", "--listCmd", + "--path:" & root] + args.add options.parseCmdLine + args.add megatestFile var (cmdLine, buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, input = "") if exitCode != 0: - echo "$ ", cmdLine - echo buf.string - quit("megatest compilation failed") + echo "$ " & cmdLine & "\n" & buf + quit(failString & "megatest compilation failed") - (buf, exitCode) = execCmdEx("./megatest") + (buf, exitCode) = execCmdEx(megatestFile.changeFileExt(ExeExt).dup normalizeExe) if exitCode != 0: - echo buf.string - quit("megatest execution failed") + echo buf + quit(failString & "megatest execution failed") - norm buf.string - writeFile("outputGotten.txt", buf.string) + const outputExceptedFile = "outputExpected.txt" + const outputGottenFile = "outputGotten.txt" + writeFile(outputGottenFile, buf) var outputExpected = "" for i, runSpec in specs: - outputExpected.add marker & runSpec.file & "\n" - outputExpected.add runSpec.output.strip - outputExpected.add '\n' - norm outputExpected - - if buf.string != outputExpected: - writeFile("outputExpected.txt", outputExpected) - discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt") - echo "output different!" - # outputGotten.txt, outputExpected.txt not removed on purpose for debugging. + outputExpected.add toMarker(runSpec.file, i) & "\n" + if runSpec.output.len > 0: + outputExpected.add runSpec.output + if not runSpec.output.endsWith "\n": + outputExpected.add '\n' + + if buf != outputExpected: + writeFile(outputExceptedFile, outputExpected) + echo diffFiles(outputGottenFile, outputExceptedFile).output + echo failString & "megatest output different, see $1 vs $2" % [outputGottenFile, outputExceptedFile] + # outputGottenFile, outputExceptedFile not removed on purpose for debugging. quit 1 else: - echo "output OK" - removeFile("outputGotten.txt") - removeFile("megatest.nim") - #testSpec r, makeTest("megatest", options, cat) + echo "megatest output OK" + # --------------------------------------------------------------------------- proc processCategory(r: var TResults, cat: Category, options, testsDir: string, runJoinableTests: bool) = - case cat.string.normalize - of "rodfiles": - when false: - compileRodFiles(r, cat, options) - runRodFiles(r, cat, options) - of "ic": - when false: - icTests(r, testsDir, cat, options) - of "js": - # only run the JS tests on Windows or Linux because Travis is bad - # and other OSes like Haiku might lack nodejs: - if not defined(linux) and isTravis: + let cat2 = cat.string.normalize + var handled = false + if isNimRepoTests(): + handled = true + case cat2 + of "js": + # only run the JS tests on Windows or Linux because Travis is bad + # and other OSes like Haiku might lack nodejs: + if not defined(linux) and isTravis: + discard + else: + jsTests(r, cat, options) + of "dll": + dllTests(r, cat, options & " -d:nimDebugDlOpen") + of "gc": + gcTests(r, cat, options) + of "debugger": + debuggerTests(r, cat, options) + of "manyloc": + manyLoc r, cat, options + of "threads": + threadTests r, cat, options & " --threads:on" + of "io": + ioTests r, cat, options + of "async": + asyncTests r, cat, options + of "lib": + testStdlib(r, "lib/pure/", options, cat) + testStdlib(r, "lib/packages/docutils/", options, cat) + of "examples": + compileExample(r, "examples/*.nim", options, cat) + compileExample(r, "examples/gtk/*.nim", options, cat) + compileExample(r, "examples/talk/*.nim", options, cat) + of "nimble-packages": + testNimblePackages(r, cat, options) + of "niminaction": + testNimInAction(r, cat, options) + of "ic": + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=false) + of "navigator": + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=true) + of "untestable": + # These require special treatment e.g. because they depend on a third party + # dependency; see `trunner_special` which runs some of those. discard else: - jsTests(r, cat, options) - of "dll": - dllTests(r, cat, options) - of "flags": - flagTests(r, cat, options) - of "gc": - gcTests(r, cat, options) - of "longgc": - longGCTests(r, cat, options) - of "debugger": - debuggerTests(r, cat, options) - of "manyloc": - manyLoc r, cat, options - of "threads": - threadTests r, cat, options & " --threads:on" - of "io": - ioTests r, cat, options - of "async": - asyncTests r, cat, options - of "lib": - testStdlib(r, "lib/pure/", options, cat) - testStdlib(r, "lib/packages/docutils/", options, cat) - of "examples": - compileExample(r, "examples/*.nim", options, cat) - compileExample(r, "examples/gtk/*.nim", options, cat) - compileExample(r, "examples/talk/*.nim", options, cat) - of "nimble-packages": - testNimblePackages(r, cat) - of "niminaction": - testNimInAction(r, cat, options) - of "untestable": - # We can't test it because it depends on a third party. - discard # TODO: Move untestable tests to someplace else, i.e. nimble repo. - of "megatest": - runJoinedTest(r, cat, testsDir) - else: - var testsRun = 0 - var files: seq[string] - for file in walkDirRec(testsDir &.? cat.string): - if isTestFile(file): files.add file - files.sort # give reproducible order - - for i, name in files: - var test = makeTest(name, options, cat) - if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories: - discard "run the test" - else: - test.spec.err = reJoined - testSpec r, test - inc testsRun - if testsRun == 0: - echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run" + handled = false + if not handled: + case cat2 + of "megatest": + runJoinedTest(r, cat, testsDir, options) + if isNimRepoTests(): + runJoinedTest(r, cat, testsDir, options & " --mm:refc") + else: + var testsRun = 0 + var files: seq[string] + for file in walkDirRec(testsDir &.? cat.string): + if isTestFile(file): files.add file + files.sort # give reproducible order + for i, name in files: + var test = makeTest(name, options, cat) + if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories: + discard "run the test" + else: + test.spec.err = reJoined + testSpec r, test + inc testsRun + if testsRun == 0: + const whiteListedDirs = ["deps", "htmldocs", "pkgs"] + # `pkgs` because bug #16556 creates `pkgs` dirs and this can affect some users + # that try an old version of choosenim. + doAssert cat.string in whiteListedDirs, + "Invalid category specified: '$#' not in whilelist: $#" % [cat.string, $whiteListedDirs] proc processPattern(r: var TResults, pattern, options: string; simulate: bool) = var testsRun = 0 - for name in walkPattern(pattern): - if simulate: - echo "Detected test: ", name - else: - var test = makeTest(name, options, Category"pattern") - testSpec r, test - inc testsRun + if dirExists(pattern): + for k, name in walkDir(pattern): + if k in {pcFile, pcLinkToFile} and name.endsWith(".nim"): + if simulate: + echo "Detected test: ", name + else: + var test = makeTest(name, options, Category"pattern") + testSpec r, test + inc testsRun + else: + for name in walkPattern(pattern): + if simulate: + echo "Detected test: ", name + else: + var test = makeTest(name, options, Category"pattern") + testSpec r, test + inc testsRun if testsRun == 0: echo "no tests were found for pattern: ", pattern diff --git a/testament/htmlgen.nim b/testament/htmlgen.nim index ee3f3bad7..174f36d0b 100644 --- a/testament/htmlgen.nim +++ b/testament/htmlgen.nim @@ -16,7 +16,7 @@ import "testamenthtml.nimf" proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) = let trId = htmlQuote(testResultRow["category"].str & "_" & testResultRow["name"].str). - multiReplace({".": "_", " ": "_", ":": "_"}) + multiReplace({".": "_", " ": "_", ":": "_", "/": "_"}) name = testResultRow["name"].str.htmlQuote() category = testResultRow["category"].str.htmlQuote() target = testResultRow["target"].str.htmlQuote() @@ -52,7 +52,7 @@ proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) = trId, name, target, category, action, resultDescription, timestamp, result, resultSign, panelCtxClass, textCtxClass, bgCtxClass ) - if expected.isNilOrWhitespace() and gotten.isNilOrWhitespace(): + if expected.isEmptyOrWhitespace() and gotten.isEmptyOrWhitespace(): outfile.generateHtmlTestresultOutputNone() else: outfile.generateHtmlTestresultOutputDetails( diff --git a/testament/important_packages.nim b/testament/important_packages.nim index f2ce8fb25..efec04b3c 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -1,95 +1,193 @@ +##[ +## note 1 +`useHead` should ideally be used as the default but lots of packages (e.g. `chronos`) +don't have release tags (or have really old ones compared to HEAD), making it +impossible to test them reliably here. -template pkg(name: string; cmd = "nimble test"; hasDeps = false; url = ""): untyped = - packages.add((name, cmd, hasDeps, url)) +packages listed here should ideally have regularly updated release tags, so that: +* we're testing recent versions of the package +* the version that's tested is stable enough even if HEAD may occasionally break -var packages*: seq[tuple[name, cmd: string; hasDeps: bool; url: string]] = @[] +## note 2: D20210308T165435:here +nimble packages should be testable as follows: +git clone $url $dir && cd $dir +NIMBLE_DIR=$TMP_NIMBLE_DIR XDG_CONFIG_HOME= nimble install --depsOnly -y +NIMBLE_DIR=$TMP_NIMBLE_DIR XDG_CONFIG_HOME= nimble test +if this fails (e.g. nimcrypto), it could be because a package lacks a `tests/nim.cfg` with `--path:..`, +so the above commands would've worked by accident with `nimble install` but not with `nimble install --depsOnly`. +When this is the case, a workaround is to test this package here by adding `--path:$srcDir` on the test `cmd`. +]## +type NimblePackage* = object + name*, cmd*, url*: string + useHead*: bool + allowFailure*: bool + ## When true, we still run the test but the test is allowed to fail. + ## This is useful for packages that currently fail but that we still want to + ## run in CI, e.g. so that we can monitor when they start working again and + ## are reminded about those failures without making CI fail for unrelated PRs. + +var packages*: seq[NimblePackage] + +proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true, allowFailure = false) = + packages.add NimblePackage(name: name, cmd: cmd, url: url, useHead: useHead, allowFailure: allowFailure) + +pkg "alea" pkg "argparse" -pkg "arraymancer", "nim c tests/tests_cpu.nim", true -pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim" -pkg "asyncmysql", "", true +pkg "arraymancer", "nim c tests/tests_cpu.nim" +pkg "ast_pattern_matching", "nim c -r tests/test1.nim" +pkg "asyncftpclient", "nimble compileExample" +pkg "asyncthreadpool", "nimble test --mm:refc" +pkg "awk" pkg "bigints" pkg "binaryheap", "nim c -r binaryheap.nim" -pkg "blscurve", "", true -pkg "bncurve", "", true +pkg "BipBuffer" +pkg "blscurve", allowFailure = true +pkg "bncurve" +pkg "brainfuck", "nim c -d:release -r tests/compile.nim" +pkg "bump", "nim c --mm:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump", allowFailure = true pkg "c2nim", "nim c testsuite/tester.nim" pkg "cascade" +pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true +pkg "checksums" pkg "chroma" -pkg "chronicles", "nim c -o:chr -r chronicles.nim", true -pkg "chronos", "", true -pkg "cligen", "nim c -o:cligenn -r cligen.nim" -pkg "coco", "", true -pkg "combparser" +pkg "chronicles", "nim c -o:chr -r chronicles.nim" +pkg "chronos", "nim c -r -d:release tests/testall" +pkg "cligen", "nim c --path:. -r cligen.nim" +pkg "combparser", "nimble test --mm:orc" pkg "compactdict" -pkg "comprehension", "", false, "https://github.com/alehander42/comprehension" -pkg "criterion" +pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehension" +pkg "constantine", "nimble make_lib" +pkg "cowstrings" +pkg "criterion", allowFailure = true # needs testing binary +pkg "datamancer" pkg "dashing", "nim c tests/functional.nim" +pkg "delaunay" +pkg "dnsclient", allowFailure = true # super fragile pkg "docopt" -pkg "easygl", "nim c -o:egl -r src/easygl.nim", true, "https://github.com/jackmott/easygl" +pkg "dotenv" +# when defined(linux): pkg "drchaos" +pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" -pkg "fragments", "nim c -r fragments/dsl.nim" +pkg "faststreams" +pkg "fidget" +pkg "fragments", "nim c -r fragments/dsl.nim", allowFailure = true # pending https://github.com/nim-lang/packages/issues/2115 +pkg "fusion" pkg "gara" pkg "glob" -pkg "gnuplot" -# pkg "godot", "nim c -r godot/godot.nim" # not yet compatible with Nim 0.19 +pkg "ggplotnim", "nim c -d:noCairo -r tests/tests.nim" +pkg "gittyup", "nimble test", "https://github.com/disruptek/gittyup", allowFailure = true +pkg "gnuplot", "nim c gnuplot.nim" +# pkg "gram", "nim c -r --mm:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram" + # pending https://github.com/nim-lang/Nim/issues/16509 pkg "hts", "nim c -o:htss src/hts.nim" +pkg "httpauth" +pkg "httputils" pkg "illwill", "nimble examples" pkg "inim" pkg "itertools", "nim doc src/itertools.nim" pkg "iterutils" +pkg "json_rpc" +pkg "json_serialization" pkg "jstin" pkg "karax", "nim c -r tests/tester.nim" +pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree" pkg "loopfusion" -pkg "msgpack4nim" +pkg "lockfreequeues" +pkg "macroutils" +pkg "manu" +pkg "markdown" +pkg "measuremancer", "nimble testDeps; nimble -y test" +pkg "memo" +pkg "msgpack4nim", "nim c -r tests/test_spec.nim" pkg "nake", "nim c nakefile.nim" -pkg "neo", "nim c -d:blas=openblas tests/all.nim", true -# pkg "nico", "", true -pkg "nicy", "nim c src/nicy.nim" +pkg "neo", "nim c -d:blas=openblas --mm:refc tests/all.nim" +pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true, allowFailure = true + # inactive, tests not adapted to #23096 +pkg "netty" +pkg "nico", allowFailure = true +pkg "nicy", "nim c -r src/nicy.nim" pkg "nigui", "nim c -o:niguii -r src/nigui.nim" -pkg "nimcrypto", "nim c -r tests/testall.nim" -pkg "NimData", "nim c -o:nimdataa src/nimdata.nim", true -pkg "nimes", "nim c src/nimes.nim", true -pkg "nimfp", "nim c -o:nfp -r src/fp.nim", true -pkg "nimgame2", "nim c nimgame2/nimgame.nim", true -pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim", true -# pkg "nimlsp", "", true -pkg "nimly", "", true -# pkg "nimongo", "nimble test_ci", true +pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround needed, see D20210308T165435 +pkg "NimData", "nim c -o:nimdataa src/nimdata.nim" +pkg "nimes", "nim c src/nimes.nim" +pkg "nimfp", "nim c -o:nfp -r src/fp.nim" +pkg "nimgame2", "nim c --mm:refc nimgame2/nimgame.nim" +pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" +pkg "nimib" +pkg "nimlsp" +pkg "nimly", "nim c -r tests/test_readme_example.nim" +pkg "nimongo", "nimble test_ci", allowFailure = true +pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true +pkg "nimPNG", useHead = true pkg "nimpy", "nim c -r tests/nimfrompy.nim" pkg "nimquery" -pkg "nimsl", "", true +pkg "nimsl" pkg "nimsvg" -# pkg "nimterop", "", true -pkg "nimx", "nim c --threads:on test/main.nim", true -pkg "norm", "nim c -r tests/tsqlite.nim", true -pkg "npeg" -pkg "ormin", "nim c -o:orminn ormin.nim", true +pkg "nimterop", "nimble minitest", url = "https://github.com/nim-lang/nimterop" +pkg "nimwc", "nim c nimwc.nim" +pkg "nimx", "nim c test/main.nim", allowFailure = true +pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" +pkg "norm", "testament r tests/common/tmodel.nim" +pkg "normalize" +pkg "npeg", "nimble testarc" +pkg "numericalnim", "nimble nimCI" +pkg "optionsutils" +pkg "ormin", "nim c -o:orminn ormin.nim" pkg "parsetoml" pkg "patty" -pkg "plotly", "nim c --oldgensym:on examples/all.nim", true +pkg "pixie" +pkg "plotly", "nim c examples/all.nim" pkg "pnm" pkg "polypbren" -pkg "protobuf", "nim c -o:protobuff -r src/protobuf.nim", true +pkg "presto" +pkg "prologue", "nimble tcompile" +# remove fork after https://github.com/PMunch/combparser/pull/7 is merged: +pkg "protobuf", "nimble install -y https://github.com/metagn/combparser@#HEAD; nim c -o:protobuff -r src/protobuf.nim" pkg "rbtree" pkg "react", "nimble example" -pkg "regex", "nim c src/regex", true -pkg "result", "nim c -r result.nim" +pkg "regex", "nim c src/regex" +pkg "results", "nim c -r results.nim" +pkg "RollingHash", "nim c -r tests/test_cyclichash.nim" pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim" pkg "sdl1", "nim c -r src/sdl.nim" pkg "sdl2_nim", "nim c -r sdl2/sdl.nim" -pkg "snip", "", false, "https://github.com/genotrance/snip" -pkg "stint", "nim c -o:stintt -r stint.nim" -pkg "strunicode", "nim c -r src/strunicode.nim", true -pkg "telebot", "nim c -o:tbot --oldgensym:on -r telebot.nim", true +pkg "serialization" +pkg "sigv4", "nim c --mm:arc -r sigv4.nim", "https://github.com/disruptek/sigv4" +pkg "sim" +pkg "smtp", "nimble compileExample" +pkg "snip", "nimble test", "https://github.com/genotrance/snip" +pkg "ssostrings" +pkg "stew" +pkg "stint", "nim c stint.nim" +pkg "strslice" +pkg "strunicode", "nim c -r --mm:refc src/strunicode.nim" +pkg "supersnappy" +pkg "synthesis" +pkg "taskpools" +pkg "telebot", "nim c -o:tbot -r src/telebot.nim" pkg "tempdir" -pkg "tensordsl", "nim c -r tests/tests.nim", false, "https://krux02@bitbucket.org/krux02/tensordslnim.git" +pkg "templates" +pkg "tensordsl", "nim c -r --mm:refc tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" +pkg "terminaltables", "nim c src/terminaltables.nim" +pkg "termstyle", "nim c -r termstyle.nim" +pkg "testutils" +pkg "timeit" +pkg "timezones" pkg "tiny_sqlite" -pkg "unicodedb" -pkg "unicodeplus", "", true +pkg "unicodedb", "nim c -d:release -r tests/tests.nim" +pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" +pkg "union", "nim c -r tests/treadme.nim", url = "https://github.com/alaviss/union" +pkg "unittest2" pkg "unpack" -# pkg "winim", "", true +pkg "weave", "nimble install -y cligen@#HEAD; nimble test_gc_arc", useHead = true +pkg "websock" +pkg "websocket", "nim c websocket.nim" +# pkg "winim", allowFailure = true pkg "with" -pkg "ws" +pkg "ws", allowFailure = true pkg "yaml" pkg "zero_functional", "nim c -r test.nim" +pkg "zippy" +pkg "zxcvbn" diff --git a/testament/lib/readme.md b/testament/lib/readme.md new file mode 100644 index 000000000..20e866338 --- /dev/null +++ b/testament/lib/readme.md @@ -0,0 +1,4 @@ +This directory contains helper files used by several tests, to avoid +code duplication, akin to a std extension tailored for testament. + +Some of these could later migrate to stdlib. diff --git a/testament/lib/stdtest/netutils.nim b/testament/lib/stdtest/netutils.nim new file mode 100644 index 000000000..5115390e0 --- /dev/null +++ b/testament/lib/stdtest/netutils.nim @@ -0,0 +1,13 @@ +import std/[nativesockets, asyncdispatch, os] + +proc bindAvailablePort*(handle: SocketHandle, port = Port(0)): Port = + ## See also `asynchttpserver.getPort`. + block: + var name: Sockaddr_in + name.sin_family = typeof(name.sin_family)(toInt(AF_INET)) + name.sin_port = htons(uint16(port)) + name.sin_addr.s_addr = htonl(INADDR_ANY) + if bindAddr(handle, cast[ptr SockAddr](addr(name)), + sizeof(name).Socklen) < 0'i32: + raiseOSError(osLastError(), $port) + result = getLocalAddr(handle, AF_INET)[1] diff --git a/testament/lib/stdtest/specialpaths.nim b/testament/lib/stdtest/specialpaths.nim index 3c8b2338c..e214d113d 100644 --- a/testament/lib/stdtest/specialpaths.nim +++ b/testament/lib/stdtest/specialpaths.nim @@ -1,30 +1,54 @@ #[ todo: move findNimStdLibCompileTime, findNimStdLib here +xxx: factor pending https://github.com/timotheecour/Nim/issues/616 + +## note: $lib vs $nim +note: these can resolve to 3 different paths if running via `nim c --lib:lib foo`, +eg if compiler was installed via nimble (or is in nim path), and nim is external +(ie not in `$lib/../bin/` dir) + +import "$lib/../compiler/nimpaths" # <- most robust if you want to favor --lib:lib +import "$nim/compiler/nimpaths" +import compiler/nimpaths ]# import os +when defined(nimPreviewSlimSystem): + import std/assertions # Note: all the const paths defined here are known at compile time and valid # so long Nim repo isn't relocated after compilation. # This means the binaries they produce will embed hardcoded paths, which # isn't appropriate for some applications that need to be relocatable. -const sourcePath = currentSourcePath() - # robust way to derive other paths here - # We don't depend on PATH so this is robust to having multiple nim - # binaries - -const nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir - ## root of Nim repo - -const stdlibDir* = nimRootDir / "lib" - # todo: make nimeval.findNimStdLibCompileTime use this - -const systemPath* = stdlibDir / "system.nim" - -const buildDir* = nimRootDir / "build" - ## refs #10268: all testament generated files should go here to avoid - ## polluting .gitignore +const + sourcePath = currentSourcePath() + # robust way to derive other paths here + # We don't depend on PATH so this is robust to having multiple nim binaries + nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir ## root of Nim repo + testsFname* = "tests" + stdlibDir* = nimRootDir / "lib" + systemPath* = stdlibDir / "system.nim" + testsDir* = nimRootDir / testsFname + buildDir* = nimRootDir / "build" + ## refs #10268: all testament generated files should go here to avoid + ## polluting .gitignore + +proc splitTestFile*(file: string): tuple[cat: string, path: string] = + ## At least one directory is required in the path, to use as a category name + runnableExamples: + doAssert splitTestFile("tests/fakedir/tfakename.nim") == ("fakedir", "tests/fakedir/tfakename.nim".unixToNativePath) + for p in file.parentDirs(inclusive = false): + let parent = p.parentDir + if parent.lastPathPart == testsFname: + result.cat = p.lastPathPart + let dir = getCurrentDir() + if file.isRelativeTo(dir): + result.path = file.relativePath(dir) + else: + result.path = file + return result + raiseAssert "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'" static: # sanity check diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim new file mode 100644 index 000000000..a490b17c8 --- /dev/null +++ b/testament/lib/stdtest/testutils.nim @@ -0,0 +1,126 @@ +import std/private/miscdollars +when defined(nimscript): + import std/os # xxx investigate why needed +else: + from std/os import getEnv +import std/[macros, genasts] + +template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = + ## API to deal with flaky or failing tests. This avoids disabling entire tests + ## altogether so that at least the parts that are working are kept being + ## tested. This also avoids making CI fail periodically for tests known to + ## be flaky. Finally, for known failures, passing `notifySuccess = true` will + ## log that the test succeeded, which may indicate that a bug was fixed + ## "by accident" and should be looked into. + const info = instantiationInfo(-1, true) + const expr = astToStr(cond) + if cond and not notifySuccess: + discard # silent success + else: + var msg2 = "" + toLocation(msg2, info.filename, info.line, info.column) + if cond: + # a flaky test is failing, we still report it but we don't fail CI + msg2.add " FLAKY_SUCCESS " + else: + # a previously failing test is now passing, a pre-existing bug might've been + # fixed by accidend + msg2.add " FLAKY_FAILURE " + msg2.add $expr & " " & msg + echo msg2 + +when not defined(js) and not defined(nimscript): + import std/strutils + + proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool = + ## Returns true if each stripped line in `lhs` appears in rhs, using a greedy matching. + # xxx improve error reporting by showing the last matched pair + iterator splitLinesClosure(): string {.closure.} = + for line in splitLines(rhs.strip): + yield line + template isMatch(lhsi, rhsi): bool = + if allowPrefixMatch: + startsWith(rhsi, lhsi) + else: + lhsi == rhsi + + var rhsIter = splitLinesClosure + var currentLine = strip(rhsIter()) + + for line in lhs.strip.splitLines: + let line = line.strip + if line.len != 0: + while not isMatch(line, currentLine): + currentLine = strip(rhsIter()) + if rhsIter.finished: + return false + + if rhsIter.finished: + return false + return true + +template enableRemoteNetworking*: bool = + ## Allows contolling whether to run some test at a statement-level granularity. + ## Using environment variables simplifies propagating this all the way across + ## process calls, e.g. `testament all` calls itself, which in turns invokes + ## a `nim` invocation (possibly via additional intermediate processes). + getEnv("NIM_TESTAMENT_REMOTE_NETWORKING") == "1" + +template disableSSLTesting*: bool = + ## TODO: workaround for GitHub Action gcc 14 matrix; remove this + ## matrix and the flag after Azure agent supports ubuntu 24.04 + getEnv("NIM_TESTAMENT_DISABLE_SSL") == "1" + +template whenRuntimeJs*(bodyIf, bodyElse) = + ##[ + Behaves as `when defined(js) and not nimvm` (which isn't legal yet). + pending improvements to `nimvm`, this sugar helps; use as follows: + + whenRuntimeJs: + doAssert defined(js) + when nimvm: doAssert false + else: discard + do: + discard + ]## + when nimvm: bodyElse + else: + when defined(js): bodyIf + else: bodyElse + +template whenVMorJs*(bodyIf, bodyElse) = + ## Behaves as: `when defined(js) or nimvm` + when nimvm: bodyIf + else: + when defined(js): bodyIf + else: bodyElse + +template accept*(a) = + doAssert compiles(a) + +template reject*(a) = + doAssert not compiles(a) + +template disableVm*(body) = + when nimvm: discard + else: body + +macro assertAll*(body) = + ## works in VM, unlike `check`, `require` + runnableExamples: + assertAll: + 1+1 == 2 + var a = @[1, 2] # statements work + a.len == 2 + # remove this once these support VM, pending #10129 (closed but not yet fixed) + result = newStmtList() + for a in body: + result.add genAst(a, a2 = a.repr, info = lineInfo(a)) do: + # D20210421T014713:here + # xxx pending https://github.com/nim-lang/Nim/issues/12030, + # `typeof` should introduce its own scope, so that this + # is sufficient: `typeof(a)` instead of `typeof(block: a)` + when typeof(block: a) is void: a + else: + if not a: + raise newException(AssertionDefect, info & " " & a2) diff --git a/testament/lib/stdtest/unittest_light.nim b/testament/lib/stdtest/unittest_light.nim new file mode 100644 index 000000000..4ab1d7543 --- /dev/null +++ b/testament/lib/stdtest/unittest_light.nim @@ -0,0 +1,37 @@ +import std/assertions + + +proc mismatch*[T](lhs: T, rhs: T): string = + ## Simplified version of `unittest.require` that satisfies a common use case, + ## while avoiding pulling too many dependencies. On failure, diagnostic + ## information is provided that in particular makes it easy to spot + ## whitespace mismatches and where the mismatch is. + proc replaceInvisible(s: string): string = + for a in s: + case a + of '\n': result.add "\\n\n" + else: result.add a + + proc quoted(s: string): string = result.addQuoted s + + result.add '\n' + result.add "lhs:{" & replaceInvisible( + $lhs) & "}\nrhs:{" & replaceInvisible($rhs) & "}\n" + when compiles(lhs.len): + if lhs.len != rhs.len: + result.add "lhs.len: " & $lhs.len & " rhs.len: " & $rhs.len & '\n' + when compiles(lhs[0]): + var i = 0 + while i < lhs.len and i < rhs.len: + if lhs[i] != rhs[i]: break + i.inc + result.add "first mismatch index: " & $i & '\n' + if i < lhs.len and i < rhs.len: + result.add "lhs[i]: {" & quoted($lhs[i]) & "}\nrhs[i]: {" & quoted( + $rhs[i]) & "}\n" + result.add "lhs[0..<i]:{" & replaceInvisible($lhs[ + 0..<i]) & '}' + +proc assertEquals*[T](lhs: T, rhs: T, msg = "") = + if lhs != rhs: + doAssert false, mismatch(lhs, rhs) & '\n' & msg diff --git a/testament/specs.nim b/testament/specs.nim index 70be6990c..c3040c1d8 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -7,7 +7,17 @@ # distribution, for details about the copyright. # -import sequtils, parseutils, strutils, os, streams, parsecfg +import sequtils, parseutils, strutils, os, streams, parsecfg, + tables, hashes, sets +import compiler/platform + +type TestamentData* = ref object + # better to group globals under 1 object; could group the other ones here too + batchArg*: string + testamentNumBatch*: int + testamentBatch*: int + +let testamentData0* = TestamentData() var compilerPrefix* = findExe("nim") @@ -25,60 +35,77 @@ type TOutputCheck* = enum ocIgnore = "ignore" - ocEqual = "equal" + ocEqual = "equal" ocSubstr = "substr" TResultEnum* = enum - reNimcCrash, # nim compiler seems to have crashed - reMsgsDiffer, # error messages differ - reFilesDiffer, # expected and given filenames differ - reLinesDiffer, # expected and given line numbers differ + reNimcCrash, # nim compiler seems to have crashed + reMsgsDiffer, # error messages differ + reFilesDiffer, # expected and given filenames differ + reLinesDiffer, # expected and given line numbers differ reOutputsDiffer, - reExitcodesDiffer, + reExitcodesDiffer, # exit codes of program or of valgrind differ reTimeout, reInvalidPeg, reCodegenFailure, reCodeNotFound, reExeNotFound, - reInstallFailed # package installation failed - reBuildFailed # package building failed - reDisabled, # test is disabled - reJoined, # test is disabled because it was joined into the megatest - reSuccess # test was successful - reInvalidSpec # test had problems to parse the spec + reInstallFailed # package installation failed + reBuildFailed # package building failed + reDisabled, # test is disabled + reJoined, # test is disabled because it was joined into the megatest + reSuccess # test was successful + reInvalidSpec # test had problems to parse the spec TTarget* = enum - targetC = "C" - targetCpp = "C++" - targetObjC = "ObjC" - targetJS = "JS" + targetC = "c" + targetCpp = "cpp" + targetObjC = "objc" + targetJS = "js" + + InlineError* = object + kind*: string + msg*: string + line*, col*: int + + ValgrindSpec* = enum + disabled, enabled, leaking TSpec* = object + # xxx make sure `isJoinableSpec` takes into account each field here. action*: TTestAction file*, cmd*: string + filename*: string ## Test filename (without path). input*: string outputCheck*: TOutputCheck sortoutput*: bool output*: string line*, column*: int - tfile*: string - tline*, tcolumn*: int exitCode*: int msg*: string - ccodeCheck*: string + ccodeCheck*: seq[string] maxCodeSize*: int err*: TResultEnum + inCurrentBatch*: bool targets*: set[TTarget] + matrix*: seq[string] nimout*: string - parseErrors*: string # when the spec definition is invalid, this is not empty. + nimoutFull*: bool # whether nimout is all compiler output or a subset + parseErrors*: string # when the spec definition is invalid, this is not empty. unjoinable*: bool - useValgrind*: bool + unbatchable*: bool + # whether this test can be batchable via `NIM_TESTAMENT_BATCH`; only very + # few tests are not batchable; the ones that are not could be turned batchable + # by making the dependencies explicit + useValgrind*: ValgrindSpec timeout*: float # in seconds, fractions possible, - # but don't rely on much precision + # but don't rely on much precision + inlineErrors*: seq[InlineError] # line information to error message + debugInfo*: string # debug info to give more context proc getCmd*(s: TSpec): string = if s.cmd.len == 0: - result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file" + result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:build/deps/pkgs2 $options $file" else: result = s.cmd @@ -86,6 +113,13 @@ const targetToExt*: array[TTarget, string] = ["nim.c", "nim.cpp", "nim.m", "js"] targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"] +proc defaultOptions*(a: TTarget): string = + case a + of targetJS: "-d:nodejs" + # once we start testing for `nim js -d:nimbrowser` (eg selenium or similar), + # we can adapt this logic; or a given js test can override with `-u:nodejs`. + else: "" + when not declared(parseCfgBool): # candidate for the stdlib: proc parseCfgBool(s: string): bool = @@ -94,21 +128,174 @@ when not declared(parseCfgBool): of "n", "no", "false", "0", "off": result = false else: raise newException(ValueError, "cannot interpret as a bool: " & s) -proc extractSpec(filename: string): string = - const tripleQuote = "\"\"\"" - var x = readFile(filename).string - var a = x.find(tripleQuote) - var b = x.find(tripleQuote, a+3) - # look for """ only in the first section - if a >= 0 and b > a and a < 40: - result = x.substr(a+3, b-1).replace("'''", tripleQuote) +proc addLine*(self: var string; pieces: varargs[string]) = + for piece in pieces: + self.add piece + self.add "\n" + + +const + inlineErrorKindMarker = "tt." + inlineErrorMarker = "#[" & inlineErrorKindMarker + +proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int = + ## Extract inline error messages. + ## + ## Can parse a single message for a line: + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name] ]# + ## ``` + ## + ## Can parse multiple messages for a line when they are separated by ';': + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error + ## ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error + ## ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name]; + ## tt.Error ^ 'no_destroy' should be: 'nodestroy' [Name]; + ## tt.Error ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` + result = i + len(inlineErrorMarker) + inc col, len(inlineErrorMarker) + let msgLine = line + var msgCol = -1 + var msg = "" + var kind = "" + + template parseKind = + while result < s.len and s[result] in IdentChars: + kind.add s[result] + inc result + inc col + if kind notin ["Hint", "Warning", "Error"]: + spec.parseErrors.addLine "expected inline message kind: Hint, Warning, Error" + + template skipWhitespace = + while result < s.len and s[result] in Whitespace: + if s[result] == '\n': + col = 1 + inc line + else: + inc col + inc result + + template parseCaret = + if result < s.len and s[result] == '^': + msgCol = col + inc result + inc col + skipWhitespace() + else: + spec.parseErrors.addLine "expected column marker ('^') for inline message" + + template isMsgDelimiter: bool = + s[result] == ';' and + (block: + let nextTokenIdx = result + 1 + parseutils.skipWhitespace(s, result + 1) + if s.len > nextTokenIdx + len(inlineErrorKindMarker) and + s[nextTokenIdx..(nextTokenIdx + len(inlineErrorKindMarker) - 1)] == inlineErrorKindMarker: + true + else: + false) + + template trimTrailingMsgWhitespace = + while msg.len > 0 and msg[^1] in Whitespace: + setLen msg, msg.len - 1 + + template addInlineError = + doAssert msg[^1] notin Whitespace + if kind == "Error": spec.action = actionReject + spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: msgLine, col: msgCol) + + parseKind() + skipWhitespace() + parseCaret() + + while result < s.len-1: + if s[result] == '\n': + if result > 0 and s[result - 1] == '\r': + msg[^1] = '\n' + else: + msg.add '\n' + inc result + inc line + col = 1 + elif isMsgDelimiter(): + trimTrailingMsgWhitespace() + inc result + skipWhitespace() + addInlineError() + inc result, len(inlineErrorKindMarker) + inc col, 1 + len(inlineErrorKindMarker) + kind.setLen 0 + msg.setLen 0 + parseKind() + skipWhitespace() + parseCaret() + elif s[result] == ']' and s[result+1] == '#': + trimTrailingMsgWhitespace() + inc result, 2 + inc col, 2 + addInlineError() + break + else: + msg.add s[result] + inc result + inc col + + if spec.inlineErrors.len > 0: + spec.unjoinable = true + +proc extractSpec(filename: string; spec: var TSpec): string = + const + tripleQuote = "\"\"\"" + specStart = "discard " & tripleQuote + var s = readFile(filename) + + var i = 0 + var a = -1 + var b = -1 + var line = 1 + var col = 1 + while i < s.len: + if (i == 0 or s[i-1] != ' ') and s.continuesWith(specStart, i): + # `s[i-1] == '\n'` would not work because of `tests/stdlib/tbase64.nim` which contains BOM (https://en.wikipedia.org/wiki/Byte_order_mark) + const lineMax = 10 + if a != -1: + raise newException(ValueError, "testament spec violation: duplicate `specStart` found: " & $(filename, a, b, line)) + elif line > lineMax: + # not overly restrictive, but prevents mistaking some `specStart` as spec if deeep inside a test file + raise newException(ValueError, "testament spec violation: `specStart` should be before line $1, or be indented; info: $2" % [$lineMax, $(filename, a, b, line)]) + i += specStart.len + a = i + elif a > -1 and b == -1 and s.continuesWith(tripleQuote, i): + b = i + i += tripleQuote.len + elif s[i] == '\n': + inc line + inc i + col = 1 + elif s.continuesWith(inlineErrorMarker, i): + i = extractErrorMsg(s, i, line, col, spec) + else: + inc col + inc i + + if a >= 0 and b > a: + result = s.substr(a, b-1).multiReplace({"'''": tripleQuote, "\\31": "\31"}) + elif a >= 0: + raise newException(ValueError, "testament spec violation: `specStart` found but not trailing `tripleQuote`: $1" % $(filename, a, b, line)) else: - #echo "warning: file does not contain spec: " & filename result = "" -when not defined(nimhygiene): - {.pragma: inject.} - proc parseTargets*(value: string): set[TTarget] = for v in value.normalize.splitWhitespace: case v @@ -116,31 +303,38 @@ proc parseTargets*(value: string): set[TTarget] = of "cpp", "c++": result.incl(targetCpp) of "objc": result.incl(targetObjC) of "js": result.incl(targetJS) - else: echo "target ignored: " & v - -proc addLine*(self: var string; a: string) = - self.add a - self.add "\n" - -proc addLine*(self: var string; a,b: string) = - self.add a - self.add b - self.add "\n" + else: raise newException(ValueError, "invalid target: '$#'" % v) proc initSpec*(filename: string): TSpec = result.file = filename +proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool = + if testamentData.testamentNumBatch != 0: + hash(filename) mod testamentData.testamentNumBatch == testamentData.testamentBatch + else: + true + proc parseSpec*(filename: string): TSpec = result.file = filename - let specStr = extractSpec(filename) + result.filename = extractFilename(filename) + let specStr = extractSpec(filename, result) var ss = newStringStream(specStr) var p: CfgParser open(p, ss, filename, 1) + var flags: HashSet[string] + var nimoutFound = false while true: var e = next(p) case e.kind of cfgKeyValuePair: - case normalize(e.key) + let key = e.key.normalize + const whiteListMulti = ["disabled", "ccodecheck"] + ## list of flags that are correctly handled when passed multiple times + ## (instead of being overwritten) + if key notin whiteListMulti: + doAssert key notin flags, $(key, filename) + flags.incl key + case key of "action": case e.value.normalize of "compile": @@ -163,16 +357,10 @@ proc parseSpec*(filename: string): TSpec = if result.msg.len == 0 and result.nimout.len == 0: result.parseErrors.addLine "errormsg or msg needs to be specified before column" discard parseInt(e.value, result.column) - of "tfile": - result.tfile = e.value - of "tline": - discard parseInt(e.value, result.tline) - of "tcolumn": - discard parseInt(e.value, result.tcolumn) of "output": if result.outputCheck != ocSubstr: result.outputCheck = ocEqual - result.output = strip(e.value) + result.output = e.value of "input": result.input = e.value of "outputsub": @@ -180,71 +368,108 @@ proc parseSpec*(filename: string): TSpec = result.output = strip(e.value) of "sortoutput": try: - result.sortoutput = parseCfgBool(e.value) + result.sortoutput = parseCfgBool(e.value) except: result.parseErrors.addLine getCurrentExceptionMsg() of "exitcode": discard parseInt(e.value, result.exitCode) result.action = actionRun - of "msg": - result.msg = e.value - if result.action != actionRun: - result.action = actionCompile - of "errormsg", "errmsg": + of "errormsg": result.msg = e.value result.action = actionReject of "nimout": result.nimout = e.value + nimoutFound = true + of "nimoutfull": + result.nimoutFull = parseCfgBool(e.value) + of "batchable": + result.unbatchable = not parseCfgBool(e.value) of "joinable": result.unjoinable = not parseCfgBool(e.value) of "valgrind": when defined(linux) and sizeof(int) == 8: - result.useValgrind = parseCfgBool(e.value) + result.useValgrind = if e.value.normalize == "leaks": leaking + else: ValgrindSpec(parseCfgBool(e.value)) result.unjoinable = true - if result.useValgrind: + if result.useValgrind != disabled: result.outputCheck = ocSubstr else: # Windows lacks valgrind. Silly OS. # Valgrind only supports OSX <= 17.x - result.useValgrind = false + result.useValgrind = disabled of "disabled": - case e.value.normalize + let value = e.value.normalize + case value of "y", "yes", "true", "1", "on": result.err = reDisabled of "n", "no", "false", "0", "off": discard - of "win", "windows": + # These values are defined in `compiler/options.isDefined` + of "win": when defined(windows): result.err = reDisabled of "linux": when defined(linux): result.err = reDisabled of "bsd": when defined(bsd): result.err = reDisabled - of "macosx": - when defined(macosx): result.err = reDisabled - of "unix": - when defined(unix): result.err = reDisabled - of "posix": + of "osx": + when defined(osx): result.err = reDisabled + of "unix", "posix": when defined(posix): result.err = reDisabled - of "travis": + of "freebsd": + when defined(freebsd): result.err = reDisabled + of "littleendian": + when defined(littleendian): result.err = reDisabled + of "bigendian": + when defined(bigendian): result.err = reDisabled + of "cpu8", "8bit": + when defined(cpu8): result.err = reDisabled + of "cpu16", "16bit": + when defined(cpu16): result.err = reDisabled + of "cpu32", "32bit": + when defined(cpu32): result.err = reDisabled + of "cpu64", "64bit": + when defined(cpu64): result.err = reDisabled + # These values are for CI environments + of "travis": # deprecated if isTravis: result.err = reDisabled - of "appveyor": + of "appveyor": # deprecated if isAppVeyor: result.err = reDisabled of "azure": if isAzure: result.err = reDisabled - of "32bit": - if sizeof(int) == 4: - result.err = reDisabled - of "freebsd": - when defined(freebsd): result.err = reDisabled - of "arm64": - when defined(arm64): result.err = reDisabled else: - result.parseErrors.addLine "cannot interpret as a bool: ", e.value + # Check whether the value exists as an OS or CPU that is + # defined in `compiler/platform`. + block checkHost: + for os in platform.OS: + # Check if the value exists as OS. + if value == os.name.normalize: + # The value exists; is it the same as the current host? + if value == hostOS.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + for cpu in platform.CPU: + # Check if the value exists as CPU. + if value == cpu.name.normalize: + # The value exists; is it the same as the current host? + if value == hostCPU.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + # The value doesn't exist as an OS, CPU, or any previous value + # defined in this case statement, so raise an error. + result.parseErrors.addLine "cannot interpret as a bool: ", e.value of "cmd": if e.value.startsWith("nim "): result.cmd = compilerPrefix & e.value[3..^1] else: result.cmd = e.value of "ccodecheck": - result.ccodeCheck = e.value + result.ccodeCheck.add e.value of "maxcodesize": discard parseInt(e.value, result.maxCodeSize) of "timeout": @@ -252,19 +477,14 @@ proc parseSpec*(filename: string): TSpec = result.timeout = parseFloat(e.value) except ValueError: result.parseErrors.addLine "cannot interpret as a float: ", e.value - of "target", "targets": - for v in e.value.normalize.splitWhitespace: - case v - of "c": - result.targets.incl(targetC) - of "cpp", "c++": - result.targets.incl(targetCpp) - of "objc": - result.targets.incl(targetObjC) - of "js": - result.targets.incl(targetJS) - else: - result.parseErrors.addLine "cannot interpret as a target: ", e.value + of "targets", "target": + try: + result.targets.incl parseTargets(e.value) + except ValueError as e: + result.parseErrors.addLine e.msg + of "matrix": + for v in e.value.split(';'): + result.matrix.add(v.strip) else: result.parseErrors.addLine "invalid key for test spec: ", e.key @@ -280,3 +500,22 @@ proc parseSpec*(filename: string): TSpec = if skips.anyIt(it in result.file): result.err = reDisabled + + if nimoutFound and result.nimout.len == 0 and not result.nimoutFull: + result.parseErrors.addLine "empty `nimout` is vacuously true, use `nimoutFull:true` if intentional" + + result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable + if not result.inCurrentBatch: + result.err = reDisabled + + # Interpolate variables in msgs: + template varSub(msg: string): string = + try: + msg % ["/", $DirSep, "file", result.filename] + except ValueError: + result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unit-tests-output-message-variable-interpolation')" + msg + result.nimout = result.nimout.varSub + result.msg = result.msg.varSub + for inlineError in result.inlineErrors.mitems: + inlineError.msg = inlineError.msg.varSub diff --git a/testament/testament.nim b/testament/testament.nim index b19f13256..1e892e636 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -10,87 +10,114 @@ ## This program verifies Nim against the testcases. import - strutils, pegs, os, osproc, streams, json, - backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, times, md5, sequtils, azure + std/[strutils, pegs, os, osproc, streams, json, + parseopt, browsers, terminal, exitprocs, + algorithm, times, intsets, macros] -include compiler/nodejs +import backend, specs, azure, htmlgen + +from std/sugar import dup +import compiler/nodejs +import lib/stdtest/testutils +from lib/stdtest/specialpaths import splitTestFile +from std/private/gitutils import diffStrings + +import ../dist/checksums/src/checksums/md5 + +proc trimUnitSep(x: var string) = + let L = x.len + if L > 0 and x[^1] == '\31': + setLen x, L-1 var useColors = true var backendLogging = true var simulate = false +var optVerbose = false +var useMegatest = true +var valgrindEnabled = true + +proc verboseCmd(cmd: string) = + if optVerbose: + echo "executing: ", cmd const + failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs testsDir = "tests" & DirSep resultsFile = "testresults.html" - #jsonFile = "testresults.json" # not used Usage = """Usage: testament [options] command [arguments] Command: p|pat|pattern <glob> run all the tests matching the given pattern - all run all tests + all run all tests in category folders c|cat|category <category> run all the tests of a certain category r|run <test> run single test file html generate $1 from the database - stats generate statistics about test cases Arguments: arguments are passed to the compiler Options: - --print also print results to the console + --print print results to the console + --verbose print commands (compiling and running tests) --simulate see what tests would be run but don't run them (for debugging) --failing only show failing/ignored tests - --targets:"c c++ js objc" run tests for specified targets (default: all) + --targets:"c cpp js objc" run tests for specified targets (default: c) --nim:path use a particular nim executable (default: $$PATH/nim) --directory:dir Change to directory dir before reading the tests or doing anything else. - --colors:on|off Turn messagescoloring on|off. + --colors:on|off Turn messages coloring on|off. --backendLogging:on|off Disable or enable backend logging. By default turned on. --megatest:on|off Enable or disable megatest. Default is on. + --valgrind:on|off Enable or disable valgrind support. Default is on. --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN. + +Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables +tests with remote networking (as in CI). """ % resultsFile +proc isNimRepoTests(): bool = + # this logic could either be specific to cwd, or to some file derived from + # the input file, eg testament r /pathto/tests/foo/tmain.nim; we choose + # the former since it's simpler and also works with `testament all`. + let file = "testament"/"testament.nim.cfg" + result = file.fileExists + type Category = distinct string TResults = object - total, passed, skipped: int + total, passed, failedButAllowed, skipped: int + ## xxx rename passed to passedOrAllowedFailure data: string - TTest = object name: string cat: Category options: string - args: seq[string] + testArgs: seq[string] spec: TSpec startTime: float + debugInfo: string # ---------------------------------------------------------------------------- let pegLineError = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" - pegLineTemplate = - peg""" - {[^(]*} '(' {\d+} ', ' {\d+} ') ' - 'template/generic instantiation' ( ' of `' [^`]+ '`' )? ' from here' .* - """ pegOtherError = peg"'Error:' \s* {.*}" pegOfInterest = pegLineError / pegOtherError var gTargets = {low(TTarget)..high(TTarget)} +var targetsSet = false proc isSuccess(input: string): bool = # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs + # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we + # could handle colors, there would be other issues such as handling other flags + # that may appear in user config (eg: `--filenames`). + # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config + # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory input.startsWith("Hint: ") and input.endsWith("[SuccessX]") -proc normalizeMsg(s: string): string = - result = newStringOfCap(s.len+1) - for x in splitLines(s): - if result.len > 0: result.add '\L' - result.add x.strip - proc getFileDir(filename: string): string = result = filename.splitFile().dir if not result.isAbsolute(): @@ -98,7 +125,7 @@ proc getFileDir(filename: string): string = proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: string = ""): tuple[ cmdLine: string, - output: TaintedString, + output: string, exitCode: int] {.tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = @@ -106,8 +133,9 @@ proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: str for arg in args: result.cmdLine.add ' ' result.cmdLine.add quoteShell(arg) - var p = startProcess(command, workingDir=workingDir, args=args, - options={poStdErrToStdOut, poUsePath}) + verboseCmd(result.cmdLine) + var p = startProcess(command, workingDir = workingDir, args = args, + options = {poStdErrToStdOut, poUsePath}) var outp = outputStream(p) # There is no way to provide input for the child process @@ -117,12 +145,12 @@ proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: str instream.write(input) close instream - result.exitCode = -1 - var line = newStringOfCap(120).TaintedString + result.exitCode = -1 + var line = newStringOfCap(120) while true: if outp.readLine(line): - result.output.string.add(line.string) - result.output.string.add("\n") + result.output.add line + result.output.add '\n' else: result.exitCode = peekExitCode(p) if result.exitCode != -1: break @@ -133,54 +161,64 @@ proc nimcacheDir(filename, options: string, target: TTarget): string = let hashInput = options & $target result = "nimcache" / (filename & '_' & hashInput.getMD5) -proc prepareTestArgs(cmdTemplate, filename, options, nimcache: string, - target: TTarget, extraOptions=""): seq[string] = - let options = options & " " & quoteShell("--nimCache:" & nimcache) & extraOptions - result = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], +proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): string = + var options = target.defaultOptions & ' ' & options + if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell) + options.add ' ' & extraOptions + # we avoid using `parseCmdLine` which is buggy, refs bug #14343 + result = cmdTemplate % ["target", targetToCmd[target], "options", options, "file", filename.quoteShell, - "filedir", filename.getFileDir()]) + "filedir", filename.getFileDir(), "nim", compilerPrefix] -proc callCompiler(cmdTemplate, filename, options, nimcache: string, - target: TTarget, - extraOptions=""): TSpec = - let c = prepareTestArgs(cmdTemplate, filename, options, nimcache, target, +proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): TSpec = + result.cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache, target, extraOptions) - result.cmd = quoteShellCommand(c) - var p = startProcess(command=c[0], args=c[1 .. ^1], - options={poStdErrToStdOut, poUsePath}) + verboseCmd(result.cmd) + var p = startProcess(command = result.cmd, + options = {poStdErrToStdOut, poUsePath, poEvalCommand}) let outp = p.outputStream - var suc = "" + var foundSuccessMsg = false + var foundErrorMsg = false var err = "" - var tmpl = "" var x = newStringOfCap(120) result.nimout = "" while true: - if outp.readLine(x.TaintedString): - result.nimout.add(x & "\n") + if outp.readLine(x): + trimUnitSep x + result.nimout.add(x & '\n') if x =~ pegOfInterest: - # `err` should contain the last error/warning message + # `err` should contain the last error message err = x - elif x =~ pegLineTemplate and err == "": - # `tmpl` contains the last template expansion before the error - tmpl = x + foundErrorMsg = true elif x.isSuccess: - suc = x + foundSuccessMsg = true elif not running(p): break - close(p) result.msg = "" result.file = "" result.output = "" result.line = 0 result.column = 0 - result.tfile = "" - result.tline = 0 - result.tcolumn = 0 + result.err = reNimcCrash - if tmpl =~ pegLineTemplate: - result.tfile = extractFilename(matches[0]) - result.tline = parseInt(matches[1]) - result.tcolumn = parseInt(matches[2]) + result.exitCode = p.peekExitCode + close p + case result.exitCode + of 0: + if foundErrorMsg: + result.debugInfo.add " compiler exit code was 0 but some Error's were found." + else: + result.err = reSuccess + of 1: + if not foundErrorMsg: + result.debugInfo.add " compiler exit code was 1 but no Error's were found." + if foundSuccessMsg: + result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true." + else: + result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode + if err =~ pegLineError: result.file = extractFilename(matches[0]) result.line = parseInt(matches[1]) @@ -188,40 +226,15 @@ proc callCompiler(cmdTemplate, filename, options, nimcache: string, result.msg = matches[3] elif err =~ pegOtherError: result.msg = matches[0] - elif suc.isSuccess: - result.err = reSuccess - -proc callCCompiler(cmdTemplate, filename, options: string, - target: TTarget): TSpec = - let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], - "options", options, "file", filename.quoteShell, - "filedir", filename.getFileDir()]) - var p = startProcess(command="gcc", args=c[5 .. ^1], - options={poStdErrToStdOut, poUsePath}) - let outp = p.outputStream - var x = newStringOfCap(120) - result.nimout = "" - result.msg = "" - result.file = "" - result.output = "" - result.line = -1 - while true: - if outp.readLine(x.TaintedString): - result.nimout.add(x & "\n") - elif not running(p): - break - close(p) - if p.peekExitCode == 0: - result.err = reSuccess + trimUnitSep result.msg proc initResults: TResults = result.total = 0 result.passed = 0 + result.failedButAllowed = 0 result.skipped = 0 result.data = "" -import macros - macro ignoreStyleEcho(args: varargs[typed]): untyped = let typForegroundColor = bindSym"ForegroundColor".getType let typBackgroundColor = bindSym"BackgroundColor".getType @@ -246,22 +259,34 @@ template maybeStyledEcho(args: varargs[untyped]): untyped = proc `$`(x: TResults): string = - result = ("Tests passed: $1 / $3 <br />\n" & - "Tests skipped: $2 / $3 <br />\n") % - [$x.passed, $x.skipped, $x.total] + result = """ +Tests passed or allowed to fail: $2 / $1 <br /> +Tests failed and allowed to fail: $3 / $1 <br /> +Tests skipped: $4 / $1 <br /> +""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] + +proc testName(test: TTest, target: TTarget, extraOptions: string, allowFailure: bool): string = + var name = test.name.replace(DirSep, '/') + name.add ' ' & $target + if allowFailure: + name.add " (allowed to fail) " + if test.options.len > 0: name.add ' ' & test.options + if extraOptions.len > 0: name.add ' ' & extraOptions + name.strip() proc addResult(r: var TResults, test: TTest, target: TTarget, - expected, given: string, successOrig: TResultEnum) = + extraOptions, expected, given: string, successOrig: TResultEnum, + allowFailure = false, givenSpec: ptr TSpec = nil) = + # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need + # instead of having to pass individual fields, or abusing existing ones like expected vs given. # test.name is easier to find than test.name.extractFilename # A bit hacky but simple and works with tests/testament/tshould_not_work.nim - var name = test.name.replace(DirSep, '/') - name.add " " & $target & test.options - + let name = testName(test, target, extraOptions, allowFailure) let duration = epochTime() - test.startTime let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout else: successOrig - let durationStr = duration.formatFloat(ffDecimal, precision = 8).align(11) + let durationStr = duration.formatFloat(ffDecimal, precision = 2).align(5) if backendLogging: backend.writeTestResult(name = name, category = test.cat.string, @@ -271,16 +296,22 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, expected = expected, given = given) r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) + template dispNonSkipped(color, outcome) = + maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)" + template disp(msg) = + maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name if success == reSuccess: - maybeStyledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)" + dispNonSkipped(fgGreen, "PASS: ") elif success == reDisabled: - maybeStyledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name - elif success == reJoined: - maybeStyledEcho styleDim, fgYellow, "JOINED: ", styleBright, fgCyan, name + if test.spec.inCurrentBatch: disp("SKIP:") + else: disp("NOTINBATCH:") + elif success == reJoined: disp("JOINED:") else: - maybeStyledEcho styleBright, fgRed, "FAIL: ", fgCyan, name + dispNonSkipped(fgRed, failString) maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" maybeStyledEcho styleBright, fgRed, "Failure: ", $success + if givenSpec != nil and givenSpec.debugInfo.len > 0: + echo "debugInfo: " & givenSpec.debugInfo if success in {reBuildFailed, reNimcCrash, reInstallFailed}: # expected is empty, no reason to print it. echo given @@ -289,7 +320,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, maybeStyledEcho styleBright, expected, "\n" maybeStyledEcho fgYellow, "Gotten:" maybeStyledEcho styleBright, given, "\n" - + echo diffStrings(expected, given).output if backendLogging and (isAppVeyor or isAzure): let (outcome, msg) = @@ -299,44 +330,66 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, of reDisabled, reJoined: ("Skipped", "") of reBuildFailed, reNimcCrash, reInstallFailed: - ("Failed", "Failure: " & $success & "\n" & given) + ("Failed", "Failure: " & $success & '\n' & given) else: ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given) if isAzure: azure.addTestResult(name, test.cat.string, int(duration * 1000), msg, success) else: - var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options, + var p = startProcess("appveyor", args = ["AddTest", test.name.replace("\\", "/") & test.options, "-Framework", "nim-testament", "-FileName", test.cat.string, "-Outcome", outcome, "-ErrorMessage", msg, - "-Duration", $(duration*1000).int], - options={poStdErrToStdOut, poUsePath, poParentStreams}) + "-Duration", $(duration * 1000).int], + options = {poStdErrToStdOut, poUsePath, poParentStreams}) discard waitForExit(p) close(p) -proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = - if strip(expected.msg) notin strip(given.msg): - r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer) - elif expected.nimout.len > 0 and expected.nimout.normalizeMsg notin given.nimout.normalizeMsg: - r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer) - elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and +proc toString(inlineError: InlineError, filename: string): string = + result.add "$file($line, $col) $kind: $msg" % [ + "file", filename, + "line", $inlineError.line, + "col", $inlineError.col, + "kind", $inlineError.kind, + "msg", $inlineError.msg + ] + +proc inlineErrorsMsgs(expected: TSpec): string = + for inlineError in expected.inlineErrors.items: + result.addLine inlineError.toString(expected.filename) + +proc checkForInlineErrors(expected, given: TSpec): bool = + for inlineError in expected.inlineErrors: + if inlineError.toString(expected.filename) notin given.nimout: + return false + true + +proc nimoutCheck(expected, given: TSpec): bool = + result = true + if expected.nimoutFull: + if expected.nimout != given.nimout: + result = false + elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): + result = false + +proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, + target: TTarget, extraOptions: string) = + if not checkForInlineErrors(expected, given) or + (not expected.nimoutFull and not nimoutCheck(expected, given)): + r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer) + elif strip(expected.msg) notin strip(given.msg): + r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer) + elif not nimoutCheck(expected, given): + r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer) + elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: - r.addResult(test, target, expected.file, given.file, reFilesDiffer) + r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer) elif expected.line != given.line and expected.line != 0 or expected.column != given.column and expected.column != 0: - r.addResult(test, target, $expected.line & ':' & $expected.column, - $given.line & ':' & $given.column, - reLinesDiffer) - elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and - "internal error:" notin expected.msg: - r.addResult(test, target, expected.tfile, given.tfile, reFilesDiffer) - elif expected.tline != given.tline and expected.tline != 0 or - expected.tcolumn != given.tcolumn and expected.tcolumn != 0: - r.addResult(test, target, $expected.tline & ':' & $expected.tcolumn, - $given.tline & ':' & $given.tcolumn, - reLinesDiffer) + r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column, + $given.line & ':' & $given.column, reLinesDiffer) else: - r.addResult(test, target, expected.msg, given.msg, reSuccess) + r.addResult(test, target, extraOptions, expected.msg, given.msg, reSuccess) inc(r.passed) proc generatedFile(test: TTest, target: TTarget): string = @@ -354,10 +407,9 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st given: var TSpec) = try: let genFile = generatedFile(test, target) - let contents = readFile(genFile).string - let check = spec.ccodeCheck - if check.len > 0: - if check[0] == '\\': + let contents = readFile(genFile) + for check in spec.ccodeCheck: + if check.len > 0 and check[0] == '\\': # little hack to get 'match' support: if not contents.match(check.peg): given.err = reCodegenFailure @@ -375,88 +427,78 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st given.err = reCodeNotFound echo getCurrentExceptionMsg() -proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) = - let giv = given.nimout.strip - var currentPos = 0 - # Only check that nimout contains all expected lines in that order. - # There may be more output in nimout. It is ignored here. - for line in expectedNimout.strip.splitLines: - currentPos = giv.find(line.strip, currentPos) - if currentPos < 0: - given.err = reMsgsDiffer - break - -proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, - expected: TSpec; r: var TResults) = +proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string, + given: var TSpec, expected: TSpec; r: var TResults) = var expectedmsg: string = "" var givenmsg: string = "" if given.err == reSuccess: if expected.needsCodegenCheck: codegenCheck(test, target, expected, expectedmsg, given) givenmsg = given.msg - if expected.nimout.len > 0: - expectedmsg = expected.nimout + if not nimoutCheck(expected, given) or + not checkForInlineErrors(expected, given): + given.err = reMsgsDiffer + expectedmsg = expected.nimout & inlineErrorsMsgs(expected) givenmsg = given.nimout.strip - nimoutCheck(test, expectedmsg, given) else: - givenmsg = "$ " & given.cmd & "\n" & given.nimout + givenmsg = "$ " & given.cmd & '\n' & given.nimout if given.err == reSuccess: inc(r.passed) - r.addResult(test, target, expectedmsg, givenmsg, given.err) + r.addResult(test, target, extraOptions, expectedmsg, givenmsg, given.err) proc getTestSpecTarget(): TTarget = - if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true": + if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": result = targetCpp else: result = targetC -proc checkDisabled(r: var TResults, test: TTest): bool = - if test.spec.err in {reDisabled, reJoined}: - # targetC is a lie, but parameter is required - r.addResult(test, targetC, "", "", test.spec.err) - inc(r.skipped) - inc(r.total) - result = false - else: - result = true - var count = 0 -proc testSpecHelper(r: var TResults, test: TTest, expected: TSpec, target: TTarget, nimcache: string) = +proc equalModuloLastNewline(a, b: string): bool = + # allow lazy output spec that omits last newline, but really those should be fixed instead + result = a == b or b.endsWith("\n") and a == b[0 ..< ^1] + +proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, + target: TTarget, extraOptions: string, nimcache: string) = + test.startTime = epochTime() + if testName(test, target, extraOptions, false) in skips: + test.spec.err = reDisabled + + if test.spec.err in {reDisabled, reJoined}: + r.addResult(test, target, extraOptions, "", "", test.spec.err) + inc(r.skipped) + return + var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions) case expected.action of actionCompile: - var given = callCompiler(expected.getCmd, test.name, test.options, nimcache, target, - extraOptions = " --stdout --hint[Path]:off --hint[Processing]:off") - compilerOutputTests(test, target, given, expected, r) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionRun: - var given = callCompiler(expected.getCmd, test.name, test.options, nimcache, target) if given.err != reSuccess: - r.addResult(test, target, "", "$ " & given.cmd & "\n" & given.nimout, given.err) + r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr) else: let isJsTarget = target == targetJS var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt) - if not existsFile(exeFile): - r.addResult(test, target, expected.output, + if not fileExists(exeFile): + r.addResult(test, target, extraOptions, expected.output, "executable not found: " & exeFile, reExeNotFound) else: let nodejs = if isJsTarget: findNodeJs() else: "" if isJsTarget and nodejs == "": - r.addResult(test, target, expected.output, "nodejs binary not in PATH", + r.addResult(test, target, extraOptions, expected.output, "nodejs binary not in PATH", reExeNotFound) else: var exeCmd: string - var args = test.args + var args = test.testArgs if isJsTarget: exeCmd = nodejs - args = concat(@[exeFile], args) + # see D20210217T215950 + args = @["--unhandled-rejections=strict", exeFile] & args else: - if defined(posix) and not exeFile.contains('/'): - # "security" in Posix is actually just a euphemism - # for "unproductive arbitrary shit" - exeCmd = "./" & exeFile - else: - exeCmd = exeFile - if expected.useValgrind: - args = @["--error-exitcode=1"] & exeCmd & args + exeCmd = exeFile.dup(normalizeExe) + if valgrindEnabled and expected.useValgrind != disabled: + var valgrindOptions = @["--error-exitcode=1"] + if expected.useValgrind != leaking: + valgrindOptions.add "--leak-check=yes" + args = valgrindOptions & exeCmd & args exeCmd = "valgrind" var (_, buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input) # Treat all failure codes from nodejs as 1. Older versions of nodejs used @@ -464,89 +506,86 @@ proc testSpecHelper(r: var TResults, test: TTest, expected: TSpec, target: TTarg if exitCode != 0: exitCode = 1 let bufB = if expected.sortoutput: - var x = splitLines(strip(buf.string)) + var buf2 = buf + buf2.stripLineEnd + var x = splitLines(buf2) sort(x, system.cmp) - join(x, "\n") + join(x, "\n") & '\n' else: - strip(buf.string) + buf if exitCode != expected.exitCode: - r.addResult(test, target, "exitcode: " & $expected.exitCode, + given.err = reExitcodesDiffer + r.addResult(test, target, extraOptions, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode & "\n\nOutput:\n" & bufB, reExitcodesDiffer) - elif (expected.outputCheck == ocEqual and expected.output != bufB) or + elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or (expected.outputCheck == ocSubstr and expected.output notin bufB): given.err = reOutputsDiffer - r.addResult(test, target, expected.output, bufB, reOutputsDiffer) - else: - compilerOutputTests(test, target, given, expected, r) + r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionReject: - var given = callCompiler(expected.getCmd, test.name, test.options, - nimcache, target) - cmpMsgs(r, expected, given, test, target) + # Make sure its the compiler rejecting and not the system (e.g. segfault) + cmpMsgs(r, expected, given, test, target, extraOptions) + if given.exitCode != QuitFailure: + r.addResult(test, target, extraOptions, "exitcode: " & $QuitFailure, + "exitcode: " & $given.exitCode & "\n\nOutput:\n" & + given.nimout, reExitcodesDiffer) -proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = - var expected = test.spec - if expected.parseErrors.len > 0: - # targetC is a lie, but a parameter is required - r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec) - inc(r.total) - return - if not checkDisabled(r, test): return - expected.targets.incl targets - # still no target specified at all - if expected.targets == {}: - expected.targets = {getTestSpecTarget()} +proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget = + result = defaultTarget + var p = parseopt.initOptParser(extraOptions) + + while true: + parseopt.next(p) + case p.kind + of cmdEnd: break + of cmdLongOption, cmdShortOption: + if p.key == "b" or p.key == "backend": + result = parseEnum[TTarget](p.val.normalize) + # chooses the last one + else: + discard + +proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) = for target in expected.targets: inc(r.total) if target notin gTargets: - r.addResult(test, target, "", "", reDisabled) + r.addResult(test, target, extraOptions, "", "", reDisabled) inc(r.skipped) elif simulate: inc count echo "testSpec count: ", count, " expected: ", expected else: let nimcache = nimcacheDir(test.name, test.options, target) - testSpecHelper(r, test, expected, target, nimcache) + var testClone = test + let target = changeTarget(extraOptions, target) + testSpecHelper(r, testClone, expected, target, extraOptions, nimcache) -proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) = - if not checkDisabled(r, test): return - for target in test.spec.targets: +proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = + var expected = test.spec + if expected.parseErrors.len > 0: + # targetC is a lie, but a parameter is required + r.addResult(test, targetC, "", "", expected.parseErrors, reInvalidSpec) inc(r.total) - testSpecHelper(r, test, test.spec, target, nimcache) - -proc testC(r: var TResults, test: TTest, action: TTestAction) = - # runs C code. Doesn't support any specs, just goes by exit code. - if not checkDisabled(r, test): return - - let tname = test.name.addFileExt(".c") - inc(r.total) - maybeStyledEcho "Processing ", fgCyan, extractFilename(tname) - var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC) - if given.err != reSuccess: - r.addResult(test, targetC, "", given.msg, given.err) - elif action == actionRun: - let exeFile = changeFileExt(test.name, ExeExt) - var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath}) - if exitCode != 0: given.err = reExitcodesDiffer - if given.err == reSuccess: inc(r.passed) - -proc testExec(r: var TResults, test: TTest) = - # runs executable or script, just goes by exit code - if not checkDisabled(r, test): return + return - inc(r.total) - let (outp, errC) = execCmdEx(test.options.strip()) - var given: TSpec - if errC == 0: - given.err = reSuccess + expected.targets.incl targets + # still no target specified at all + if expected.targets == {}: + expected.targets = {getTestSpecTarget()} + if test.spec.matrix.len > 0: + for m in test.spec.matrix: + targetHelper(r, test, expected, m) else: - given.err = reExitcodesDiffer - given.msg = outp.string + targetHelper(r, test, expected, "") - if given.err == reSuccess: inc(r.passed) - r.addResult(test, targetC, "", given.msg, given.err) +proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} = + for target in test.spec.targets: + inc(r.total) + var testClone = test + testSpecHelper(r, testClone, test.spec, target, "", nimcache) proc makeTest(test, options: string, cat: Category): TTest = result.cat = cat @@ -555,19 +594,17 @@ proc makeTest(test, options: string, cat: Category): TTest = result.spec = parseSpec(addFileExt(test, ".nim")) result.startTime = epochTime() -proc makeRawTest(test, options: string, cat: Category): TTest = +proc makeRawTest(test, options: string, cat: Category): TTest {.used.} = result.cat = cat result.name = test result.options = options result.spec = initSpec(addFileExt(test, ".nim")) - result.startTime = epochTime() result.spec.action = actionCompile result.spec.targets = {getTestSpecTarget()} + result.startTime = epochTime() # TODO: fix these files const disabledFilesDefault = @[ - "LockFreeHash.nim", - "sharedstrings.nim", "tableimpl.nim", "setimpl.nim", "hashcommon.nim", @@ -602,13 +639,10 @@ proc loadSkipFrom(name: string): seq[string] = # used by `nlvm` (at least) for line in lines(name): let sline = line.strip() - if sline.len > 0 and not sline.startsWith("#"): + if sline.len > 0 and not sline.startsWith('#'): result.add sline proc main() = - os.putEnv "NIMTEST_COLOR", "never" - os.putEnv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES" - azure.init() backend.open() var optPrintResults = false @@ -616,42 +650,60 @@ proc main() = var targetsStr = "" var isMainProcess = true var skipFrom = "" - var useMegatest = true var p = initOptParser() p.next() while p.kind in {cmdLongOption, cmdShortOption}: - case p.key.string.normalize - of "print", "verbose": optPrintResults = true + case p.key.normalize + of "print": optPrintResults = true + of "verbose": optVerbose = true of "failing": optFailing = true - of "pedantic": discard "now always enabled" + of "pedantic": discard # deadcode refs https://github.com/nim-lang/Nim/issues/16731 of "targets": - targetsStr = p.val.string + targetsStr = p.val gTargets = parseTargets(targetsStr) + targetsSet = true of "nim": - compilerPrefix = addFileExt(p.val.string, ExeExt) + compilerPrefix = addFileExt(p.val.absolutePath, ExeExt) of "directory": - setCurrentDir(p.val.string) + setCurrentDir(p.val) of "colors": - case p.val.string: + case p.val: of "on": useColors = true of "off": useColors = false else: quit Usage + of "batch": + testamentData0.batchArg = p.val + if p.val != "_" and p.val.len > 0 and p.val[0] in {'0'..'9'}: + let s = p.val.split("_") + doAssert s.len == 2, $(p.val, s) + testamentData0.testamentBatch = s[0].parseInt + testamentData0.testamentNumBatch = s[1].parseInt + doAssert testamentData0.testamentNumBatch > 0 + doAssert testamentData0.testamentBatch >= 0 and testamentData0.testamentBatch < testamentData0.testamentNumBatch of "simulate": simulate = true of "megatest": - case p.val.string: + case p.val: of "on": useMegatest = true of "off": useMegatest = false else: quit Usage + of "valgrind": + case p.val: + of "on": + valgrindEnabled = true + of "off": + valgrindEnabled = false + else: + quit Usage of "backendlogging": - case p.val.string: + case p.val: of "on": backendLogging = true of "off": @@ -659,36 +711,39 @@ proc main() = else: quit Usage of "skipfrom": - skipFrom = p.val.string + skipFrom = p.val else: quit Usage p.next() if p.kind != cmdArgument: quit Usage - var action = p.key.string.normalize + var action = p.key.normalize p.next() var r = initResults() case action of "all": - #processCategory(r, Category"megatest", p.cmdLineRest.string, testsDir, runJoinableTests = false) + #processCategory(r, Category"megatest", p.cmdLineRest, testsDir, runJoinableTests = false) - var myself = quoteShell(findExe("testament" / "testament")) + var myself = quoteShell(getAppFilename()) if targetsStr.len > 0: myself &= " " & quoteShell("--targets:" & targetsStr) myself &= " " & quoteShell("--nim:" & compilerPrefix) + if testamentData0.batchArg.len > 0: + myself &= " --batch:" & testamentData0.batchArg if skipFrom.len > 0: myself &= " " & quoteShell("--skipFrom:" & skipFrom) var cats: seq[string] - let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: "" + let rest = if p.cmdLineRest.len > 0: " " & p.cmdLineRest else: "" for kind, dir in walkDir(testsDir): assert testsDir.startsWith(testsDir) let cat = dir[testsDir.len .. ^1] if kind == pcDir and cat notin ["testdata", "nimcache"]: cats.add cat - cats.add AdditionalCategories + if isNimRepoTests(): + cats.add AdditionalCategories if useMegatest: cats.add MegaTestCat var cmds: seq[string] @@ -697,21 +752,20 @@ proc main() = cmds.add(myself & runtype & quoteShell(cat) & rest) proc progressStatus(idx: int) = - echo "progress[all]: i: " & $idx & " / " & $cats.len & " cat: " & cats[idx] + echo "progress[all]: $1/$2 starting: cat: $3" % [$idx, $cats.len, cats[idx]] if simulate: skips = loadSkipFrom(skipFrom) for i, cati in cats: progressStatus(i) - processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, runJoinableTests = false) + processCategory(r, Category(cati), p.cmdLineRest, testsDir, runJoinableTests = false) else: - addQuitProc azure.finalize + addExitProc azure.finalize quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus) of "c", "cat", "category": skips = loadSkipFrom(skipFrom) var cat = Category(p.key) - p.next - processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true) + processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = true) of "pcat": skips = loadSkipFrom(skipFrom) # 'pcat' is used for running a category in parallel. Currently the only @@ -720,19 +774,15 @@ proc main() = isMainProcess = false var cat = Category(p.key) p.next - processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = false) + processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = false) of "p", "pat", "pattern": skips = loadSkipFrom(skipFrom) let pattern = p.key p.next - processPattern(r, pattern, p.cmdLineRest.string, simulate) + processPattern(r, pattern, p.cmdLineRest, simulate) of "r", "run": - # at least one directory is required in the path, to use as a category name - let pathParts = split(p.key.string, {DirSep, AltSep}) - # "stdlib/nre/captures.nim" -> "stdlib" + "nre/captures.nim" - let cat = Category(pathParts[0]) - let subPath = joinPath(pathParts[1..^1]) - processSingleTest(r, cat, p.cmdLineRest.string, subPath) + let (cat, path) = splitTestFile(p.key) + processSingleTest(r, cat.Category, p.cmdLineRest, path, gTargets, targetsSet) of "html": generateHtml(resultsFile, optFailing) else: diff --git a/testament/testament.nim.cfg b/testament/testament.nim.cfg index 69f044554..c97284129 100644 --- a/testament/testament.nim.cfg +++ b/testament/testament.nim.cfg @@ -1,2 +1,6 @@ +# don't move this file without updating the logic in `isNimRepoTests` + path = "$nim" # For compiler/nodejs -d:ssl # For azure +# my SSL doesn't have this feature and I don't care: +-d:nimDisableCertificateValidation diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim index a8d216a5b..477da1e23 100644 --- a/testament/tests/shouldfail/tccodecheck.nim +++ b/testament/tests/shouldfail/tccodecheck.nim @@ -1,5 +1,5 @@ discard """ -ccodecheck: "baz" + ccodecheck: "baz" """ proc foo(): void {.exportc: "bar".}= diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim index 89482e673..809ddec74 100644 --- a/testament/tests/shouldfail/tcolumn.nim +++ b/testament/tests/shouldfail/tcolumn.nim @@ -1,7 +1,7 @@ discard """ -errormsg: "undeclared identifier: 'undeclared'" -line: 8 -column: 7 + errormsg: "undeclared identifier: 'undeclared'" + line: 9 + column: 7 """ # test should fail because the line directive is wrong diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim index dbbdf5021..6c130d107 100644 --- a/testament/tests/shouldfail/terrormsg.nim +++ b/testament/tests/shouldfail/terrormsg.nim @@ -1,7 +1,7 @@ discard """ -errormsg: "wrong error message" -line: 8 -column: 6 + errormsg: "wrong error message" + line: 9 + column: 6 """ # test should fail because the line directive is wrong diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim index 1b38b4f2e..605f046db 100644 --- a/testament/tests/shouldfail/texitcode1.nim +++ b/testament/tests/shouldfail/texitcode1.nim @@ -1,3 +1,3 @@ discard """ -exitcode: 1 + exitcode: 1 """ diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim index a6c2ad359..b40a4f44f 100644 --- a/testament/tests/shouldfail/tfile.nim +++ b/testament/tests/shouldfail/tfile.nim @@ -1,6 +1,6 @@ discard """ -errmsg: "undeclared identifier: 'undefined'" -file: "notthisfile.nim" + errormsg: "undeclared identifier: 'undefined'" + file: "notthisfile.nim" """ echo undefined diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim index f7a09875c..fe782eb03 100644 --- a/testament/tests/shouldfail/tline.nim +++ b/testament/tests/shouldfail/tline.nim @@ -1,7 +1,7 @@ discard """ -errormsg: "undeclared identifier: 'undeclared'" -line: 9 -column: 6 + errormsg: "undeclared identifier: 'undeclared'" + line: 10 + column: 6 """ # test should fail because the line directive is wrong diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim index 9879e4181..92022ee97 100644 --- a/testament/tests/shouldfail/tmaxcodesize.nim +++ b/testament/tests/shouldfail/tmaxcodesize.nim @@ -1,5 +1,5 @@ discard """ -maxcodesize: 1 + maxcodesize: 1 """ echo "Hello World" diff --git a/testament/tests/shouldfail/tmsg.nim b/testament/tests/shouldfail/tmsg.nim deleted file mode 100644 index 4ad17fa95..000000000 --- a/testament/tests/shouldfail/tmsg.nim +++ /dev/null @@ -1,6 +0,0 @@ -discard """ -msg: "Hello World" -""" - -static: - echo "something else" diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim index c0e332053..0a65bfb70 100644 --- a/testament/tests/shouldfail/tnimout.nim +++ b/testament/tests/shouldfail/tnimout.nim @@ -1,6 +1,6 @@ discard """ -nimout: "Hello World!" -action: compile + nimout: "Hello World!" + action: compile """ static: diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim new file mode 100644 index 000000000..4fc93f6d2 --- /dev/null +++ b/testament/tests/shouldfail/tnimoutfull.nim @@ -0,0 +1,14 @@ +discard """ + nimout: ''' +msg1 +msg2 +''' + action: compile + nimoutFull: true +""" + +# should fail because `msg3` is not in nimout and `nimoutFill: true` was given +static: + echo "msg1" + echo "msg2" + echo "msg3" diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim index ac0bc7a46..eaf9e8652 100644 --- a/testament/tests/shouldfail/toutput.nim +++ b/testament/tests/shouldfail/toutput.nim @@ -1,7 +1,7 @@ discard """ -output: ''' -done -''' + output: ''' + done + ''' """ echo "broken" diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim index 7cc51ee8d..47324ecee 100644 --- a/testament/tests/shouldfail/toutputsub.nim +++ b/testament/tests/shouldfail/toutputsub.nim @@ -1,5 +1,5 @@ discard """ -outputsub: "something else" + outputsub: "something else" """ echo "Hello World!" diff --git a/testament/tests/shouldfail/treject.nim b/testament/tests/shouldfail/treject.nim new file mode 100644 index 000000000..1e7258f70 --- /dev/null +++ b/testament/tests/shouldfail/treject.nim @@ -0,0 +1,7 @@ +discard """ + action: "reject" +""" + +# Because we set action="reject", we expect this line not to compile. But the +# line does compile, therefore the test fails. +assert true diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim index 4ce9ce26d..69dfbc0a0 100644 --- a/testament/tests/shouldfail/tsortoutput.nim +++ b/testament/tests/shouldfail/tsortoutput.nim @@ -1,6 +1,6 @@ discard """ -sortoutput: true -output: ''' + sortoutput: true + output: ''' 2 1 ''' diff --git a/testament/tests/shouldfail/tvalgrind.nim b/testament/tests/shouldfail/tvalgrind.nim new file mode 100644 index 000000000..d551ff12e --- /dev/null +++ b/testament/tests/shouldfail/tvalgrind.nim @@ -0,0 +1,17 @@ +discard """ + valgrind: true + cmd: "nim $target --gc:arc -d:useMalloc $options $file" +""" + +# this is the same check used by testament/specs.nim whether or not valgrind +# tests are supported +when defined(linux) and sizeof(int) == 8: + # discarding this allocation will cause valgrind to fail (which is what we + # want), but valgrind only runs on 64-bit Linux machines... + discard alloc(1) +else: + # ...so on all other OS/architectures, simulate any non-zero exit code to + # mimic how valgrind would have failed on this test. We cannot use things like + # `disabled: "freebsd"` in the Testament configs above or else the tests will + # be SKIP-ed rather than FAIL-ed + quit(1) # choose 1 to match valgrind's `--error-exit=1`, but could be anything |