diff options
Diffstat (limited to 'testament')
26 files changed, 1127 insertions, 868 deletions
diff --git a/testament/backend.nim b/testament/backend.nim index 0a6d7d8b9..1770c6657 100644 --- a/testament/backend.nim +++ b/testament/backend.nim @@ -13,7 +13,6 @@ type CommitId = distinct string proc `$`*(id: MachineId): string {.borrow.} -#proc `$`(id: CommitId): string {.borrow.} # not used var thisMachine: MachineId @@ -21,10 +20,10 @@ var thisBranch: string proc getMachine*(): MachineId = - var name = execProcess("hostname").string.strip + var name = execProcess("hostname").strip if name.len == 0: - name = when defined(posix): getEnv("HOSTNAME").string - else: getEnv("COMPUTERNAME").string + name = when defined(posix): getEnv("HOSTNAME") + else: getEnv("COMPUTERNAME") if name.len == 0: quit "cannot determine the machine name" @@ -32,8 +31,8 @@ proc getMachine*(): MachineId = proc getCommit(): CommitId = const commLen = "commit ".len - let hash = execProcess("git log -n 1").string.strip[commLen..commLen+10] - thisBranch = execProcess("git symbolic-ref --short HEAD").string.strip + let hash = execProcess("git log -n 1").strip[commLen..commLen+10] + thisBranch = execProcess("git symbolic-ref --short HEAD").strip if hash.len == 0 or thisBranch.len == 0: quit "cannot determine git HEAD" result = CommitId(hash) diff --git a/testament/caasdriver.nim b/testament/caasdriver.nim index 30383bddb..01e402e07 100644 --- a/testament/caasdriver.nim +++ b/testament/caasdriver.nim @@ -62,7 +62,7 @@ proc doCaasCommand(session: var NimSession, command: string): string = result = "" while true: - var line = TaintedString("") + var line = "" if session.nim.outputStream.readLine(line): if line.string == "": break result.add(line.string & "\n") @@ -78,7 +78,7 @@ proc doProcCommand(session: var NimSession, command: string): string = var process = startProcess(NimBin, args = session.replaceVars(command).split) stream = outputStream(process) - line = TaintedString("") + line = "" result = "" while stream.readLine(line): @@ -113,12 +113,12 @@ proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): result = true var f = open(script) - var project = TaintedString("") + var project = "" if f.readLine(project): var s = startNimSession(script.parentDir / project.string, script, mode) - tline = TaintedString("") + tline = "" ln = 1 while f.readLine(tline): @@ -175,7 +175,7 @@ when isMainModule: verbose = false for i in 0..paramCount() - 1: - let param = string(paramStr(i + 1)) + let param = paramStr(i + 1) case param of "verbose": verbose = true else: filter = param diff --git a/testament/categories.nim b/testament/categories.nim index c894bc9f9..843bef3f9 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -13,6 +13,8 @@ # included from testament.nim import important_packages +import std/[strformat, strutils] +from std/sequtils import filterIt const specialCategories = [ @@ -21,25 +23,21 @@ const "debugger", "dll", "examples", - "flags", "gc", "io", "js", "ic", "lib", - "longgc", "manyloc", - "nimble-packages-1", - "nimble-packages-2", + "nimble-packages", "niminaction", "threads", - "untestable", + "untestable", # see trunner_special "testdata", "nimcache", "coroutines", "osproc", "shouldfail", - "dir with space", "destructor" ] @@ -47,72 +45,9 @@ proc isTestFile*(file: string): bool = let (_, name, ext) = splitFile(file) result = ext == ".nim" and name.startsWith("t") -# ---------------- IC tests --------------------------------------------- - -proc icTests(r: var TResults; testsDir: string, cat: Category, options: string) = - const - tooltests = ["compiler/nim.nim", "tools/nimgrep.nim"] - writeOnly = " --incremental:writeonly " - readOnly = " --incremental:readonly " - incrementalOn = " --incremental:on " - - template test(x: untyped) = - testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) - - template editedTest(x: untyped) = - var test = makeTest(file, x & options, cat) - test.spec.targets = {getTestSpecTarget()} - testSpecWithNimcache(r, test, nimcache) - - const tempExt = "_temp.nim" - for it in walkDirRec(testsDir / "ic"): - if isTestFile(it) and not it.endsWith(tempExt): - let nimcache = nimcacheDir(it, options, getTestSpecTarget()) - removeDir(nimcache) - - let content = readFile(it) - for fragment in content.split("#!EDIT!#"): - let file = it.replace(".nim", tempExt) - writeFile(file, fragment) - let oldPassed = r.passed - editedTest incrementalOn - if r.passed != oldPassed+1: break - - for file in tooltests: - let nimcache = nimcacheDir(file, options, getTestSpecTarget()) - removeDir(nimcache) - - let oldPassed = r.passed - test writeOnly - - if r.passed == oldPassed+1: - test readOnly - if r.passed == oldPassed+2: - test readOnly & "-d:nimBackendAssumesChange " - -# --------------------- flags tests ------------------------------------------- - -proc flagTests(r: var TResults, cat: Category, options: string) = - # --genscript - const filename = testsDir/"flags"/"tgenscript" - const genopts = " --genscript" - let nimcache = nimcacheDir(filename, genopts, targetC) - testSpec r, makeTest(filename, genopts, cat) - - when defined(windows): - testExec r, makeTest(filename, " cmd /c cd " & nimcache & - " && compile_tgenscript.bat", cat) - - elif defined(posix): - testExec r, makeTest(filename, " sh -c \"cd " & nimcache & - " && sh compile_tgenscript.sh\"", cat) - - # Run - testExec r, makeTest(filename, " " & nimcache / "tgenscript", cat) - # --------------------- DLL generation tests ---------------------------------- -proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = +proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = false) = const rpath = when defined(macosx): " --passL:-rpath --passL:@loader_path" else: @@ -124,42 +59,53 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) test2.spec.action = actionCompile testSpec c, test2 - var test3 = makeTest("lib/nimhcr.nim", options & " --outdir:tests/dll" & rpath, cat) + + var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) test3.spec.action = actionCompile testSpec c, test3 - var test4 = makeTest("tests/dll/visibility.nim", options & " --app:lib" & rpath, cat) + var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat) test4.spec.action = actionCompile testSpec c, test4 # windows looks in the dir of the exe (yay!): - when not defined(Windows): + when not defined(windows): # posix relies on crappy LD_LIBRARY_PATH (ugh!): const libpathenv = when defined(haiku): "LIBRARY_PATH" else: "LD_LIBRARY_PATH" - var libpath = getEnv(libpathenv).string + var libpath = getEnv(libpathenv) # Temporarily add the lib directory to LD_LIBRARY_PATH: putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: "")) defer: putEnv(libpathenv, libpath) testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) - testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & rpath, cat) - testSpec r, makeTest("tests/dll/visibility.nim", options & rpath, cat) + testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) + testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) if "boehm" notin options: + # hcr tests + + var basicHcrTest = makeTest("tests/dll/nimhcr_basic.nim", options & " --threads:off --forceBuild --hotCodeReloading:on " & rpath, cat) + # test segfaults for now but compiles: + if isOrc: basicHcrTest.spec.action = actionCompile + testSpec r, basicHcrTest + # force build required - see the comments in the .nim file for more details var hcri = makeTest("tests/dll/nimhcr_integration.nim", - options & " --forceBuild --hotCodeReloading:on" & rpath, cat) + options & " --threads:off --forceBuild --hotCodeReloading:on" & rpath, cat) let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget()) - hcri.args = prepareTestArgs(hcri.spec.getCmd, hcri.name, + let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name, hcri.options, nimcache, getTestSpecTarget()) + hcri.testArgs = cmd.parseCmdLine testSpec r, hcri proc dllTests(r: var TResults, cat: Category, options: string) = # dummy compile result: var c = initResults() - runBasicDLLTest c, r, cat, options - runBasicDLLTest c, r, cat, options & " -d:release" + runBasicDLLTest c, r, cat, options & " --mm:refc" + runBasicDLLTest c, r, cat, options & " -d:release --mm:refc" + runBasicDLLTest c, r, cat, options, isOrc = true + runBasicDLLTest c, r, cat, options & " -d:release", isOrc = true when not defined(windows): # still cannot find a recent Windows version of boehm.dll: runBasicDLLTest c, r, cat, options & " --gc:boehm" @@ -169,9 +115,9 @@ proc dllTests(r: var TResults, cat: Category, options: string) = proc gcTests(r: var TResults, cat: Category, options: string) = template testWithoutMs(filename: untyped) = - testSpec r, makeTest("tests/gc" / filename, options, cat) + testSpec r, makeTest("tests/gc" / filename, options & "--mm:refc", cat) testSpec r, makeTest("tests/gc" / filename, options & - " -d:release -d:useRealtimeGC", cat) + " -d:release -d:useRealtimeGC --mm:refc", cat) when filename != "gctest": testSpec r, makeTest("tests/gc" / filename, options & " --gc:orc", cat) @@ -216,19 +162,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "stackrefleak" test "cyclecollector" - -proc longGCTests(r: var TResults, cat: Category, options: string) = - when defined(windows): - let cOptions = "-ldl -DWIN" - else: - let cOptions = "-ldl" - - var c = initResults() - # According to ioTests, this should compile the file - testSpec c, makeTest("tests/realtimeGC/shared", options, cat) - # ^- why is this not appended to r? Should this be discarded? - testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun - testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat) + testWithoutBoehm "trace_globals" # ------------------------- threading tests ----------------------------------- @@ -247,6 +181,11 @@ proc ioTests(r: var TResults, cat: Category, options: string) = # dummy compile result: var c = initResults() testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat) + # ^- why is this not appended to r? Should this be discarded? + # EDIT: this should be replaced by something like in D20210524T180826, + # likewise in similar instances where `testSpec c` is used, or more generally + # when a test depends on another test, as it makes tests non-independent, + # creating complications for batching and megatest logic. testSpec r, makeTest("tests/system/tio", options, cat) # ------------------------- async tests --------------------------------------- @@ -264,7 +203,7 @@ proc debuggerTests(r: var TResults, cat: Category, options: string) = t.spec.action = actionCompile # force target to C because of MacOS 10.15 SDK headers bug # https://github.com/nim-lang/Nim/pull/15612#issuecomment-712471879 - t.spec.targets = { targetC } + t.spec.targets = {targetC} testSpec r, t # ------------------------- JS tests ------------------------------------------ @@ -279,9 +218,9 @@ proc jsTests(r: var TResults, cat: Category, options: string) = for testfile in ["exception/texceptions", "exception/texcpt1", "exception/texcsub", "exception/tfinally", "exception/tfinally2", "exception/tfinally3", - "actiontable/tactiontable", "method/tmultimjs", + "collections/tactiontable", "method/tmultimjs", "varres/tvarres0", "varres/tvarres3", "varres/tvarres4", - "varres/tvartup", "misc/tints", "misc/tunsignedinc", + "varres/tvartup", "int/tints", "int/tunsignedinc", "async/tjsandnativeasync"]: test "tests/" & testfile & ".nim" @@ -291,8 +230,6 @@ proc jsTests(r: var TResults, cat: Category, options: string) = # ------------------------- nim in action ----------- proc testNimInAction(r: var TResults, cat: Category, options: string) = - let options = options & " --nilseqs:on" - template test(filename: untyped) = testSpec r, makeTest(filename, options, cat) @@ -383,7 +320,7 @@ proc findMainFile(dir: string): string = var nimFiles = 0 for kind, file in os.walkDir(dir): if kind == pcFile: - if file.endsWith(cfgExt): return file[.. ^(cfgExt.len+1)] & ".nim" + if file.endsWith(cfgExt): return file[0..^(cfgExt.len+1)] & ".nim" elif file.endsWith(".nim"): if result.len == 0: result = file inc nimFiles @@ -428,7 +365,7 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = files.sort # reproducible order for testFile in files: - let contents = readFile(testFile).string + let contents = readFile(testFile) var testObj = makeTest(testFile, options, cat) #[ todo: @@ -443,131 +380,165 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = testSpec r, testObj # ----------------------------- nimble ---------------------------------------- - -var nimbleDir = getEnv("NIMBLE_DIR").string -if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" -let - nimbleExe = findExe("nimble") - packageIndex = nimbleDir / "packages_official.json" - -type - PkgPart = enum - ppOne - ppTwo - -iterator listPackages(part: PkgPart): tuple[name, cmd, url: string, useHead: bool] = +proc listPackagesAll(): seq[NimblePackage] = + var nimbleDir = getEnv("NIMBLE_DIR") + if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" + let packageIndex = nimbleDir / "packages_official.json" let packageList = parseFile(packageIndex) - let importantList = - case part - of ppOne: important_packages.packages1 - of ppTwo: important_packages.packages2 - for n, cmd, url, useHead in importantList.items: - if url.len != 0: - yield (n, cmd, url, useHead) + proc findPackage(name: string): JsonNode = + for a in packageList: + if a["name"].str == name: return a + for pkg in important_packages.packages.items: + var pkg = pkg + if pkg.url.len == 0: + let pkg2 = findPackage(pkg.name) + if pkg2 == nil: + raise newException(ValueError, "Cannot find package '$#'." % pkg.name) + pkg.url = pkg2["url"].str + result.add pkg + +proc listPackages(packageFilter: string): seq[NimblePackage] = + let pkgs = listPackagesAll() + if packageFilter.len != 0: + # xxx document `packageFilter`, seems like a bad API, + # at least should be a regex; a substring match makes no sense. + result = pkgs.filterIt(packageFilter in it.name) + else: + if testamentData0.batchArg == "allowed_failures": + result = pkgs.filterIt(it.allowFailure) + elif testamentData0.testamentNumBatch == 0: + result = pkgs else: - var found = false - for package in packageList.items: - let name = package["name"].str - if name == n: - found = true - let pUrl = package["url"].str - yield (name, cmd, pUrl, useHead) - break - if not found: - raise newException(ValueError, "Cannot find package '$#'." % n) - -proc makeSupTest(test, options: string, cat: Category): TTest = + let pkgs2 = pkgs.filterIt(not it.allowFailure) + for i in 0..<pkgs2.len: + if i mod testamentData0.testamentNumBatch == testamentData0.testamentBatch: + result.add pkgs2[i] + +proc makeSupTest(test, options: string, cat: Category, debugInfo = ""): TTest = result.cat = cat result.name = test result.options = options + result.debugInfo = debugInfo result.startTime = epochTime() -const maxRetries = 3 -template retryCommand(call): untyped = - var res: typeof(call) - var backoff = 1 - for i in 0..<maxRetries: - res = call - if res.exitCode == QuitSuccess or i == maxRetries-1: break - sleep(backoff * 1000) - backoff *= 2 - res - -proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string, part: PkgPart) = - if nimbleExe == "": - echo "[Warning] - Cannot run nimble tests: Nimble binary not found." - return - if execCmd("$# update" % nimbleExe) == QuitFailure: - echo "[Warning] - Cannot run nimble tests: Nimble update failed." - return +import std/private/gitutils +proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = + let nimbleExe = findExe("nimble") + doAssert nimbleExe != "", "Cannot run nimble tests: Nimble binary not found." + doAssert execCmd("$# update" % nimbleExe) == 0, "Cannot run nimble tests: Nimble update failed." let packageFileTest = makeSupTest("PackageFileParsed", "", cat) let packagesDir = "pkgstemp" createDir(packagesDir) var errors = 0 try: - for name, cmd, url, useHead in listPackages(part): - if packageFilter notin name: - continue + let pkgs = listPackages(packageFilter) + for i, pkg in pkgs: inc r.total - var test = makeSupTest(name, "", cat) - let buildPath = packagesDir / name - - if not dirExists(buildPath): - let (cloneCmd, cloneOutput, cloneStatus) = retryCommand execCmdEx2("git", ["clone", url, buildPath]) - if cloneStatus != QuitSuccess: - r.addResult(test, targetC, "", cloneCmd & "\n" & cloneOutput, reInstallFailed) - continue - - if not useHead: - let (fetchCmd, fetchOutput, fetchStatus) = retryCommand execCmdEx2("git", ["fetch", "--tags"], workingDir = buildPath) - if fetchStatus != QuitSuccess: - r.addResult(test, targetC, "", fetchCmd & "\n" & fetchOutput, reInstallFailed) - continue - - let (describeCmd, describeOutput, describeStatus) = retryCommand execCmdEx2("git", ["describe", "--tags", "--abbrev=0"], workingDir = buildPath) - if describeStatus != QuitSuccess: - r.addResult(test, targetC, "", describeCmd & "\n" & describeOutput, reInstallFailed) - continue - - let (checkoutCmd, checkoutOutput, checkoutStatus) = retryCommand execCmdEx2("git", ["checkout", describeOutput.strip], workingDir = buildPath) - if checkoutStatus != QuitSuccess: - r.addResult(test, targetC, "", checkoutCmd & "\n" & checkoutOutput, reInstallFailed) - continue - - let (installDepsCmd, installDepsOutput, installDepsStatus) = retryCommand execCmdEx2("nimble", ["install", "--depsOnly", "-y"], workingDir = buildPath) - if installDepsStatus != QuitSuccess: - r.addResult(test, targetC, "", "installing dependencies failed:\n$ " & installDepsCmd & "\n" & installDepsOutput, reInstallFailed) + var test = makeSupTest(pkg.name, "", cat, "[$#/$#] " % [$i, $pkgs.len]) + let buildPath = packagesDir / pkg.name + template tryCommand(cmd: string, workingDir2 = buildPath, reFailed = reInstallFailed, maxRetries = 1): string = + var outp: string + let ok = retryCall(maxRetry = maxRetries, backoffDuration = 10.0): + var status: int + (outp, status) = execCmdEx(cmd, workingDir = workingDir2) + status == QuitSuccess + if not ok: + if pkg.allowFailure: + inc r.passed + inc r.failedButAllowed + addResult(r, test, targetC, "", "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure) continue + outp - let cmdArgs = parseCmdLine(cmd) - - let (buildCmd, buildOutput, status) = execCmdEx2(cmdArgs[0], cmdArgs[1..^1], workingDir = buildPath) - if status != QuitSuccess: - r.addResult(test, targetC, "", "package test failed\n$ " & buildCmd & "\n" & buildOutput, reBuildFailed) - else: - inc r.passed - r.addResult(test, targetC, "", "", reSuccess) + if not dirExists(buildPath): + discard tryCommand("git clone $# $#" % [pkg.url.quoteShell, buildPath.quoteShell], workingDir2 = ".", maxRetries = 3) + if not pkg.useHead: + discard tryCommand("git fetch --tags", maxRetries = 3) + let describeOutput = tryCommand("git describe --tags --abbrev=0") + discard tryCommand("git checkout $#" % [describeOutput.strip.quoteShell]) + discard tryCommand("nimble install --depsOnly -y", maxRetries = 3) + let cmds = pkg.cmd.split(';') + for i in 0 ..< cmds.len - 1: + discard tryCommand(cmds[i], maxRetries = 3) + discard tryCommand(cmds[^1], reFailed = reBuildFailed) + inc r.passed + r.addResult(test, targetC, "", "", "", reSuccess, allowFailure = pkg.allowFailure) errors = r.total - r.passed if errors == 0: - r.addResult(packageFileTest, targetC, "", "", reSuccess) + r.addResult(packageFileTest, targetC, "", "", "", reSuccess) else: - r.addResult(packageFileTest, targetC, "", "", reBuildFailed) + r.addResult(packageFileTest, targetC, "", "", "", reBuildFailed) except JsonParsingError: - echo "[Warning] - Cannot run nimble tests: Invalid package file." - r.addResult(packageFileTest, targetC, "", "Invalid package file", reBuildFailed) + errors = 1 + r.addResult(packageFileTest, targetC, "", "", "Invalid package file", reBuildFailed) + raise except ValueError: - echo "[Warning] - $#" % getCurrentExceptionMsg() - r.addResult(packageFileTest, targetC, "", "Unknown package", reBuildFailed) + errors = 1 + r.addResult(packageFileTest, targetC, "", "", "Unknown package", reBuildFailed) + raise # bug #18805 finally: if errors == 0: removeDir(packagesDir) +# ---------------- IC tests --------------------------------------------- + +proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; + isNavigatorTest: bool) = + const + tooltests = ["compiler/nim.nim"] + writeOnly = " --incremental:writeonly " + readOnly = " --incremental:readonly " + incrementalOn = " --incremental:on -d:nimIcIntegrityChecks " + navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint:Conf:off --warnings:off " + + template test(x: untyped) = + testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) + + template editedTest(x: untyped) = + var test = makeTest(file, x & options, cat) + if isNavigatorTest: + test.spec.action = actionCompile + test.spec.targets = {getTestSpecTarget()} + testSpecWithNimcache(r, test, nimcache) + + template checkTest() = + var test = makeRawTest(file, options, cat) + test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:on $options " & file + testSpecWithNimcache(r, test, nimcache) + + if not isNavigatorTest: + for file in tooltests: + let nimcache = nimcacheDir(file, options, getTestSpecTarget()) + removeDir(nimcache) + + let oldPassed = r.passed + checkTest() + + if r.passed == oldPassed+1: + checkTest() + if r.passed == oldPassed+2: + checkTest() + + const tempExt = "_temp.nim" + for it in walkDirRec(testsDir): + # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test + if isTestFile(it) and not it.endsWith(tempExt): + let nimcache = nimcacheDir(it, options, getTestSpecTarget()) + removeDir(nimcache) + + let content = readFile(it) + for fragment in content.split("#!EDIT!#"): + let file = it.replace(".nim", tempExt) + writeFile(file, fragment) + let oldPassed = r.passed + editedTest(if isNavigatorTest: navTestConfig else: incrementalOn) + if r.passed != oldPassed+1: break # ---------------------------------------------------------------------------- -const AdditionalCategories = ["debugger", "examples", "lib", "ic"] +const AdditionalCategories = ["debugger", "examples", "lib", "ic", "navigator"] const MegaTestCat = "megatest" proc `&.?`(a, b: string): string = @@ -583,7 +554,9 @@ proc processSingleTest(r: var TResults, cat: Category, options, test: string, ta testSpec r, makeTest(test, options, cat), targets proc isJoinableSpec(spec: TSpec): bool = - result = not spec.sortoutput and + # xxx simplify implementation using a whitelist of fields that are allowed to be + # set to non-default values (use `fieldPairs`), to avoid issues like bug #16576. + result = useMegatest and not spec.sortoutput and spec.action == actionRun and not fileExists(spec.file.changeFileExt("cfg")) and not fileExists(spec.file.changeFileExt("nims")) and @@ -595,6 +568,10 @@ proc isJoinableSpec(spec: TSpec): bool = spec.exitCode == 0 and spec.input.len == 0 and spec.nimout.len == 0 and + spec.nimoutFull == false and + # so that tests can have `nimoutFull: true` with `nimout.len == 0` with + # the meaning that they expect empty output. + spec.matrix.len == 0 and spec.outputCheck != ocSubstr and spec.ccodeCheck.len == 0 and (spec.targets == {} or spec.targets == {targetC}) @@ -602,34 +579,37 @@ proc isJoinableSpec(spec: TSpec): bool = if spec.file.readFile.contains "when isMainModule": result = false -when false: - proc norm(s: var string) = - ## strip empty newlines - while true: - let tmp = s.replace("\n\n", "\n") - if tmp == s: break - s = tmp - s = s.strip - proc quoted(a: string): string = # todo: consider moving to system.nim result.addQuoted(a) -proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = +proc runJoinedTest(r: var TResults, cat: Category, testsDir: string, options: string) = ## returns a list of tests that have problems + #[ + xxx create a reusable megatest API after abstracting out testament specific code, + refs https://github.com/timotheecour/Nim/issues/655 + and https://github.com/nim-lang/gtk2/pull/28; it's useful in other contexts. + ]# var specs: seq[TSpec] = @[] for kind, dir in walkDir(testsDir): - assert testsDir.startsWith(testsDir) + assert dir.startsWith(testsDir) let cat = dir[testsDir.len .. ^1] if kind == pcDir and cat notin specialCategories: for file in walkDirRec(testsDir / cat): if isTestFile(file): - let spec = parseSpec(file) + var spec: TSpec + try: + spec = parseSpec(file) + except ValueError: + # e.g. for `tests/navigator/tincludefile.nim` which have multiple + # specs; this will be handled elsewhere + echo "parseSpec raised ValueError for: '$1', assuming this will be handled outside of megatest" % file + continue if isJoinableSpec(spec): specs.add spec - proc cmp(a: TSpec, b:TSpec): auto = cmp(a.file, b.file) - sort(specs, cmp=cmp) # reproducible order + proc cmp(a: TSpec, b: TSpec): auto = cmp(a.file, b.file) + sort(specs, cmp = cmp) # reproducible order echo "joinable specs: ", specs.len if simulate: @@ -641,127 +621,135 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = var megatest: string # xxx (minor) put outputExceptedFile, outputGottenFile, megatestFile under here or `buildDir` var outDir = nimcacheDir(testsDir / "megatest", "", targetC) - const marker = "megatest:processing: " - + template toMarker(file, i): string = + "megatest:processing: [$1] $2" % [$i, file] for i, runSpec in specs: let file = runSpec.file - let file2 = outDir / ("megatest_$1.nim" % $i) + let file2 = outDir / ("megatest_a_$1.nim" % $i) # `include` didn't work with `trecmod2.nim`, so using `import` - let code = "echo \"$1\", $2\n" % [marker, quoted(file)] + let code = "echo $1\nstatic: echo \"CT:\", $1\n" % [toMarker(file, i).quoted] createDir(file2.parentDir) writeFile(file2, code) - megatest.add "import $1\nimport $2\n" % [quoted(file2), quoted(file)] + megatest.add "import $1\nimport $2 as megatest_b_$3\n" % [file2.quoted, file.quoted, $i] let megatestFile = testsDir / "megatest.nim" # so it uses testsDir / "config.nims" writeFile(megatestFile, megatest) let root = getCurrentDir() - let args = ["c", "--nimCache:" & outDir, "-d:testing", "--listCmd", "--path:" & root, megatestFile] + + var args = @["c", "--nimCache:" & outDir, "-d:testing", "-d:nimMegatest", "--listCmd", + "--path:" & root] + args.add options.parseCmdLine + args.add megatestFile var (cmdLine, buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, input = "") if exitCode != 0: - echo "$ " & cmdLine & "\n" & buf.string + echo "$ " & cmdLine & "\n" & buf quit(failString & "megatest compilation failed") (buf, exitCode) = execCmdEx(megatestFile.changeFileExt(ExeExt).dup normalizeExe) if exitCode != 0: - echo buf.string + echo buf quit(failString & "megatest execution failed") const outputExceptedFile = "outputExpected.txt" const outputGottenFile = "outputGotten.txt" - writeFile(outputGottenFile, buf.string) + writeFile(outputGottenFile, buf) var outputExpected = "" for i, runSpec in specs: - outputExpected.add marker & runSpec.file & "\n" + outputExpected.add toMarker(runSpec.file, i) & "\n" if runSpec.output.len > 0: outputExpected.add runSpec.output if not runSpec.output.endsWith "\n": outputExpected.add '\n' - if buf.string != outputExpected: + if buf != outputExpected: writeFile(outputExceptedFile, outputExpected) - discard execShellCmd("diff -uNdr $1 $2" % [outputExceptedFile, outputGottenFile]) - echo failString & "megatest output different!" + echo diffFiles(outputGottenFile, outputExceptedFile).output + echo failString & "megatest output different, see $1 vs $2" % [outputGottenFile, outputExceptedFile] # outputGottenFile, outputExceptedFile not removed on purpose for debugging. quit 1 else: echo "megatest output OK" - when false: # no point removing those, always good for debugging - removeFile(outputGottenFile) - removeFile(megatestFile) # keep it around - #testSpec r, makeTest("megatest", options, cat) + # --------------------------------------------------------------------------- proc processCategory(r: var TResults, cat: Category, options, testsDir: string, runJoinableTests: bool) = - case cat.string.normalize - of "ic": - when false: - icTests(r, testsDir, cat, options) - of "js": - # only run the JS tests on Windows or Linux because Travis is bad - # and other OSes like Haiku might lack nodejs: - if not defined(linux) and isTravis: + let cat2 = cat.string.normalize + var handled = false + if isNimRepoTests(): + handled = true + case cat2 + of "js": + # only run the JS tests on Windows or Linux because Travis is bad + # and other OSes like Haiku might lack nodejs: + if not defined(linux) and isTravis: + discard + else: + jsTests(r, cat, options) + of "dll": + dllTests(r, cat, options & " -d:nimDebugDlOpen") + of "gc": + gcTests(r, cat, options) + of "debugger": + debuggerTests(r, cat, options) + of "manyloc": + manyLoc r, cat, options + of "threads": + threadTests r, cat, options & " --threads:on" + of "io": + ioTests r, cat, options + of "async": + asyncTests r, cat, options + of "lib": + testStdlib(r, "lib/pure/", options, cat) + testStdlib(r, "lib/packages/docutils/", options, cat) + of "examples": + compileExample(r, "examples/*.nim", options, cat) + compileExample(r, "examples/gtk/*.nim", options, cat) + compileExample(r, "examples/talk/*.nim", options, cat) + of "nimble-packages": + testNimblePackages(r, cat, options) + of "niminaction": + testNimInAction(r, cat, options) + of "ic": + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=false) + of "navigator": + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=true) + of "untestable": + # These require special treatment e.g. because they depend on a third party + # dependency; see `trunner_special` which runs some of those. discard else: - jsTests(r, cat, options) - of "dll": - dllTests(r, cat, options) - of "flags": - flagTests(r, cat, options) - of "gc": - gcTests(r, cat, options) - of "longgc": - longGCTests(r, cat, options) - of "debugger": - debuggerTests(r, cat, options) - of "manyloc": - manyLoc r, cat, options - of "threads": - threadTests r, cat, options & " --threads:on" - of "io": - ioTests r, cat, options - of "async": - asyncTests r, cat, options - of "lib": - testStdlib(r, "lib/pure/", options, cat) - testStdlib(r, "lib/packages/docutils/", options, cat) - of "examples": - compileExample(r, "examples/*.nim", options, cat) - compileExample(r, "examples/gtk/*.nim", options, cat) - compileExample(r, "examples/talk/*.nim", options, cat) - of "nimble-packages-1": - testNimblePackages(r, cat, options, ppOne) - of "nimble-packages-2": - testNimblePackages(r, cat, options, ppTwo) - of "niminaction": - testNimInAction(r, cat, options) - of "untestable": - # We can't test it because it depends on a third party. - discard # TODO: Move untestable tests to someplace else, i.e. nimble repo. - of "megatest": - runJoinedTest(r, cat, testsDir) - else: - var testsRun = 0 - var files: seq[string] - for file in walkDirRec(testsDir &.? cat.string): - if isTestFile(file): files.add file - files.sort # give reproducible order - - for i, name in files: - var test = makeTest(name, options, cat) - if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories: - discard "run the test" - else: - test.spec.err = reJoined - testSpec r, test - inc testsRun - if testsRun == 0: - const whiteListedDirs = ["deps"] - doAssert cat.string in whiteListedDirs, - "Invalid category specified: '$#' not in whilelist: $#" % [cat.string, $whiteListedDirs] + handled = false + if not handled: + case cat2 + of "megatest": + runJoinedTest(r, cat, testsDir, options) + if isNimRepoTests(): + runJoinedTest(r, cat, testsDir, options & " --mm:refc") + else: + var testsRun = 0 + var files: seq[string] + for file in walkDirRec(testsDir &.? cat.string): + if isTestFile(file): files.add file + files.sort # give reproducible order + for i, name in files: + var test = makeTest(name, options, cat) + if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories: + discard "run the test" + else: + test.spec.err = reJoined + testSpec r, test + inc testsRun + if testsRun == 0: + const whiteListedDirs = ["deps", "htmldocs", "pkgs"] + # `pkgs` because bug #16556 creates `pkgs` dirs and this can affect some users + # that try an old version of choosenim. + doAssert cat.string in whiteListedDirs, + "Invalid category specified: '$#' not in whilelist: $#" % [cat.string, $whiteListedDirs] proc processPattern(r: var TResults, pattern, options: string; simulate: bool) = var testsRun = 0 diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 65104e1fe..efec04b3c 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -1,140 +1,193 @@ -template pkg1(name: string; cmd = "nimble test"; url = "", useHead = true): untyped = - packages1.add((name, cmd, url, useHead)) +##[ +## note 1 +`useHead` should ideally be used as the default but lots of packages (e.g. `chronos`) +don't have release tags (or have really old ones compared to HEAD), making it +impossible to test them reliably here. -template pkg2(name: string; cmd = "nimble test"; url = "", useHead = true): untyped = - packages2.add((name, cmd, url, useHead)) +packages listed here should ideally have regularly updated release tags, so that: +* we're testing recent versions of the package +* the version that's tested is stable enough even if HEAD may occasionally break -var packages1*: seq[tuple[name, cmd: string; url: string, useHead: bool]] = @[] -var packages2*: seq[tuple[name, cmd: string; url: string, useHead: bool]] = @[] +## note 2: D20210308T165435:here +nimble packages should be testable as follows: +git clone $url $dir && cd $dir +NIMBLE_DIR=$TMP_NIMBLE_DIR XDG_CONFIG_HOME= nimble install --depsOnly -y +NIMBLE_DIR=$TMP_NIMBLE_DIR XDG_CONFIG_HOME= nimble test -# packages A-M -# pkg1 "alea" -pkg1 "argparse" -pkg1 "arraymancer", "nim c tests/tests_cpu.nim" -# pkg1 "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim" -pkg1 "awk" -pkg1 "bigints", url = "https://github.com/Araq/nim-bigints" -pkg1 "binaryheap", "nim c -r binaryheap.nim" -pkg1 "BipBuffer" -# pkg1 "blscurve" # pending https://github.com/status-im/nim-blscurve/issues/39 -pkg1 "bncurve" -pkg1 "brainfuck", "nim c -d:release -r tests/compile.nim" -pkg1 "bump", "nim c --gc:arc -r tests/tbump.nim", "https://github.com/disruptek/bump" -pkg1 "c2nim", "nim c testsuite/tester.nim" -pkg1 "cascade" -pkg1 "cello" -pkg1 "chroma" -pkg1 "chronicles", "nim c -o:chr -r chronicles.nim" -when not defined(osx): # testdatagram.nim(560, 54): Check failed - pkg1 "chronos", "nim c -r -d:release tests/testall" -pkg1 "cligen", "nim c --path:. -r cligen.nim" -pkg1 "combparser", "nimble test --gc:orc" -pkg1 "compactdict" -pkg1 "comprehension", "nimble test", "https://github.com/alehander42/comprehension" -# pkg1 "criterion" # pending https://github.com/disruptek/criterion/issues/3 (wrongly closed) -pkg1 "dashing", "nim c tests/functional.nim" -pkg1 "delaunay" -pkg1 "docopt" -pkg1 "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" -pkg1 "elvis" -pkg1 "fidget" -pkg1 "fragments", "nim c -r fragments/dsl.nim" -pkg1 "gara" -pkg1 "glob" -# pkg1 "ggplotnim", "nim c -d:noCairo -r tests/tests.nim" # pending bug #16523 -# pkg1 "gittyup", "nimble test", "https://github.com/disruptek/gittyup" -pkg1 "gnuplot", "nim c gnuplot.nim" -# pkg1 "gram", "nim c -r --gc:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram" - # pending https://github.com/nim-lang/Nim/issues/16509 -pkg1 "hts", "nim c -o:htss src/hts.nim" -# pkg1 "httpauth" -pkg1 "illwill", "nimble examples" -pkg1 "inim" -pkg1 "itertools", "nim doc src/itertools.nim" -pkg1 "iterutils" -pkg1 "jstin" -pkg1 "karax", "nim c -r tests/tester.nim" -pkg1 "kdtree", "nimble test", "https://github.com/jblindsay/kdtree" -pkg1 "loopfusion" -pkg1 "macroutils" -pkg1 "manu" -pkg1 "markdown" -pkg1 "memo" -pkg1 "msgpack4nim", "nim c -r tests/test_spec.nim" +if this fails (e.g. nimcrypto), it could be because a package lacks a `tests/nim.cfg` with `--path:..`, +so the above commands would've worked by accident with `nimble install` but not with `nimble install --depsOnly`. +When this is the case, a workaround is to test this package here by adding `--path:$srcDir` on the test `cmd`. +]## + +type NimblePackage* = object + name*, cmd*, url*: string + useHead*: bool + allowFailure*: bool + ## When true, we still run the test but the test is allowed to fail. + ## This is useful for packages that currently fail but that we still want to + ## run in CI, e.g. so that we can monitor when they start working again and + ## are reminded about those failures without making CI fail for unrelated PRs. -# these two are special snowflakes -pkg1 "nimcrypto", "nim c -r tests/testall.nim" -pkg1 "stint", "nim c -o:stintt -r stint.nim" +var packages*: seq[NimblePackage] +proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true, allowFailure = false) = + packages.add NimblePackage(name: name, cmd: cmd, url: url, useHead: useHead, allowFailure: allowFailure) -# packages N-Z -pkg2 "nake", "nim c nakefile.nim" -pkg2 "neo", "nim c -d:blas=openblas tests/all.nim" -# pkg2 "nesm", "nimble tests" # notice plural 'tests' -# pkg2 "nico" -pkg2 "nicy", "nim c -r src/nicy.nim" -pkg2 "nigui", "nim c -o:niguii -r src/nigui.nim" -pkg2 "NimData", "nim c -o:nimdataa src/nimdata.nim" -pkg2 "nimes", "nim c src/nimes.nim" -pkg2 "nimfp", "nim c -o:nfp -r src/fp.nim" -when false: - pkg2 "nimgame2", "nim c nimgame2/nimgame.nim" - # XXX Doesn't work with deprecated 'randomize', will create a PR. -pkg2 "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" -pkg2 "nimlsp" -pkg2 "nimly", "nim c -r tests/test_readme_example.nim" -# pkg2 "nimongo", "nimble test_ci" -# pkg2 "nimph", "nimble test", "https://github.com/disruptek/nimph" -pkg2 "nimpy", "nim c -r tests/nimfrompy.nim" -pkg2 "nimquery" -pkg2 "nimsl" -pkg2 "nimsvg" -pkg2 "nimterop", "nimble minitest" -pkg2 "nimwc", "nim c nimwc.nim" -# pkg2 "nimx", "nim c --threads:on test/main.nim" -# pkg2 "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" -pkg2 "norm", "nim c -r tests/sqlite/trows.nim" -pkg2 "npeg", "nimble testarc" -pkg2 "numericalnim", "nim c -r tests/test_integrate.nim" -pkg2 "optionsutils" -pkg2 "ormin", "nim c -o:orminn ormin.nim" -pkg2 "parsetoml" -pkg2 "patty" -pkg2 "plotly", "nim c examples/all.nim" -pkg2 "pnm" -pkg2 "polypbren" -pkg2 "prologue", "nimble tcompile" -pkg2 "protobuf", "nim c -o:protobuff -r src/protobuf.nim" -pkg2 "pylib" -pkg2 "rbtree" -pkg2 "react", "nimble example" -pkg2 "regex", "nim c src/regex" -pkg2 "result", "nim c -r result.nim" -pkg2 "RollingHash", "nim c -r tests/test_cyclichash.nim" -pkg2 "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim" -pkg2 "sdl1", "nim c -r src/sdl.nim" -pkg2 "sdl2_nim", "nim c -r sdl2/sdl.nim" -pkg2 "sigv4", "nim c --gc:arc -r sigv4.nim", "https://github.com/disruptek/sigv4" -pkg2 "snip", "nimble test", "https://github.com/genotrance/snip" -pkg2 "strslice" -pkg2 "strunicode", "nim c -r src/strunicode.nim" -pkg2 "synthesis" -pkg2 "telebot", "nim c -o:tbot -r src/telebot.nim" -pkg2 "tempdir" -pkg2 "templates" -pkg2 "tensordsl", "nim c -r tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" -pkg2 "terminaltables", "nim c src/terminaltables.nim" -pkg2 "termstyle", "nim c -r termstyle.nim" -pkg2 "timeit" -pkg2 "timezones" -pkg2 "tiny_sqlite" -pkg2 "unicodedb", "nim c -d:release -r tests/tests.nim" -pkg2 "unicodeplus", "nim c -d:release -r tests/tests.nim" -pkg2 "unpack" -pkg2 "websocket", "nim c websocket.nim" -# pkg2 "winim" -pkg2 "with" -pkg2 "ws" -pkg2 "yaml", "nim build" -pkg2 "zero_functional", "nim c -r -d:nimWorkaround14447 test.nim" -pkg2 "zippy" +pkg "alea" +pkg "argparse" +pkg "arraymancer", "nim c tests/tests_cpu.nim" +pkg "ast_pattern_matching", "nim c -r tests/test1.nim" +pkg "asyncftpclient", "nimble compileExample" +pkg "asyncthreadpool", "nimble test --mm:refc" +pkg "awk" +pkg "bigints" +pkg "binaryheap", "nim c -r binaryheap.nim" +pkg "BipBuffer" +pkg "blscurve", allowFailure = true +pkg "bncurve" +pkg "brainfuck", "nim c -d:release -r tests/compile.nim" +pkg "bump", "nim c --mm:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump", allowFailure = true +pkg "c2nim", "nim c testsuite/tester.nim" +pkg "cascade" +pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true +pkg "checksums" +pkg "chroma" +pkg "chronicles", "nim c -o:chr -r chronicles.nim" +pkg "chronos", "nim c -r -d:release tests/testall" +pkg "cligen", "nim c --path:. -r cligen.nim" +pkg "combparser", "nimble test --mm:orc" +pkg "compactdict" +pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehension" +pkg "constantine", "nimble make_lib" +pkg "cowstrings" +pkg "criterion", allowFailure = true # needs testing binary +pkg "datamancer" +pkg "dashing", "nim c tests/functional.nim" +pkg "delaunay" +pkg "dnsclient", allowFailure = true # super fragile +pkg "docopt" +pkg "dotenv" +# when defined(linux): pkg "drchaos" +pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" +pkg "elvis" +pkg "faststreams" +pkg "fidget" +pkg "fragments", "nim c -r fragments/dsl.nim", allowFailure = true # pending https://github.com/nim-lang/packages/issues/2115 +pkg "fusion" +pkg "gara" +pkg "glob" +pkg "ggplotnim", "nim c -d:noCairo -r tests/tests.nim" +pkg "gittyup", "nimble test", "https://github.com/disruptek/gittyup", allowFailure = true +pkg "gnuplot", "nim c gnuplot.nim" +# pkg "gram", "nim c -r --mm:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram" + # pending https://github.com/nim-lang/Nim/issues/16509 +pkg "hts", "nim c -o:htss src/hts.nim" +pkg "httpauth" +pkg "httputils" +pkg "illwill", "nimble examples" +pkg "inim" +pkg "itertools", "nim doc src/itertools.nim" +pkg "iterutils" +pkg "json_rpc" +pkg "json_serialization" +pkg "jstin" +pkg "karax", "nim c -r tests/tester.nim" +pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree" +pkg "loopfusion" +pkg "lockfreequeues" +pkg "macroutils" +pkg "manu" +pkg "markdown" +pkg "measuremancer", "nimble testDeps; nimble -y test" +pkg "memo" +pkg "msgpack4nim", "nim c -r tests/test_spec.nim" +pkg "nake", "nim c nakefile.nim" +pkg "neo", "nim c -d:blas=openblas --mm:refc tests/all.nim" +pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true, allowFailure = true + # inactive, tests not adapted to #23096 +pkg "netty" +pkg "nico", allowFailure = true +pkg "nicy", "nim c -r src/nicy.nim" +pkg "nigui", "nim c -o:niguii -r src/nigui.nim" +pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround needed, see D20210308T165435 +pkg "NimData", "nim c -o:nimdataa src/nimdata.nim" +pkg "nimes", "nim c src/nimes.nim" +pkg "nimfp", "nim c -o:nfp -r src/fp.nim" +pkg "nimgame2", "nim c --mm:refc nimgame2/nimgame.nim" +pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" +pkg "nimib" +pkg "nimlsp" +pkg "nimly", "nim c -r tests/test_readme_example.nim" +pkg "nimongo", "nimble test_ci", allowFailure = true +pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true +pkg "nimPNG", useHead = true +pkg "nimpy", "nim c -r tests/nimfrompy.nim" +pkg "nimquery" +pkg "nimsl" +pkg "nimsvg" +pkg "nimterop", "nimble minitest", url = "https://github.com/nim-lang/nimterop" +pkg "nimwc", "nim c nimwc.nim" +pkg "nimx", "nim c test/main.nim", allowFailure = true +pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" +pkg "norm", "testament r tests/common/tmodel.nim" +pkg "normalize" +pkg "npeg", "nimble testarc" +pkg "numericalnim", "nimble nimCI" +pkg "optionsutils" +pkg "ormin", "nim c -o:orminn ormin.nim" +pkg "parsetoml" +pkg "patty" +pkg "pixie" +pkg "plotly", "nim c examples/all.nim" +pkg "pnm" +pkg "polypbren" +pkg "presto" +pkg "prologue", "nimble tcompile" +# remove fork after https://github.com/PMunch/combparser/pull/7 is merged: +pkg "protobuf", "nimble install -y https://github.com/metagn/combparser@#HEAD; nim c -o:protobuff -r src/protobuf.nim" +pkg "rbtree" +pkg "react", "nimble example" +pkg "regex", "nim c src/regex" +pkg "results", "nim c -r results.nim" +pkg "RollingHash", "nim c -r tests/test_cyclichash.nim" +pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim" +pkg "sdl1", "nim c -r src/sdl.nim" +pkg "sdl2_nim", "nim c -r sdl2/sdl.nim" +pkg "serialization" +pkg "sigv4", "nim c --mm:arc -r sigv4.nim", "https://github.com/disruptek/sigv4" +pkg "sim" +pkg "smtp", "nimble compileExample" +pkg "snip", "nimble test", "https://github.com/genotrance/snip" +pkg "ssostrings" +pkg "stew" +pkg "stint", "nim c stint.nim" +pkg "strslice" +pkg "strunicode", "nim c -r --mm:refc src/strunicode.nim" +pkg "supersnappy" +pkg "synthesis" +pkg "taskpools" +pkg "telebot", "nim c -o:tbot -r src/telebot.nim" +pkg "tempdir" +pkg "templates" +pkg "tensordsl", "nim c -r --mm:refc tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" +pkg "terminaltables", "nim c src/terminaltables.nim" +pkg "termstyle", "nim c -r termstyle.nim" +pkg "testutils" +pkg "timeit" +pkg "timezones" +pkg "tiny_sqlite" +pkg "unicodedb", "nim c -d:release -r tests/tests.nim" +pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" +pkg "union", "nim c -r tests/treadme.nim", url = "https://github.com/alaviss/union" +pkg "unittest2" +pkg "unpack" +pkg "weave", "nimble install -y cligen@#HEAD; nimble test_gc_arc", useHead = true +pkg "websock" +pkg "websocket", "nim c websocket.nim" +# pkg "winim", allowFailure = true +pkg "with" +pkg "ws", allowFailure = true +pkg "yaml" +pkg "zero_functional", "nim c -r test.nim" +pkg "zippy" +pkg "zxcvbn" diff --git a/testament/lib/stdtest/netutils.nim b/testament/lib/stdtest/netutils.nim index eb913a56a..5115390e0 100644 --- a/testament/lib/stdtest/netutils.nim +++ b/testament/lib/stdtest/netutils.nim @@ -1,6 +1,7 @@ import std/[nativesockets, asyncdispatch, os] proc bindAvailablePort*(handle: SocketHandle, port = Port(0)): Port = + ## See also `asynchttpserver.getPort`. block: var name: Sockaddr_in name.sin_family = typeof(name.sin_family)(toInt(AF_INET)) @@ -8,5 +9,5 @@ proc bindAvailablePort*(handle: SocketHandle, port = Port(0)): Port = name.sin_addr.s_addr = htonl(INADDR_ANY) if bindAddr(handle, cast[ptr SockAddr](addr(name)), sizeof(name).Socklen) < 0'i32: - raiseOSError(osLastError()) + raiseOSError(osLastError(), $port) result = getLocalAddr(handle, AF_INET)[1] diff --git a/testament/lib/stdtest/specialpaths.nim b/testament/lib/stdtest/specialpaths.nim index 42f656d76..e214d113d 100644 --- a/testament/lib/stdtest/specialpaths.nim +++ b/testament/lib/stdtest/specialpaths.nim @@ -1,6 +1,6 @@ #[ todo: move findNimStdLibCompileTime, findNimStdLib here -xxx: consider moving this to $nim/compiler/relpaths.nim to get relocatable paths +xxx: factor pending https://github.com/timotheecour/Nim/issues/616 ## note: $lib vs $nim note: these can resolve to 3 different paths if running via `nim c --lib:lib foo`, @@ -13,6 +13,8 @@ import compiler/nimpaths ]# import os +when defined(nimPreviewSlimSystem): + import std/assertions # Note: all the const paths defined here are known at compile time and valid # so long Nim repo isn't relocated after compilation. @@ -24,13 +26,30 @@ const # robust way to derive other paths here # We don't depend on PATH so this is robust to having multiple nim binaries nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir ## root of Nim repo + testsFname* = "tests" stdlibDir* = nimRootDir / "lib" systemPath* = stdlibDir / "system.nim" - testsDir* = nimRootDir / "tests" + testsDir* = nimRootDir / testsFname buildDir* = nimRootDir / "build" ## refs #10268: all testament generated files should go here to avoid ## polluting .gitignore +proc splitTestFile*(file: string): tuple[cat: string, path: string] = + ## At least one directory is required in the path, to use as a category name + runnableExamples: + doAssert splitTestFile("tests/fakedir/tfakename.nim") == ("fakedir", "tests/fakedir/tfakename.nim".unixToNativePath) + for p in file.parentDirs(inclusive = false): + let parent = p.parentDir + if parent.lastPathPart == testsFname: + result.cat = p.lastPathPart + let dir = getCurrentDir() + if file.isRelativeTo(dir): + result.path = file.relativePath(dir) + else: + result.path = file + return result + raiseAssert "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'" + static: # sanity check doAssert fileExists(systemPath) diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index 34aa3b751..a490b17c8 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -1,5 +1,9 @@ import std/private/miscdollars -import std/strutils +when defined(nimscript): + import std/os # xxx investigate why needed +else: + from std/os import getEnv +import std/[macros, genasts] template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = ## API to deal with flaky or failing tests. This avoids disabling entire tests @@ -25,12 +29,98 @@ template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = msg2.add $expr & " " & msg echo msg2 -proc greedyOrderedSubsetLines*(lhs, rhs: string): bool = - ## returns true if each stripped line in `lhs` appears in rhs, using a greedy matching. - let rhs = rhs.strip - var currentPos = 0 - for line in lhs.strip.splitLines: - currentPos = rhs.find(line.strip, currentPos) - if currentPos < 0: - return false - return true +when not defined(js) and not defined(nimscript): + import std/strutils + + proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool = + ## Returns true if each stripped line in `lhs` appears in rhs, using a greedy matching. + # xxx improve error reporting by showing the last matched pair + iterator splitLinesClosure(): string {.closure.} = + for line in splitLines(rhs.strip): + yield line + template isMatch(lhsi, rhsi): bool = + if allowPrefixMatch: + startsWith(rhsi, lhsi) + else: + lhsi == rhsi + + var rhsIter = splitLinesClosure + var currentLine = strip(rhsIter()) + + for line in lhs.strip.splitLines: + let line = line.strip + if line.len != 0: + while not isMatch(line, currentLine): + currentLine = strip(rhsIter()) + if rhsIter.finished: + return false + + if rhsIter.finished: + return false + return true + +template enableRemoteNetworking*: bool = + ## Allows contolling whether to run some test at a statement-level granularity. + ## Using environment variables simplifies propagating this all the way across + ## process calls, e.g. `testament all` calls itself, which in turns invokes + ## a `nim` invocation (possibly via additional intermediate processes). + getEnv("NIM_TESTAMENT_REMOTE_NETWORKING") == "1" + +template disableSSLTesting*: bool = + ## TODO: workaround for GitHub Action gcc 14 matrix; remove this + ## matrix and the flag after Azure agent supports ubuntu 24.04 + getEnv("NIM_TESTAMENT_DISABLE_SSL") == "1" + +template whenRuntimeJs*(bodyIf, bodyElse) = + ##[ + Behaves as `when defined(js) and not nimvm` (which isn't legal yet). + pending improvements to `nimvm`, this sugar helps; use as follows: + + whenRuntimeJs: + doAssert defined(js) + when nimvm: doAssert false + else: discard + do: + discard + ]## + when nimvm: bodyElse + else: + when defined(js): bodyIf + else: bodyElse + +template whenVMorJs*(bodyIf, bodyElse) = + ## Behaves as: `when defined(js) or nimvm` + when nimvm: bodyIf + else: + when defined(js): bodyIf + else: bodyElse + +template accept*(a) = + doAssert compiles(a) + +template reject*(a) = + doAssert not compiles(a) + +template disableVm*(body) = + when nimvm: discard + else: body + +macro assertAll*(body) = + ## works in VM, unlike `check`, `require` + runnableExamples: + assertAll: + 1+1 == 2 + var a = @[1, 2] # statements work + a.len == 2 + # remove this once these support VM, pending #10129 (closed but not yet fixed) + result = newStmtList() + for a in body: + result.add genAst(a, a2 = a.repr, info = lineInfo(a)) do: + # D20210421T014713:here + # xxx pending https://github.com/nim-lang/Nim/issues/12030, + # `typeof` should introduce its own scope, so that this + # is sufficient: `typeof(a)` instead of `typeof(block: a)` + when typeof(block: a) is void: a + else: + if not a: + raise newException(AssertionDefect, info & " " & a2) diff --git a/testament/lib/stdtest/unittest_light.nim b/testament/lib/stdtest/unittest_light.nim index bf254c11f..4ab1d7543 100644 --- a/testament/lib/stdtest/unittest_light.nim +++ b/testament/lib/stdtest/unittest_light.nim @@ -1,3 +1,6 @@ +import std/assertions + + proc mismatch*[T](lhs: T, rhs: T): string = ## Simplified version of `unittest.require` that satisfies a common use case, ## while avoiding pulling too many dependencies. On failure, diagnostic @@ -11,26 +14,24 @@ proc mismatch*[T](lhs: T, rhs: T): string = proc quoted(s: string): string = result.addQuoted s - result.add "\n" + result.add '\n' result.add "lhs:{" & replaceInvisible( $lhs) & "}\nrhs:{" & replaceInvisible($rhs) & "}\n" when compiles(lhs.len): if lhs.len != rhs.len: - result.add "lhs.len: " & $lhs.len & " rhs.len: " & $rhs.len & "\n" + result.add "lhs.len: " & $lhs.len & " rhs.len: " & $rhs.len & '\n' when compiles(lhs[0]): var i = 0 while i < lhs.len and i < rhs.len: if lhs[i] != rhs[i]: break i.inc - result.add "first mismatch index: " & $i & "\n" + result.add "first mismatch index: " & $i & '\n' if i < lhs.len and i < rhs.len: result.add "lhs[i]: {" & quoted($lhs[i]) & "}\nrhs[i]: {" & quoted( $rhs[i]) & "}\n" result.add "lhs[0..<i]:{" & replaceInvisible($lhs[ - 0..<i]) & "}" + 0..<i]) & '}' proc assertEquals*[T](lhs: T, rhs: T, msg = "") = - when false: # can be useful for debugging to see all that's fed to this. - echo "----" & $lhs - if lhs!=rhs: - doAssert false, mismatch(lhs, rhs) & "\n" & msg + if lhs != rhs: + doAssert false, mismatch(lhs, rhs) & '\n' & msg diff --git a/testament/specs.nim b/testament/specs.nim index a7f0fd4bb..c3040c1d8 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -9,6 +9,7 @@ import sequtils, parseutils, strutils, os, streams, parsecfg, tables, hashes, sets +import compiler/platform type TestamentData* = ref object # better to group globals under 1 object; could group the other ones here too @@ -34,27 +35,27 @@ type TOutputCheck* = enum ocIgnore = "ignore" - ocEqual = "equal" + ocEqual = "equal" ocSubstr = "substr" TResultEnum* = enum - reNimcCrash, # nim compiler seems to have crashed - reMsgsDiffer, # error messages differ - reFilesDiffer, # expected and given filenames differ - reLinesDiffer, # expected and given line numbers differ + reNimcCrash, # nim compiler seems to have crashed + reMsgsDiffer, # error messages differ + reFilesDiffer, # expected and given filenames differ + reLinesDiffer, # expected and given line numbers differ reOutputsDiffer, - reExitcodesDiffer, # exit codes of program or of valgrind differ + reExitcodesDiffer, # exit codes of program or of valgrind differ reTimeout, reInvalidPeg, reCodegenFailure, reCodeNotFound, reExeNotFound, - reInstallFailed # package installation failed - reBuildFailed # package building failed - reDisabled, # test is disabled - reJoined, # test is disabled because it was joined into the megatest - reSuccess # test was successful - reInvalidSpec # test had problems to parse the spec + reInstallFailed # package installation failed + reBuildFailed # package building failed + reDisabled, # test is disabled + reJoined, # test is disabled because it was joined into the megatest + reSuccess # test was successful + reInvalidSpec # test had problems to parse the spec TTarget* = enum targetC = "c" @@ -69,17 +70,17 @@ type ValgrindSpec* = enum disabled, enabled, leaking - + TSpec* = object + # xxx make sure `isJoinableSpec` takes into account each field here. action*: TTestAction file*, cmd*: string + filename*: string ## Test filename (without path). input*: string outputCheck*: TOutputCheck sortoutput*: bool output*: string line*, column*: int - tfile*: string - tline*, tcolumn*: int exitCode*: int msg*: string ccodeCheck*: seq[string] @@ -89,7 +90,8 @@ type targets*: set[TTarget] matrix*: seq[string] nimout*: string - parseErrors*: string # when the spec definition is invalid, this is not empty. + nimoutFull*: bool # whether nimout is all compiler output or a subset + parseErrors*: string # when the spec definition is invalid, this is not empty. unjoinable*: bool unbatchable*: bool # whether this test can be batchable via `NIM_TESTAMENT_BATCH`; only very @@ -97,12 +99,13 @@ type # by making the dependencies explicit useValgrind*: ValgrindSpec timeout*: float # in seconds, fractions possible, - # but don't rely on much precision + # but don't rely on much precision inlineErrors*: seq[InlineError] # line information to error message + debugInfo*: string # debug info to give more context proc getCmd*(s: TSpec): string = if s.cmd.len == 0: - result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file" + result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:build/deps/pkgs2 $options $file" else: result = s.cmd @@ -125,19 +128,55 @@ when not declared(parseCfgBool): of "n", "no", "false", "0", "off": result = false else: raise newException(ValueError, "cannot interpret as a bool: " & s) +proc addLine*(self: var string; pieces: varargs[string]) = + for piece in pieces: + self.add piece + self.add "\n" + + const - inlineErrorMarker = "#[tt." + inlineErrorKindMarker = "tt." + inlineErrorMarker = "#[" & inlineErrorKindMarker proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int = + ## Extract inline error messages. + ## + ## Can parse a single message for a line: + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name] ]# + ## ``` + ## + ## Can parse multiple messages for a line when they are separated by ';': + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error + ## ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error + ## ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name]; + ## tt.Error ^ 'no_destroy' should be: 'nodestroy' [Name]; + ## tt.Error ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` result = i + len(inlineErrorMarker) inc col, len(inlineErrorMarker) + let msgLine = line + var msgCol = -1 + var msg = "" var kind = "" - while result < s.len and s[result] in IdentChars: - kind.add s[result] - inc result - inc col - var caret = (line, -1) + template parseKind = + while result < s.len and s[result] in IdentChars: + kind.add s[result] + inc result + inc col + if kind notin ["Hint", "Warning", "Error"]: + spec.parseErrors.addLine "expected inline message kind: Hint, Warning, Error" template skipWhitespace = while result < s.len and s[result] in Whitespace: @@ -148,38 +187,78 @@ proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var T inc col inc result + template parseCaret = + if result < s.len and s[result] == '^': + msgCol = col + inc result + inc col + skipWhitespace() + else: + spec.parseErrors.addLine "expected column marker ('^') for inline message" + + template isMsgDelimiter: bool = + s[result] == ';' and + (block: + let nextTokenIdx = result + 1 + parseutils.skipWhitespace(s, result + 1) + if s.len > nextTokenIdx + len(inlineErrorKindMarker) and + s[nextTokenIdx..(nextTokenIdx + len(inlineErrorKindMarker) - 1)] == inlineErrorKindMarker: + true + else: + false) + + template trimTrailingMsgWhitespace = + while msg.len > 0 and msg[^1] in Whitespace: + setLen msg, msg.len - 1 + + template addInlineError = + doAssert msg[^1] notin Whitespace + if kind == "Error": spec.action = actionReject + spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: msgLine, col: msgCol) + + parseKind() skipWhitespace() - if result < s.len and s[result] == '^': - caret = (line-1, col) - inc result - inc col - skipWhitespace() + parseCaret() - var msg = "" while result < s.len-1: if s[result] == '\n': + if result > 0 and s[result - 1] == '\r': + msg[^1] = '\n' + else: + msg.add '\n' inc result inc line col = 1 - elif s[result] == ']' and s[result+1] == '#': - while msg.len > 0 and msg[^1] in Whitespace: - setLen msg, msg.len - 1 - + elif isMsgDelimiter(): + trimTrailingMsgWhitespace() inc result + skipWhitespace() + addInlineError() + inc result, len(inlineErrorKindMarker) + inc col, 1 + len(inlineErrorKindMarker) + kind.setLen 0 + msg.setLen 0 + parseKind() + skipWhitespace() + parseCaret() + elif s[result] == ']' and s[result+1] == '#': + trimTrailingMsgWhitespace() + inc result, 2 inc col, 2 - if kind == "Error": spec.action = actionReject - spec.unjoinable = true - spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: caret[0], col: caret[1]) + addInlineError() break else: msg.add s[result] inc result inc col + if spec.inlineErrors.len > 0: + spec.unjoinable = true + proc extractSpec(filename: string; spec: var TSpec): string = const tripleQuote = "\"\"\"" - var s = readFile(filename).string + specStart = "discard " & tripleQuote + var s = readFile(filename) var i = 0 var a = -1 @@ -187,30 +266,36 @@ proc extractSpec(filename: string; spec: var TSpec): string = var line = 1 var col = 1 while i < s.len: - if s.continuesWith(tripleQuote, i): - if a < 0: a = i - elif b < 0: b = i - inc i, 2 - inc col + if (i == 0 or s[i-1] != ' ') and s.continuesWith(specStart, i): + # `s[i-1] == '\n'` would not work because of `tests/stdlib/tbase64.nim` which contains BOM (https://en.wikipedia.org/wiki/Byte_order_mark) + const lineMax = 10 + if a != -1: + raise newException(ValueError, "testament spec violation: duplicate `specStart` found: " & $(filename, a, b, line)) + elif line > lineMax: + # not overly restrictive, but prevents mistaking some `specStart` as spec if deeep inside a test file + raise newException(ValueError, "testament spec violation: `specStart` should be before line $1, or be indented; info: $2" % [$lineMax, $(filename, a, b, line)]) + i += specStart.len + a = i + elif a > -1 and b == -1 and s.continuesWith(tripleQuote, i): + b = i + i += tripleQuote.len elif s[i] == '\n': inc line + inc i col = 1 elif s.continuesWith(inlineErrorMarker, i): i = extractErrorMsg(s, i, line, col, spec) else: inc col - inc i + inc i - # look for """ only in the first section - if a >= 0 and b > a and a < 40: - result = s.substr(a+3, b-1).replace("'''", tripleQuote) + if a >= 0 and b > a: + result = s.substr(a, b-1).multiReplace({"'''": tripleQuote, "\\31": "\31"}) + elif a >= 0: + raise newException(ValueError, "testament spec violation: `specStart` found but not trailing `tripleQuote`: $1" % $(filename, a, b, line)) else: - #echo "warning: file does not contain spec: " & filename result = "" -when not defined(nimhygiene): - {.pragma: inject.} - proc parseTargets*(value: string): set[TTarget] = for v in value.normalize.splitWhitespace: case v @@ -220,19 +305,10 @@ proc parseTargets*(value: string): set[TTarget] = of "js": result.incl(targetJS) else: raise newException(ValueError, "invalid target: '$#'" % v) -proc addLine*(self: var string; a: string) = - self.add a - self.add "\n" - -proc addLine*(self: var string; a,b: string) = - self.add a - self.add b - self.add "\n" - proc initSpec*(filename: string): TSpec = result.file = filename -proc isCurrentBatch(testamentData: TestamentData, filename: string): bool = +proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool = if testamentData.testamentNumBatch != 0: hash(filename) mod testamentData.testamentNumBatch == testamentData.testamentBatch else: @@ -240,11 +316,13 @@ proc isCurrentBatch(testamentData: TestamentData, filename: string): bool = proc parseSpec*(filename: string): TSpec = result.file = filename + result.filename = extractFilename(filename) let specStr = extractSpec(filename, result) var ss = newStringStream(specStr) var p: CfgParser open(p, ss, filename, 1) var flags: HashSet[string] + var nimoutFound = false while true: var e = next(p) case e.kind @@ -279,12 +357,6 @@ proc parseSpec*(filename: string): TSpec = if result.msg.len == 0 and result.nimout.len == 0: result.parseErrors.addLine "errormsg or msg needs to be specified before column" discard parseInt(e.value, result.column) - of "tfile": - result.tfile = e.value - of "tline": - discard parseInt(e.value, result.tline) - of "tcolumn": - discard parseInt(e.value, result.tcolumn) of "output": if result.outputCheck != ocSubstr: result.outputCheck = ocEqual @@ -296,21 +368,20 @@ proc parseSpec*(filename: string): TSpec = result.output = strip(e.value) of "sortoutput": try: - result.sortoutput = parseCfgBool(e.value) + result.sortoutput = parseCfgBool(e.value) except: result.parseErrors.addLine getCurrentExceptionMsg() of "exitcode": discard parseInt(e.value, result.exitCode) result.action = actionRun - of "msg": - result.msg = e.value - if result.action != actionRun: - result.action = actionCompile of "errormsg": result.msg = e.value result.action = actionReject of "nimout": result.nimout = e.value + nimoutFound = true + of "nimoutfull": + result.nimoutFull = parseCfgBool(e.value) of "batchable": result.unbatchable = not parseCfgBool(e.value) of "joinable": @@ -327,42 +398,71 @@ proc parseSpec*(filename: string): TSpec = # Valgrind only supports OSX <= 17.x result.useValgrind = disabled of "disabled": - case e.value.normalize + let value = e.value.normalize + case value of "y", "yes", "true", "1", "on": result.err = reDisabled of "n", "no", "false", "0", "off": discard - of "win", "windows": + # These values are defined in `compiler/options.isDefined` + of "win": when defined(windows): result.err = reDisabled of "linux": when defined(linux): result.err = reDisabled of "bsd": when defined(bsd): result.err = reDisabled - of "macosx": - when defined(macosx): result.err = reDisabled - of "unix": - when defined(unix): result.err = reDisabled - of "posix": + of "osx": + when defined(osx): result.err = reDisabled + of "unix", "posix": when defined(posix): result.err = reDisabled - of "travis": + of "freebsd": + when defined(freebsd): result.err = reDisabled + of "littleendian": + when defined(littleendian): result.err = reDisabled + of "bigendian": + when defined(bigendian): result.err = reDisabled + of "cpu8", "8bit": + when defined(cpu8): result.err = reDisabled + of "cpu16", "16bit": + when defined(cpu16): result.err = reDisabled + of "cpu32", "32bit": + when defined(cpu32): result.err = reDisabled + of "cpu64", "64bit": + when defined(cpu64): result.err = reDisabled + # These values are for CI environments + of "travis": # deprecated if isTravis: result.err = reDisabled - of "appveyor": + of "appveyor": # deprecated if isAppVeyor: result.err = reDisabled of "azure": if isAzure: result.err = reDisabled - of "32bit": - if sizeof(int) == 4: - result.err = reDisabled - of "freebsd": - when defined(freebsd): result.err = reDisabled - of "arm64": - when defined(arm64): result.err = reDisabled - of "i386": - when defined(i386): result.err = reDisabled - of "openbsd": - when defined(openbsd): result.err = reDisabled - of "netbsd": - when defined(netbsd): result.err = reDisabled else: - result.parseErrors.addLine "cannot interpret as a bool: ", e.value + # Check whether the value exists as an OS or CPU that is + # defined in `compiler/platform`. + block checkHost: + for os in platform.OS: + # Check if the value exists as OS. + if value == os.name.normalize: + # The value exists; is it the same as the current host? + if value == hostOS.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + for cpu in platform.CPU: + # Check if the value exists as CPU. + if value == cpu.name.normalize: + # The value exists; is it the same as the current host? + if value == hostCPU.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + # The value doesn't exist as an OS, CPU, or any previous value + # defined in this case statement, so raise an error. + result.parseErrors.addLine "cannot interpret as a bool: ", e.value of "cmd": if e.value.startsWith("nim "): result.cmd = compilerPrefix & e.value[3..^1] @@ -401,6 +501,21 @@ proc parseSpec*(filename: string): TSpec = if skips.anyIt(it in result.file): result.err = reDisabled + if nimoutFound and result.nimout.len == 0 and not result.nimoutFull: + result.parseErrors.addLine "empty `nimout` is vacuously true, use `nimoutFull:true` if intentional" + result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable if not result.inCurrentBatch: result.err = reDisabled + + # Interpolate variables in msgs: + template varSub(msg: string): string = + try: + msg % ["/", $DirSep, "file", result.filename] + except ValueError: + result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unit-tests-output-message-variable-interpolation')" + msg + result.nimout = result.nimout.varSub + result.msg = result.msg.varSub + for inlineError in result.inlineErrors.mitems: + inlineError.msg = inlineError.msg.varSub diff --git a/testament/testament.nim b/testament/testament.nim index 7b1a25bf0..1e892e636 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -10,74 +10,99 @@ ## This program verifies Nim against the testcases. import - strutils, pegs, os, osproc, streams, json, - backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, times, md5, sequtils, azure, intsets + std/[strutils, pegs, os, osproc, streams, json, + parseopt, browsers, terminal, exitprocs, + algorithm, times, intsets, macros] + +import backend, specs, azure, htmlgen + from std/sugar import dup import compiler/nodejs import lib/stdtest/testutils +from lib/stdtest/specialpaths import splitTestFile +from std/private/gitutils import diffStrings + +import ../dist/checksums/src/checksums/md5 + +proc trimUnitSep(x: var string) = + let L = x.len + if L > 0 and x[^1] == '\31': + setLen x, L-1 var useColors = true var backendLogging = true var simulate = false +var optVerbose = false +var useMegatest = true +var valgrindEnabled = true + +proc verboseCmd(cmd: string) = + if optVerbose: + echo "executing: ", cmd const failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs testsDir = "tests" & DirSep resultsFile = "testresults.html" - #jsonFile = "testresults.json" # not used Usage = """Usage: testament [options] command [arguments] Command: p|pat|pattern <glob> run all the tests matching the given pattern - all run all tests + all run all tests in category folders c|cat|category <category> run all the tests of a certain category r|run <test> run single test file html generate $1 from the database - stats generate statistics about test cases Arguments: arguments are passed to the compiler Options: - --print also print results to the console + --print print results to the console + --verbose print commands (compiling and running tests) --simulate see what tests would be run but don't run them (for debugging) --failing only show failing/ignored tests - --targets:"c cpp js objc" run tests for specified targets (default: all) + --targets:"c cpp js objc" run tests for specified targets (default: c) --nim:path use a particular nim executable (default: $$PATH/nim) --directory:dir Change to directory dir before reading the tests or doing anything else. --colors:on|off Turn messages coloring on|off. --backendLogging:on|off Disable or enable backend logging. By default turned on. --megatest:on|off Enable or disable megatest. Default is on. + --valgrind:on|off Enable or disable valgrind support. Default is on. --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN. + +Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables +tests with remote networking (as in CI). """ % resultsFile +proc isNimRepoTests(): bool = + # this logic could either be specific to cwd, or to some file derived from + # the input file, eg testament r /pathto/tests/foo/tmain.nim; we choose + # the former since it's simpler and also works with `testament all`. + let file = "testament"/"testament.nim.cfg" + result = file.fileExists + type Category = distinct string TResults = object - total, passed, skipped: int + total, passed, failedButAllowed, skipped: int + ## xxx rename passed to passedOrAllowedFailure data: string TTest = object name: string cat: Category options: string - args: seq[string] + testArgs: seq[string] spec: TSpec startTime: float + debugInfo: string # ---------------------------------------------------------------------------- let pegLineError = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" - - pegLineTemplate = - peg""" - {[^(]*} '(' {\d+} ', ' {\d+} ') ' - 'template/generic instantiation' ( ' of `' [^`]+ '`' )? ' from here' .* - """ pegOtherError = peg"'Error:' \s* {.*}" pegOfInterest = pegLineError / pegOtherError @@ -86,14 +111,13 @@ var targetsSet = false proc isSuccess(input: string): bool = # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs + # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we + # could handle colors, there would be other issues such as handling other flags + # that may appear in user config (eg: `--filenames`). + # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config + # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory input.startsWith("Hint: ") and input.endsWith("[SuccessX]") -proc normalizeMsg(s: string): string = - result = newStringOfCap(s.len+1) - for x in splitLines(s): - if result.len > 0: result.add '\L' - result.add x.strip - proc getFileDir(filename: string): string = result = filename.splitFile().dir if not result.isAbsolute(): @@ -101,7 +125,7 @@ proc getFileDir(filename: string): string = proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: string = ""): tuple[ cmdLine: string, - output: TaintedString, + output: string, exitCode: int] {.tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = @@ -109,8 +133,9 @@ proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: str for arg in args: result.cmdLine.add ' ' result.cmdLine.add quoteShell(arg) - var p = startProcess(command, workingDir=workingDir, args=args, - options={poStdErrToStdOut, poUsePath}) + verboseCmd(result.cmdLine) + var p = startProcess(command, workingDir = workingDir, args = args, + options = {poStdErrToStdOut, poUsePath}) var outp = outputStream(p) # There is no way to provide input for the child process @@ -120,12 +145,12 @@ proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: str instream.write(input) close instream - result.exitCode = -1 - var line = newStringOfCap(120).TaintedString + result.exitCode = -1 + var line = newStringOfCap(120) while true: if outp.readLine(line): - result.output.string.add(line.string) - result.output.string.add("\n") + result.output.add line + result.output.add '\n' else: result.exitCode = peekExitCode(p) if result.exitCode != -1: break @@ -136,56 +161,64 @@ proc nimcacheDir(filename, options: string, target: TTarget): string = let hashInput = options & $target result = "nimcache" / (filename & '_' & hashInput.getMD5) -proc prepareTestArgs(cmdTemplate, filename, options, nimcache: string, - target: TTarget, extraOptions = ""): seq[string] = - var options = target.defaultOptions & " " & options - # improve pending https://github.com/nim-lang/Nim/issues/14343 - if nimcache.len > 0: options.add " " & ("--nimCache:" & nimcache).quoteShell - options.add " " & extraOptions - result = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], +proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): string = + var options = target.defaultOptions & ' ' & options + if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell) + options.add ' ' & extraOptions + # we avoid using `parseCmdLine` which is buggy, refs bug #14343 + result = cmdTemplate % ["target", targetToCmd[target], "options", options, "file", filename.quoteShell, - "filedir", filename.getFileDir(), "nim", compilerPrefix]) + "filedir", filename.getFileDir(), "nim", compilerPrefix] -proc callCompiler(cmdTemplate, filename, options, nimcache: string, - target: TTarget, extraOptions = ""): TSpec = - let c = prepareTestArgs(cmdTemplate, filename, options, nimcache, target, +proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): TSpec = + result.cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache, target, extraOptions) - result.cmd = quoteShellCommand(c) - var p = startProcess(command=c[0], args=c[1 .. ^1], - options={poStdErrToStdOut, poUsePath}) + verboseCmd(result.cmd) + var p = startProcess(command = result.cmd, + options = {poStdErrToStdOut, poUsePath, poEvalCommand}) let outp = p.outputStream - var suc = "" + var foundSuccessMsg = false + var foundErrorMsg = false var err = "" - var tmpl = "" var x = newStringOfCap(120) result.nimout = "" while true: - if outp.readLine(x.TaintedString): - result.nimout.add(x & "\n") + if outp.readLine(x): + trimUnitSep x + result.nimout.add(x & '\n') if x =~ pegOfInterest: - # `err` should contain the last error/warning message + # `err` should contain the last error message err = x - elif x =~ pegLineTemplate and err == "": - # `tmpl` contains the last template expansion before the error - tmpl = x + foundErrorMsg = true elif x.isSuccess: - suc = x + foundSuccessMsg = true elif not running(p): break - close(p) result.msg = "" result.file = "" result.output = "" result.line = 0 result.column = 0 - result.tfile = "" - result.tline = 0 - result.tcolumn = 0 + result.err = reNimcCrash - if tmpl =~ pegLineTemplate: - result.tfile = extractFilename(matches[0]) - result.tline = parseInt(matches[1]) - result.tcolumn = parseInt(matches[2]) + result.exitCode = p.peekExitCode + close p + case result.exitCode + of 0: + if foundErrorMsg: + result.debugInfo.add " compiler exit code was 0 but some Error's were found." + else: + result.err = reSuccess + of 1: + if not foundErrorMsg: + result.debugInfo.add " compiler exit code was 1 but no Error's were found." + if foundSuccessMsg: + result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true." + else: + result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode + if err =~ pegLineError: result.file = extractFilename(matches[0]) result.line = parseInt(matches[1]) @@ -193,38 +226,15 @@ proc callCompiler(cmdTemplate, filename, options, nimcache: string, result.msg = matches[3] elif err =~ pegOtherError: result.msg = matches[0] - elif suc.isSuccess: - result.err = reSuccess - -proc callCCompiler(cmdTemplate, filename, options: string, - target: TTarget): TSpec = - let c = prepareTestArgs(cmdTemplate, filename, options, nimcache = "", target) - var p = startProcess(command="gcc", args=c[5 .. ^1], - options={poStdErrToStdOut, poUsePath}) - let outp = p.outputStream - var x = newStringOfCap(120) - result.nimout = "" - result.msg = "" - result.file = "" - result.output = "" - result.line = -1 - while true: - if outp.readLine(x.TaintedString): - result.nimout.add(x & "\n") - elif not running(p): - break - close(p) - if p.peekExitCode == 0: - result.err = reSuccess + trimUnitSep result.msg proc initResults: TResults = result.total = 0 result.passed = 0 + result.failedButAllowed = 0 result.skipped = 0 result.data = "" -import macros - macro ignoreStyleEcho(args: varargs[typed]): untyped = let typForegroundColor = bindSym"ForegroundColor".getType let typBackgroundColor = bindSym"BackgroundColor".getType @@ -249,18 +259,29 @@ template maybeStyledEcho(args: varargs[untyped]): untyped = proc `$`(x: TResults): string = - result = ("Tests passed: $1 / $3 <br />\n" & - "Tests skipped: $2 / $3 <br />\n") % - [$x.passed, $x.skipped, $x.total] + result = """ +Tests passed or allowed to fail: $2 / $1 <br /> +Tests failed and allowed to fail: $3 / $1 <br /> +Tests skipped: $4 / $1 <br /> +""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] + +proc testName(test: TTest, target: TTarget, extraOptions: string, allowFailure: bool): string = + var name = test.name.replace(DirSep, '/') + name.add ' ' & $target + if allowFailure: + name.add " (allowed to fail) " + if test.options.len > 0: name.add ' ' & test.options + if extraOptions.len > 0: name.add ' ' & extraOptions + name.strip() proc addResult(r: var TResults, test: TTest, target: TTarget, - expected, given: string, successOrig: TResultEnum) = + extraOptions, expected, given: string, successOrig: TResultEnum, + allowFailure = false, givenSpec: ptr TSpec = nil) = + # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need + # instead of having to pass individual fields, or abusing existing ones like expected vs given. # test.name is easier to find than test.name.extractFilename # A bit hacky but simple and works with tests/testament/tshould_not_work.nim - var name = test.name.replace(DirSep, '/') - name.add " " & $target - if test.options.len > 0: name.add " " & test.options - + let name = testName(test, target, extraOptions, allowFailure) let duration = epochTime() - test.startTime let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout else: successOrig @@ -275,18 +296,22 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, expected = expected, given = given) r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) + template dispNonSkipped(color, outcome) = + maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)" template disp(msg) = - maybeStyledEcho styleDim, fgYellow, msg & " ", styleBright, fgCyan, name + maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name if success == reSuccess: - maybeStyledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)" + dispNonSkipped(fgGreen, "PASS: ") elif success == reDisabled: if test.spec.inCurrentBatch: disp("SKIP:") else: disp("NOTINBATCH:") elif success == reJoined: disp("JOINED:") else: - maybeStyledEcho styleBright, fgRed, failString, fgCyan, name + dispNonSkipped(fgRed, failString) maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" maybeStyledEcho styleBright, fgRed, "Failure: ", $success + if givenSpec != nil and givenSpec.debugInfo.len > 0: + echo "debugInfo: " & givenSpec.debugInfo if success in {reBuildFailed, reNimcCrash, reInstallFailed}: # expected is empty, no reason to print it. echo given @@ -295,7 +320,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, maybeStyledEcho styleBright, expected, "\n" maybeStyledEcho fgYellow, "Gotten:" maybeStyledEcho styleBright, given, "\n" - + echo diffStrings(expected, given).output if backendLogging and (isAppVeyor or isAzure): let (outcome, msg) = @@ -305,86 +330,66 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, of reDisabled, reJoined: ("Skipped", "") of reBuildFailed, reNimcCrash, reInstallFailed: - ("Failed", "Failure: " & $success & "\n" & given) + ("Failed", "Failure: " & $success & '\n' & given) else: ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given) if isAzure: azure.addTestResult(name, test.cat.string, int(duration * 1000), msg, success) else: - var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options, + var p = startProcess("appveyor", args = ["AddTest", test.name.replace("\\", "/") & test.options, "-Framework", "nim-testament", "-FileName", test.cat.string, "-Outcome", outcome, "-ErrorMessage", msg, - "-Duration", $(duration*1000).int], - options={poStdErrToStdOut, poUsePath, poParentStreams}) + "-Duration", $(duration * 1000).int], + options = {poStdErrToStdOut, poUsePath, poParentStreams}) discard waitForExit(p) close(p) -proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = - let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}" - var covered = initIntSet() - for line in splitLines(given.nimout): - - if line =~ pegLine: - let file = extractFilename(matches[0]) - let line = try: parseInt(matches[1]) except: -1 - let col = try: parseInt(matches[2]) except: -1 - let kind = matches[3] - let msg = matches[4] - - if file == extractFilename test.name: - var i = 0 - for x in expected.inlineErrors: - if x.line == line and (x.col == col or x.col < 0) and - x.kind == kind and x.msg in msg: - covered.incl i - inc i - - block coverCheck: - for j in 0..high(expected.inlineErrors): - if j notin covered: - var e = test.name - e.add "(" - e.addInt expected.inlineErrors[j].line - if expected.inlineErrors[j].col > 0: - e.add ", " - e.addInt expected.inlineErrors[j].col - e.add ") " - e.add expected.inlineErrors[j].kind - e.add ": " - e.add expected.inlineErrors[j].msg - - r.addResult(test, target, e, given.nimout, reMsgsDiffer) - break coverCheck - - r.addResult(test, target, "", given.msg, reSuccess) - inc(r.passed) +proc toString(inlineError: InlineError, filename: string): string = + result.add "$file($line, $col) $kind: $msg" % [ + "file", filename, + "line", $inlineError.line, + "col", $inlineError.col, + "kind", $inlineError.kind, + "msg", $inlineError.msg + ] + +proc inlineErrorsMsgs(expected: TSpec): string = + for inlineError in expected.inlineErrors.items: + result.addLine inlineError.toString(expected.filename) + +proc checkForInlineErrors(expected, given: TSpec): bool = + for inlineError in expected.inlineErrors: + if inlineError.toString(expected.filename) notin given.nimout: + return false + true + +proc nimoutCheck(expected, given: TSpec): bool = + result = true + if expected.nimoutFull: + if expected.nimout != given.nimout: + result = false + elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): + result = false -proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = - if expected.inlineErrors.len > 0: - checkForInlineErrors(r, expected, given, test, target) +proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, + target: TTarget, extraOptions: string) = + if not checkForInlineErrors(expected, given) or + (not expected.nimoutFull and not nimoutCheck(expected, given)): + r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer) elif strip(expected.msg) notin strip(given.msg): - r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer) - elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): - r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer) - elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and + r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer) + elif not nimoutCheck(expected, given): + r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer) + elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: - r.addResult(test, target, expected.file, given.file, reFilesDiffer) + r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer) elif expected.line != given.line and expected.line != 0 or expected.column != given.column and expected.column != 0: - r.addResult(test, target, $expected.line & ':' & $expected.column, - $given.line & ':' & $given.column, - reLinesDiffer) - elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and - "internal error:" notin expected.msg: - r.addResult(test, target, expected.tfile, given.tfile, reFilesDiffer) - elif expected.tline != given.tline and expected.tline != 0 or - expected.tcolumn != given.tcolumn and expected.tcolumn != 0: - r.addResult(test, target, $expected.tline & ':' & $expected.tcolumn, - $given.tline & ':' & $given.tcolumn, - reLinesDiffer) + r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column, + $given.line & ':' & $given.column, reLinesDiffer) else: - r.addResult(test, target, expected.msg, given.msg, reSuccess) + r.addResult(test, target, extraOptions, expected.msg, given.msg, reSuccess) inc(r.passed) proc generatedFile(test: TTest, target: TTarget): string = @@ -402,7 +407,7 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st given: var TSpec) = try: let genFile = generatedFile(test, target) - let contents = readFile(genFile).string + let contents = readFile(genFile) for check in spec.ccodeCheck: if check.len > 0 and check[0] == '\\': # little hack to get 'match' support: @@ -422,43 +427,30 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st given.err = reCodeNotFound echo getCurrentExceptionMsg() -proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) = - if not greedyOrderedSubsetLines(expectedNimout, given.nimout): - given.err = reMsgsDiffer - -proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, - expected: TSpec; r: var TResults) = +proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string, + given: var TSpec, expected: TSpec; r: var TResults) = var expectedmsg: string = "" var givenmsg: string = "" if given.err == reSuccess: if expected.needsCodegenCheck: codegenCheck(test, target, expected, expectedmsg, given) givenmsg = given.msg - if expected.nimout.len > 0: - expectedmsg = expected.nimout + if not nimoutCheck(expected, given) or + not checkForInlineErrors(expected, given): + given.err = reMsgsDiffer + expectedmsg = expected.nimout & inlineErrorsMsgs(expected) givenmsg = given.nimout.strip - nimoutCheck(test, expectedmsg, given) else: - givenmsg = "$ " & given.cmd & "\n" & given.nimout + givenmsg = "$ " & given.cmd & '\n' & given.nimout if given.err == reSuccess: inc(r.passed) - r.addResult(test, target, expectedmsg, givenmsg, given.err) + r.addResult(test, target, extraOptions, expectedmsg, givenmsg, given.err) proc getTestSpecTarget(): TTarget = - if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true": + if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": result = targetCpp else: result = targetC -proc checkDisabled(r: var TResults, test: TTest): bool = - if test.spec.err in {reDisabled, reJoined}: - # targetC is a lie, but parameter is required - r.addResult(test, targetC, "", "", test.spec.err) - inc(r.skipped) - inc(r.total) - result = false - else: - result = true - var count = 0 proc equalModuloLastNewline(a, b: string): bool = @@ -466,38 +458,43 @@ proc equalModuloLastNewline(a, b: string): bool = result = a == b or b.endsWith("\n") and a == b[0 ..< ^1] proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, - target: TTarget, nimcache: string, extraOptions = "") = + target: TTarget, extraOptions: string, nimcache: string) = test.startTime = epochTime() + if testName(test, target, extraOptions, false) in skips: + test.spec.err = reDisabled + + if test.spec.err in {reDisabled, reJoined}: + r.addResult(test, target, extraOptions, "", "", test.spec.err) + inc(r.skipped) + return + var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions) case expected.action of actionCompile: - var given = callCompiler(expected.getCmd, test.name, test.options, nimcache, target, - extraOptions = " --stdout --hint[Path]:off --hint[Processing]:off") - compilerOutputTests(test, target, given, expected, r) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionRun: - var given = callCompiler(expected.getCmd, test.name, test.options, - nimcache, target, extraOptions) if given.err != reSuccess: - r.addResult(test, target, "", "$ " & given.cmd & "\n" & given.nimout, given.err) + r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr) else: let isJsTarget = target == targetJS var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt) if not fileExists(exeFile): - r.addResult(test, target, expected.output, + r.addResult(test, target, extraOptions, expected.output, "executable not found: " & exeFile, reExeNotFound) else: let nodejs = if isJsTarget: findNodeJs() else: "" if isJsTarget and nodejs == "": - r.addResult(test, target, expected.output, "nodejs binary not in PATH", + r.addResult(test, target, extraOptions, expected.output, "nodejs binary not in PATH", reExeNotFound) else: var exeCmd: string - var args = test.args + var args = test.testArgs if isJsTarget: exeCmd = nodejs - args = concat(@[exeFile], args) + # see D20210217T215950 + args = @["--unhandled-rejections=strict", exeFile] & args else: exeCmd = exeFile.dup(normalizeExe) - if expected.useValgrind != disabled: + if valgrindEnabled and expected.useValgrind != disabled: var valgrindOptions = @["--error-exitcode=1"] if expected.useValgrind != leaking: valgrindOptions.add "--leak-check=yes" @@ -509,33 +506,53 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, if exitCode != 0: exitCode = 1 let bufB = if expected.sortoutput: - var buf2 = buf.string + var buf2 = buf buf2.stripLineEnd var x = splitLines(buf2) sort(x, system.cmp) - join(x, "\n") & "\n" + join(x, "\n") & '\n' else: - buf.string + buf if exitCode != expected.exitCode: - r.addResult(test, target, "exitcode: " & $expected.exitCode, + given.err = reExitcodesDiffer + r.addResult(test, target, extraOptions, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode & "\n\nOutput:\n" & bufB, reExitcodesDiffer) elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or (expected.outputCheck == ocSubstr and expected.output notin bufB): given.err = reOutputsDiffer - r.addResult(test, target, expected.output, bufB, reOutputsDiffer) - else: - compilerOutputTests(test, target, given, expected, r) + r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionReject: - var given = callCompiler(expected.getCmd, test.name, test.options, - nimcache, target) - cmpMsgs(r, expected, given, test, target) + # Make sure its the compiler rejecting and not the system (e.g. segfault) + cmpMsgs(r, expected, given, test, target, extraOptions) + if given.exitCode != QuitFailure: + r.addResult(test, target, extraOptions, "exitcode: " & $QuitFailure, + "exitcode: " & $given.exitCode & "\n\nOutput:\n" & + given.nimout, reExitcodesDiffer) + + -proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions = "") = +proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget = + result = defaultTarget + var p = parseopt.initOptParser(extraOptions) + + while true: + parseopt.next(p) + case p.kind + of cmdEnd: break + of cmdLongOption, cmdShortOption: + if p.key == "b" or p.key == "backend": + result = parseEnum[TTarget](p.val.normalize) + # chooses the last one + else: + discard + +proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) = for target in expected.targets: inc(r.total) if target notin gTargets: - r.addResult(test, target, "", "", reDisabled) + r.addResult(test, target, extraOptions, "", "", reDisabled) inc(r.skipped) elif simulate: inc count @@ -543,16 +560,16 @@ proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions = else: let nimcache = nimcacheDir(test.name, test.options, target) var testClone = test - testSpecHelper(r, testClone, expected, target, nimcache, extraOptions) + let target = changeTarget(extraOptions, target) + testSpecHelper(r, testClone, expected, target, extraOptions, nimcache) proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = var expected = test.spec if expected.parseErrors.len > 0: # targetC is a lie, but a parameter is required - r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec) + r.addResult(test, targetC, "", "", expected.parseErrors, reInvalidSpec) inc(r.total) return - if not checkDisabled(r, test): return expected.targets.incl targets # still no target specified at all @@ -562,46 +579,13 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = for m in test.spec.matrix: targetHelper(r, test, expected, m) else: - targetHelper(r, test, expected) + targetHelper(r, test, expected, "") -proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) = - if not checkDisabled(r, test): return +proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} = for target in test.spec.targets: inc(r.total) var testClone = test - testSpecHelper(r, testClone, test.spec, target, nimcache) - -proc testC(r: var TResults, test: TTest, action: TTestAction) = - # runs C code. Doesn't support any specs, just goes by exit code. - if not checkDisabled(r, test): return - - let tname = test.name.addFileExt(".c") - inc(r.total) - maybeStyledEcho "Processing ", fgCyan, extractFilename(tname) - var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC) - if given.err != reSuccess: - r.addResult(test, targetC, "", given.msg, given.err) - elif action == actionRun: - let exeFile = changeFileExt(test.name, ExeExt) - var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath}) - if exitCode != 0: given.err = reExitcodesDiffer - if given.err == reSuccess: inc(r.passed) - -proc testExec(r: var TResults, test: TTest) = - # runs executable or script, just goes by exit code - if not checkDisabled(r, test): return - - inc(r.total) - let (outp, errC) = execCmdEx(test.options.strip()) - var given: TSpec - if errC == 0: - given.err = reSuccess - else: - given.err = reExitcodesDiffer - given.msg = outp.string - - if given.err == reSuccess: inc(r.passed) - r.addResult(test, targetC, "", given.msg, given.err) + testSpecHelper(r, testClone, test.spec, target, "", nimcache) proc makeTest(test, options: string, cat: Category): TTest = result.cat = cat @@ -610,19 +594,17 @@ proc makeTest(test, options: string, cat: Category): TTest = result.spec = parseSpec(addFileExt(test, ".nim")) result.startTime = epochTime() -proc makeRawTest(test, options: string, cat: Category): TTest = +proc makeRawTest(test, options: string, cat: Category): TTest {.used.} = result.cat = cat result.name = test result.options = options result.spec = initSpec(addFileExt(test, ".nim")) - result.startTime = epochTime() result.spec.action = actionCompile result.spec.targets = {getTestSpecTarget()} + result.startTime = epochTime() # TODO: fix these files const disabledFilesDefault = @[ - "LockFreeHash.nim", - "sharedstrings.nim", "tableimpl.nim", "setimpl.nim", "hashcommon.nim", @@ -657,7 +639,7 @@ proc loadSkipFrom(name: string): seq[string] = # used by `nlvm` (at least) for line in lines(name): let sline = line.strip() - if sline.len > 0 and not sline.startsWith("#"): + if sline.len > 0 and not sline.startsWith('#'): result.add sline proc main() = @@ -668,25 +650,25 @@ proc main() = var targetsStr = "" var isMainProcess = true var skipFrom = "" - var useMegatest = true var p = initOptParser() p.next() while p.kind in {cmdLongOption, cmdShortOption}: - case p.key.string.normalize - of "print", "verbose": optPrintResults = true + case p.key.normalize + of "print": optPrintResults = true + of "verbose": optVerbose = true of "failing": optFailing = true - of "pedantic": discard "now always enabled" + of "pedantic": discard # deadcode refs https://github.com/nim-lang/Nim/issues/16731 of "targets": - targetsStr = p.val.string + targetsStr = p.val gTargets = parseTargets(targetsStr) targetsSet = true of "nim": - compilerPrefix = addFileExt(p.val.string.absolutePath, ExeExt) + compilerPrefix = addFileExt(p.val.absolutePath, ExeExt) of "directory": - setCurrentDir(p.val.string) + setCurrentDir(p.val) of "colors": - case p.val.string: + case p.val: of "on": useColors = true of "off": @@ -695,7 +677,7 @@ proc main() = quit Usage of "batch": testamentData0.batchArg = p.val - if p.val != "_": + if p.val != "_" and p.val.len > 0 and p.val[0] in {'0'..'9'}: let s = p.val.split("_") doAssert s.len == 2, $(p.val, s) testamentData0.testamentBatch = s[0].parseInt @@ -705,15 +687,23 @@ proc main() = of "simulate": simulate = true of "megatest": - case p.val.string: + case p.val: of "on": useMegatest = true of "off": useMegatest = false else: quit Usage + of "valgrind": + case p.val: + of "on": + valgrindEnabled = true + of "off": + valgrindEnabled = false + else: + quit Usage of "backendlogging": - case p.val.string: + case p.val: of "on": backendLogging = true of "off": @@ -721,18 +711,18 @@ proc main() = else: quit Usage of "skipfrom": - skipFrom = p.val.string + skipFrom = p.val else: quit Usage p.next() if p.kind != cmdArgument: quit Usage - var action = p.key.string.normalize + var action = p.key.normalize p.next() var r = initResults() case action of "all": - #processCategory(r, Category"megatest", p.cmdLineRest.string, testsDir, runJoinableTests = false) + #processCategory(r, Category"megatest", p.cmdLineRest, testsDir, runJoinableTests = false) var myself = quoteShell(getAppFilename()) if targetsStr.len > 0: @@ -746,13 +736,14 @@ proc main() = myself &= " " & quoteShell("--skipFrom:" & skipFrom) var cats: seq[string] - let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: "" + let rest = if p.cmdLineRest.len > 0: " " & p.cmdLineRest else: "" for kind, dir in walkDir(testsDir): assert testsDir.startsWith(testsDir) let cat = dir[testsDir.len .. ^1] if kind == pcDir and cat notin ["testdata", "nimcache"]: cats.add cat - cats.add AdditionalCategories + if isNimRepoTests(): + cats.add AdditionalCategories if useMegatest: cats.add MegaTestCat var cmds: seq[string] @@ -767,14 +758,14 @@ proc main() = skips = loadSkipFrom(skipFrom) for i, cati in cats: progressStatus(i) - processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, runJoinableTests = false) + processCategory(r, Category(cati), p.cmdLineRest, testsDir, runJoinableTests = false) else: - addQuitProc azure.finalize + addExitProc azure.finalize quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus) of "c", "cat", "category": skips = loadSkipFrom(skipFrom) var cat = Category(p.key) - processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true) + processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = true) of "pcat": skips = loadSkipFrom(skipFrom) # 'pcat' is used for running a category in parallel. Currently the only @@ -783,23 +774,15 @@ proc main() = isMainProcess = false var cat = Category(p.key) p.next - processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = false) + processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = false) of "p", "pat", "pattern": skips = loadSkipFrom(skipFrom) let pattern = p.key p.next - processPattern(r, pattern, p.cmdLineRest.string, simulate) + processPattern(r, pattern, p.cmdLineRest, simulate) of "r", "run": - # "/pathto/tests/stdlib/nre/captures.nim" -> "stdlib" + "tests/stdlib/nre/captures.nim" - var subPath = p.key.string - let nimRoot = currentSourcePath / "../.." - # makes sure points to this regardless of cwd or which nim is used to compile this. - doAssert existsDir(nimRoot/testsDir) # sanity check - if subPath.isAbsolute: subPath = subPath.relativePath(nimRoot) - # at least one directory is required in the path, to use as a category name - let pathParts = subPath.relativePath(testsDir).split({DirSep, AltSep}) - let cat = Category(pathParts[0]) - processSingleTest(r, cat, p.cmdLineRest.string, subPath, gTargets, targetsSet) + let (cat, path) = splitTestFile(p.key) + processSingleTest(r, cat.Category, p.cmdLineRest, path, gTargets, targetsSet) of "html": generateHtml(resultsFile, optFailing) else: diff --git a/testament/testament.nim.cfg b/testament/testament.nim.cfg index f1fcd95f3..c97284129 100644 --- a/testament/testament.nim.cfg +++ b/testament/testament.nim.cfg @@ -1,3 +1,5 @@ +# don't move this file without updating the logic in `isNimRepoTests` + path = "$nim" # For compiler/nodejs -d:ssl # For azure # my SSL doesn't have this feature and I don't care: diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim index a8d216a5b..477da1e23 100644 --- a/testament/tests/shouldfail/tccodecheck.nim +++ b/testament/tests/shouldfail/tccodecheck.nim @@ -1,5 +1,5 @@ discard """ -ccodecheck: "baz" + ccodecheck: "baz" """ proc foo(): void {.exportc: "bar".}= diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim index 89482e673..809ddec74 100644 --- a/testament/tests/shouldfail/tcolumn.nim +++ b/testament/tests/shouldfail/tcolumn.nim @@ -1,7 +1,7 @@ discard """ -errormsg: "undeclared identifier: 'undeclared'" -line: 8 -column: 7 + errormsg: "undeclared identifier: 'undeclared'" + line: 9 + column: 7 """ # test should fail because the line directive is wrong diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim index dbbdf5021..6c130d107 100644 --- a/testament/tests/shouldfail/terrormsg.nim +++ b/testament/tests/shouldfail/terrormsg.nim @@ -1,7 +1,7 @@ discard """ -errormsg: "wrong error message" -line: 8 -column: 6 + errormsg: "wrong error message" + line: 9 + column: 6 """ # test should fail because the line directive is wrong diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim index 1b38b4f2e..605f046db 100644 --- a/testament/tests/shouldfail/texitcode1.nim +++ b/testament/tests/shouldfail/texitcode1.nim @@ -1,3 +1,3 @@ discard """ -exitcode: 1 + exitcode: 1 """ diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim index 20d4bd1f3..b40a4f44f 100644 --- a/testament/tests/shouldfail/tfile.nim +++ b/testament/tests/shouldfail/tfile.nim @@ -1,6 +1,6 @@ discard """ -errormsg: "undeclared identifier: 'undefined'" -file: "notthisfile.nim" + errormsg: "undeclared identifier: 'undefined'" + file: "notthisfile.nim" """ echo undefined diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim index f7a09875c..fe782eb03 100644 --- a/testament/tests/shouldfail/tline.nim +++ b/testament/tests/shouldfail/tline.nim @@ -1,7 +1,7 @@ discard """ -errormsg: "undeclared identifier: 'undeclared'" -line: 9 -column: 6 + errormsg: "undeclared identifier: 'undeclared'" + line: 10 + column: 6 """ # test should fail because the line directive is wrong diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim index 9879e4181..92022ee97 100644 --- a/testament/tests/shouldfail/tmaxcodesize.nim +++ b/testament/tests/shouldfail/tmaxcodesize.nim @@ -1,5 +1,5 @@ discard """ -maxcodesize: 1 + maxcodesize: 1 """ echo "Hello World" diff --git a/testament/tests/shouldfail/tmsg.nim b/testament/tests/shouldfail/tmsg.nim deleted file mode 100644 index 4ad17fa95..000000000 --- a/testament/tests/shouldfail/tmsg.nim +++ /dev/null @@ -1,6 +0,0 @@ -discard """ -msg: "Hello World" -""" - -static: - echo "something else" diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim index c0e332053..0a65bfb70 100644 --- a/testament/tests/shouldfail/tnimout.nim +++ b/testament/tests/shouldfail/tnimout.nim @@ -1,6 +1,6 @@ discard """ -nimout: "Hello World!" -action: compile + nimout: "Hello World!" + action: compile """ static: diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim new file mode 100644 index 000000000..4fc93f6d2 --- /dev/null +++ b/testament/tests/shouldfail/tnimoutfull.nim @@ -0,0 +1,14 @@ +discard """ + nimout: ''' +msg1 +msg2 +''' + action: compile + nimoutFull: true +""" + +# should fail because `msg3` is not in nimout and `nimoutFill: true` was given +static: + echo "msg1" + echo "msg2" + echo "msg3" diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim index ac0bc7a46..eaf9e8652 100644 --- a/testament/tests/shouldfail/toutput.nim +++ b/testament/tests/shouldfail/toutput.nim @@ -1,7 +1,7 @@ discard """ -output: ''' -done -''' + output: ''' + done + ''' """ echo "broken" diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim index 7cc51ee8d..47324ecee 100644 --- a/testament/tests/shouldfail/toutputsub.nim +++ b/testament/tests/shouldfail/toutputsub.nim @@ -1,5 +1,5 @@ discard """ -outputsub: "something else" + outputsub: "something else" """ echo "Hello World!" diff --git a/testament/tests/shouldfail/treject.nim b/testament/tests/shouldfail/treject.nim index aaf2b4a63..1e7258f70 100644 --- a/testament/tests/shouldfail/treject.nim +++ b/testament/tests/shouldfail/treject.nim @@ -1,5 +1,5 @@ discard """ -action: "reject" + action: "reject" """ # Because we set action="reject", we expect this line not to compile. But the diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim index 4ce9ce26d..69dfbc0a0 100644 --- a/testament/tests/shouldfail/tsortoutput.nim +++ b/testament/tests/shouldfail/tsortoutput.nim @@ -1,6 +1,6 @@ discard """ -sortoutput: true -output: ''' + sortoutput: true + output: ''' 2 1 ''' diff --git a/testament/tests/shouldfail/tvalgrind.nim b/testament/tests/shouldfail/tvalgrind.nim index 4f699fd3b..d551ff12e 100644 --- a/testament/tests/shouldfail/tvalgrind.nim +++ b/testament/tests/shouldfail/tvalgrind.nim @@ -1,6 +1,6 @@ discard """ -valgrind: true -cmd: "nim $target --gc:arc -d:useMalloc $options $file" + valgrind: true + cmd: "nim $target --gc:arc -d:useMalloc $options $file" """ # this is the same check used by testament/specs.nim whether or not valgrind |