diff options
Diffstat (limited to 'testament/categories.nim')
| -rw-r--r-- | testament/categories.nim | 649 |
1 files changed, 338 insertions, 311 deletions
diff --git a/testament/categories.nim b/testament/categories.nim index f8cc625c4..843bef3f9 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -10,9 +10,11 @@ ## Include for the tester that contains test suites that test special features ## of the compiler. -# included from tester.nim +# included from testament.nim import important_packages +import std/[strformat, strutils] +from std/sequtils import filterIt const specialCategories = [ @@ -21,26 +23,21 @@ const "debugger", "dll", "examples", - "flags", "gc", "io", "js", "ic", "lib", - "longgc", "manyloc", "nimble-packages", "niminaction", - "rodfiles", "threads", - "untestable", - "stdlib", + "untestable", # see trunner_special "testdata", "nimcache", "coroutines", "osproc", "shouldfail", - "dir with space", "destructor" ] @@ -48,72 +45,9 @@ proc isTestFile*(file: string): bool = let (_, name, ext) = splitFile(file) result = ext == ".nim" and name.startsWith("t") -# ---------------- IC tests --------------------------------------------- - -proc icTests(r: var TResults; testsDir: string, cat: Category, options: string) = - const - tooltests = ["compiler/nim.nim", "tools/nimgrep.nim"] - writeOnly = " --incremental:writeonly " - readOnly = " --incremental:readonly " - incrementalOn = " --incremental:on " - - template test(x: untyped) = - testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) - - template editedTest(x: untyped) = - var test = makeTest(file, x & options, cat) - test.spec.targets = {getTestSpecTarget()} - testSpecWithNimcache(r, test, nimcache) - - const tempExt = "_temp.nim" - for it in walkDirRec(testsDir / "ic"): - if isTestFile(it) and not it.endsWith(tempExt): - let nimcache = nimcacheDir(it, options, getTestSpecTarget()) - removeDir(nimcache) - - let content = readFile(it) - for fragment in content.split("#!EDIT!#"): - let file = it.replace(".nim", tempExt) - writeFile(file, fragment) - let oldPassed = r.passed - editedTest incrementalOn - if r.passed != oldPassed+1: break - - for file in tooltests: - let nimcache = nimcacheDir(file, options, getTestSpecTarget()) - removeDir(nimcache) - - let oldPassed = r.passed - test writeOnly - - if r.passed == oldPassed+1: - test readOnly - if r.passed == oldPassed+2: - test readOnly & "-d:nimBackendAssumesChange " - -# --------------------- flags tests ------------------------------------------- - -proc flagTests(r: var TResults, cat: Category, options: string) = - # --genscript - const filename = testsDir/"flags"/"tgenscript" - const genopts = " --genscript" - let nimcache = nimcacheDir(filename, genopts, targetC) - testSpec r, makeTest(filename, genopts, cat) - - when defined(windows): - testExec r, makeTest(filename, " cmd /c cd " & nimcache & - " && compile_tgenscript.bat", cat) - - elif defined(posix): - testExec r, makeTest(filename, " sh -c \"cd " & nimcache & - " && sh compile_tgenscript.sh\"", cat) - - # Run - testExec r, makeTest(filename, " " & nimcache / "tgenscript", cat) - # --------------------- DLL generation tests ---------------------------------- -proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = +proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = false) = const rpath = when defined(macosx): " --passL:-rpath --passL:@loader_path" else: @@ -125,42 +59,53 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) test2.spec.action = actionCompile testSpec c, test2 - var test3 = makeTest("lib/nimhcr.nim", options & " --outdir:tests/dll" & rpath, cat) + + var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) test3.spec.action = actionCompile testSpec c, test3 - var test4 = makeTest("tests/dll/visibility.nim", options & " --app:lib" & rpath, cat) + var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat) test4.spec.action = actionCompile testSpec c, test4 # windows looks in the dir of the exe (yay!): - when not defined(Windows): + when not defined(windows): # posix relies on crappy LD_LIBRARY_PATH (ugh!): const libpathenv = when defined(haiku): "LIBRARY_PATH" else: "LD_LIBRARY_PATH" - var libpath = getEnv(libpathenv).string + var libpath = getEnv(libpathenv) # Temporarily add the lib directory to LD_LIBRARY_PATH: putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: "")) defer: putEnv(libpathenv, libpath) testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) - testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & rpath, cat) - testSpec r, makeTest("tests/dll/visibility.nim", options & rpath, cat) + testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) + testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) if "boehm" notin options: + # hcr tests + + var basicHcrTest = makeTest("tests/dll/nimhcr_basic.nim", options & " --threads:off --forceBuild --hotCodeReloading:on " & rpath, cat) + # test segfaults for now but compiles: + if isOrc: basicHcrTest.spec.action = actionCompile + testSpec r, basicHcrTest + # force build required - see the comments in the .nim file for more details var hcri = makeTest("tests/dll/nimhcr_integration.nim", - options & " --forceBuild --hotCodeReloading:on" & rpath, cat) + options & " --threads:off --forceBuild --hotCodeReloading:on" & rpath, cat) let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget()) - hcri.args = prepareTestArgs(hcri.spec.getCmd, hcri.name, + let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name, hcri.options, nimcache, getTestSpecTarget()) + hcri.testArgs = cmd.parseCmdLine testSpec r, hcri proc dllTests(r: var TResults, cat: Category, options: string) = # dummy compile result: var c = initResults() - runBasicDLLTest c, r, cat, options - runBasicDLLTest c, r, cat, options & " -d:release" + runBasicDLLTest c, r, cat, options & " --mm:refc" + runBasicDLLTest c, r, cat, options & " -d:release --mm:refc" + runBasicDLLTest c, r, cat, options, isOrc = true + runBasicDLLTest c, r, cat, options & " -d:release", isOrc = true when not defined(windows): # still cannot find a recent Windows version of boehm.dll: runBasicDLLTest c, r, cat, options & " --gc:boehm" @@ -169,18 +114,10 @@ proc dllTests(r: var TResults, cat: Category, options: string) = # ------------------------------ GC tests ------------------------------------- proc gcTests(r: var TResults, cat: Category, options: string) = - template testWithNone(filename: untyped) = - testSpec r, makeTest("tests/gc" / filename, options & - " --gc:none", cat) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release --gc:none", cat) - template testWithoutMs(filename: untyped) = - testSpec r, makeTest("tests/gc" / filename, options, cat) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release", cat) + testSpec r, makeTest("tests/gc" / filename, options & "--mm:refc", cat) testSpec r, makeTest("tests/gc" / filename, options & - " -d:release -d:useRealtimeGC", cat) + " -d:release -d:useRealtimeGC --mm:refc", cat) when filename != "gctest": testSpec r, makeTest("tests/gc" / filename, options & " --gc:orc", cat) @@ -193,6 +130,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) = " --gc:markAndSweep", cat) testSpec r, makeTest("tests/gc" / filename, options & " -d:release --gc:markAndSweep", cat) + template test(filename: untyped) = testWithoutBoehm filename when not defined(windows) and not defined(android): @@ -210,7 +148,6 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "gcleak" test "gcleak2" testWithoutBoehm "gctest" - testWithNone "gctest" test "gcleak3" test "gcleak4" # Disabled because it works and takes too long to run: @@ -225,19 +162,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "stackrefleak" test "cyclecollector" - -proc longGCTests(r: var TResults, cat: Category, options: string) = - when defined(windows): - let cOptions = "-ldl -DWIN" - else: - let cOptions = "-ldl" - - var c = initResults() - # According to ioTests, this should compile the file - testSpec c, makeTest("tests/realtimeGC/shared", options, cat) - # ^- why is this not appended to r? Should this be discarded? - testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun - testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat) + testWithoutBoehm "trace_globals" # ------------------------- threading tests ----------------------------------- @@ -256,6 +181,11 @@ proc ioTests(r: var TResults, cat: Category, options: string) = # dummy compile result: var c = initResults() testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat) + # ^- why is this not appended to r? Should this be discarded? + # EDIT: this should be replaced by something like in D20210524T180826, + # likewise in similar instances where `testSpec c` is used, or more generally + # when a test depends on another test, as it makes tests non-independent, + # creating complications for batching and megatest logic. testSpec r, makeTest("tests/system/tio", options, cat) # ------------------------- async tests --------------------------------------- @@ -268,25 +198,29 @@ proc asyncTests(r: var TResults, cat: Category, options: string) = # ------------------------- debugger tests ------------------------------------ proc debuggerTests(r: var TResults, cat: Category, options: string) = - var t = makeTest("tools/nimgrep", options & " --debugger:on", cat) - t.spec.action = actionCompile - testSpec r, t + if fileExists("tools/nimgrep.nim"): + var t = makeTest("tools/nimgrep", options & " --debugger:on", cat) + t.spec.action = actionCompile + # force target to C because of MacOS 10.15 SDK headers bug + # https://github.com/nim-lang/Nim/pull/15612#issuecomment-712471879 + t.spec.targets = {targetC} + testSpec r, t # ------------------------- JS tests ------------------------------------------ proc jsTests(r: var TResults, cat: Category, options: string) = template test(filename: untyped) = - testSpec r, makeTest(filename, options & " -d:nodejs", cat), {targetJS} - testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat), {targetJS} + testSpec r, makeTest(filename, options, cat), {targetJS} + testSpec r, makeTest(filename, options & " -d:release", cat), {targetJS} for t in os.walkFiles("tests/js/t*.nim"): test(t) for testfile in ["exception/texceptions", "exception/texcpt1", "exception/texcsub", "exception/tfinally", "exception/tfinally2", "exception/tfinally3", - "actiontable/tactiontable", "method/tmultimjs", + "collections/tactiontable", "method/tmultimjs", "varres/tvarres0", "varres/tvarres3", "varres/tvarres4", - "varres/tvartup", "misc/tints", "misc/tunsignedinc", + "varres/tvartup", "int/tints", "int/tunsignedinc", "async/tjsandnativeasync"]: test "tests/" & testfile & ".nim" @@ -296,8 +230,6 @@ proc jsTests(r: var TResults, cat: Category, options: string) = # ------------------------- nim in action ----------- proc testNimInAction(r: var TResults, cat: Category, options: string) = - let options = options & " --nilseqs:on" - template test(filename: untyped) = testSpec r, makeTest(filename, options, cat) @@ -388,7 +320,7 @@ proc findMainFile(dir: string): string = var nimFiles = 0 for kind, file in os.walkDir(dir): if kind == pcFile: - if file.endsWith(cfgExt): return file[.. ^(cfgExt.len+1)] & ".nim" + if file.endsWith(cfgExt): return file[0..^(cfgExt.len+1)] & ".nim" elif file.endsWith(".nim"): if result.len == 0: result = file inc nimFiles @@ -418,7 +350,7 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = proc isValid(file: string): bool = for dir in parentDirs(file, inclusive = false): if dir.lastPathPart in ["includes", "nimcache"]: - # eg: lib/pure/includes/osenv.nim gives: Error: This is an include file for os.nim! + # e.g.: lib/pure/includes/osenv.nim gives: Error: This is an include file for os.nim! return false let name = extractFilename(file) if name.splitFile.ext != ".nim": return false @@ -433,12 +365,12 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = files.sort # reproducible order for testFile in files: - let contents = readFile(testFile).string + let contents = readFile(testFile) var testObj = makeTest(testFile, options, cat) #[ todo: this logic is fragile: - false positives (if appears in a comment), or false negatives, eg + false positives (if appears in a comment), or false negatives, e.g. `when defined(osx) and isMainModule`. Instead of fixing this, see https://github.com/nim-lang/Nim/issues/10045 for a much better way. @@ -448,114 +380,183 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = testSpec r, testObj # ----------------------------- nimble ---------------------------------------- - -var nimbleDir = getEnv("NIMBLE_DIR").string -if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" -let - nimbleExe = findExe("nimble") - packageIndex = nimbleDir / "packages_official.json" - -iterator listPackages(): tuple[name, url, cmd: string, hasDeps: bool] = - let defaultCmd = "nimble test" +proc listPackagesAll(): seq[NimblePackage] = + var nimbleDir = getEnv("NIMBLE_DIR") + if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" + let packageIndex = nimbleDir / "packages_official.json" let packageList = parseFile(packageIndex) - for n, cmd, hasDeps, url in important_packages.packages.items: - let cmd = if cmd.len == 0: defaultCmd else: cmd - if url.len != 0: - yield (n, url, cmd, hasDeps) + proc findPackage(name: string): JsonNode = + for a in packageList: + if a["name"].str == name: return a + for pkg in important_packages.packages.items: + var pkg = pkg + if pkg.url.len == 0: + let pkg2 = findPackage(pkg.name) + if pkg2 == nil: + raise newException(ValueError, "Cannot find package '$#'." % pkg.name) + pkg.url = pkg2["url"].str + result.add pkg + +proc listPackages(packageFilter: string): seq[NimblePackage] = + let pkgs = listPackagesAll() + if packageFilter.len != 0: + # xxx document `packageFilter`, seems like a bad API, + # at least should be a regex; a substring match makes no sense. + result = pkgs.filterIt(packageFilter in it.name) + else: + if testamentData0.batchArg == "allowed_failures": + result = pkgs.filterIt(it.allowFailure) + elif testamentData0.testamentNumBatch == 0: + result = pkgs else: - var found = false - for package in packageList.items: - let name = package["name"].str - if name == n: - found = true - let pUrl = package["url"].str - yield (name, pUrl, cmd, hasDeps) - break - if not found: - raise newException(ValueError, "Cannot find package '$#'." % n) - -proc makeSupTest(test, options: string, cat: Category): TTest = + let pkgs2 = pkgs.filterIt(not it.allowFailure) + for i in 0..<pkgs2.len: + if i mod testamentData0.testamentNumBatch == testamentData0.testamentBatch: + result.add pkgs2[i] + +proc makeSupTest(test, options: string, cat: Category, debugInfo = ""): TTest = result.cat = cat result.name = test result.options = options + result.debugInfo = debugInfo result.startTime = epochTime() -proc testNimblePackages(r: var TResults, cat: Category) = - if nimbleExe == "": - echo "[Warning] - Cannot run nimble tests: Nimble binary not found." - return - if execCmd("$# update" % nimbleExe) == QuitFailure: - echo "[Warning] - Cannot run nimble tests: Nimble update failed." - return +import std/private/gitutils +proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = + let nimbleExe = findExe("nimble") + doAssert nimbleExe != "", "Cannot run nimble tests: Nimble binary not found." + doAssert execCmd("$# update" % nimbleExe) == 0, "Cannot run nimble tests: Nimble update failed." let packageFileTest = makeSupTest("PackageFileParsed", "", cat) let packagesDir = "pkgstemp" + createDir(packagesDir) var errors = 0 try: - for name, url, cmd, hasDep in listPackages(): + let pkgs = listPackages(packageFilter) + for i, pkg in pkgs: inc r.total - var test = makeSupTest(url, "", cat) - let buildPath = packagesDir / name - if not existsDir(buildPath): - if hasDep: - let installName = if url.len != 0: url else: name - let (nimbleCmdLine, nimbleOutput, nimbleStatus) = execCmdEx2("nimble", ["install", "-y", installName]) - if nimbleStatus != QuitSuccess: - let message = "nimble install failed:\n$ " & nimbleCmdLine & "\n" & nimbleOutput - r.addResult(test, targetC, "", message, reInstallFailed) - continue - - let (installCmdLine, installOutput, installStatus) = execCmdEx2("git", ["clone", url, buildPath]) - if installStatus != QuitSuccess: - let message = "git clone failed:\n$ " & installCmdLine & "\n" & installOutput - r.addResult(test, targetC, "", message, reInstallFailed) + var test = makeSupTest(pkg.name, "", cat, "[$#/$#] " % [$i, $pkgs.len]) + let buildPath = packagesDir / pkg.name + template tryCommand(cmd: string, workingDir2 = buildPath, reFailed = reInstallFailed, maxRetries = 1): string = + var outp: string + let ok = retryCall(maxRetry = maxRetries, backoffDuration = 10.0): + var status: int + (outp, status) = execCmdEx(cmd, workingDir = workingDir2) + status == QuitSuccess + if not ok: + if pkg.allowFailure: + inc r.passed + inc r.failedButAllowed + addResult(r, test, targetC, "", "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure) continue - - let cmdArgs = parseCmdLine(cmd) - - let (buildCmdLine, buildOutput, buildStatus) = execCmdEx2(cmdArgs[0], cmdArgs[1..^1], workingDir=buildPath) - if buildStatus != QuitSuccess: - let message = "package test failed\n$ " & buildCmdLine & "\n" & buildOutput - r.addResult(test, targetC, "", message, reBuildFailed) - else: - inc r.passed - r.addResult(test, targetC, "", "", reSuccess) + outp + + if not dirExists(buildPath): + discard tryCommand("git clone $# $#" % [pkg.url.quoteShell, buildPath.quoteShell], workingDir2 = ".", maxRetries = 3) + if not pkg.useHead: + discard tryCommand("git fetch --tags", maxRetries = 3) + let describeOutput = tryCommand("git describe --tags --abbrev=0") + discard tryCommand("git checkout $#" % [describeOutput.strip.quoteShell]) + discard tryCommand("nimble install --depsOnly -y", maxRetries = 3) + let cmds = pkg.cmd.split(';') + for i in 0 ..< cmds.len - 1: + discard tryCommand(cmds[i], maxRetries = 3) + discard tryCommand(cmds[^1], reFailed = reBuildFailed) + inc r.passed + r.addResult(test, targetC, "", "", "", reSuccess, allowFailure = pkg.allowFailure) errors = r.total - r.passed if errors == 0: - r.addResult(packageFileTest, targetC, "", "", reSuccess) + r.addResult(packageFileTest, targetC, "", "", "", reSuccess) else: - r.addResult(packageFileTest, targetC, "", "", reBuildFailed) + r.addResult(packageFileTest, targetC, "", "", "", reBuildFailed) except JsonParsingError: - echo "[Warning] - Cannot run nimble tests: Invalid package file." - r.addResult(packageFileTest, targetC, "", "Invalid package file", reBuildFailed) + errors = 1 + r.addResult(packageFileTest, targetC, "", "", "Invalid package file", reBuildFailed) + raise except ValueError: - echo "[Warning] - $#" % getCurrentExceptionMsg() - r.addResult(packageFileTest, targetC, "", "Unknown package", reBuildFailed) + errors = 1 + r.addResult(packageFileTest, targetC, "", "", "Unknown package", reBuildFailed) + raise # bug #18805 finally: if errors == 0: removeDir(packagesDir) +# ---------------- IC tests --------------------------------------------- + +proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; + isNavigatorTest: bool) = + const + tooltests = ["compiler/nim.nim"] + writeOnly = " --incremental:writeonly " + readOnly = " --incremental:readonly " + incrementalOn = " --incremental:on -d:nimIcIntegrityChecks " + navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint:Conf:off --warnings:off " + + template test(x: untyped) = + testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) + + template editedTest(x: untyped) = + var test = makeTest(file, x & options, cat) + if isNavigatorTest: + test.spec.action = actionCompile + test.spec.targets = {getTestSpecTarget()} + testSpecWithNimcache(r, test, nimcache) + + template checkTest() = + var test = makeRawTest(file, options, cat) + test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:on $options " & file + testSpecWithNimcache(r, test, nimcache) + + if not isNavigatorTest: + for file in tooltests: + let nimcache = nimcacheDir(file, options, getTestSpecTarget()) + removeDir(nimcache) + + let oldPassed = r.passed + checkTest() + + if r.passed == oldPassed+1: + checkTest() + if r.passed == oldPassed+2: + checkTest() + + const tempExt = "_temp.nim" + for it in walkDirRec(testsDir): + # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test + if isTestFile(it) and not it.endsWith(tempExt): + let nimcache = nimcacheDir(it, options, getTestSpecTarget()) + removeDir(nimcache) + + let content = readFile(it) + for fragment in content.split("#!EDIT!#"): + let file = it.replace(".nim", tempExt) + writeFile(file, fragment) + let oldPassed = r.passed + editedTest(if isNavigatorTest: navTestConfig else: incrementalOn) + if r.passed != oldPassed+1: break # ---------------------------------------------------------------------------- -const AdditionalCategories = ["debugger", "examples", "lib", "ic"] +const AdditionalCategories = ["debugger", "examples", "lib", "ic", "navigator"] const MegaTestCat = "megatest" proc `&.?`(a, b: string): string = # candidate for the stdlib? result = if b.startsWith(a): b else: a & b -proc processSingleTest(r: var TResults, cat: Category, options, test: string) = - let test = testsDir &.? cat.string / test - let target = if cat.string.normalize == "js": targetJS else: targetC - if existsFile(test): - testSpec r, makeTest(test, options, cat), {target} - else: - echo "[Warning] - ", test, " test does not exist" +proc processSingleTest(r: var TResults, cat: Category, options, test: string, targets: set[TTarget], targetsSet: bool) = + var targets = targets + if not targetsSet: + let target = if cat.string.normalize == "js": targetJS else: targetC + targets = {target} + doAssert fileExists(test), test & " test does not exist" + testSpec r, makeTest(test, options, cat), targets proc isJoinableSpec(spec: TSpec): bool = - result = not spec.sortoutput and + # xxx simplify implementation using a whitelist of fields that are allowed to be + # set to non-default values (use `fieldPairs`), to avoid issues like bug #16576. + result = useMegatest and not spec.sortoutput and spec.action == actionRun and not fileExists(spec.file.changeFileExt("cfg")) and not fileExists(spec.file.changeFileExt("nims")) and @@ -567,36 +568,48 @@ proc isJoinableSpec(spec: TSpec): bool = spec.exitCode == 0 and spec.input.len == 0 and spec.nimout.len == 0 and + spec.nimoutFull == false and + # so that tests can have `nimoutFull: true` with `nimout.len == 0` with + # the meaning that they expect empty output. + spec.matrix.len == 0 and spec.outputCheck != ocSubstr and spec.ccodeCheck.len == 0 and (spec.targets == {} or spec.targets == {targetC}) - -proc norm(s: var string) = - while true: - let tmp = s.replace("\n\n", "\n") - if tmp == s: break - s = tmp - s = s.strip + if result: + if spec.file.readFile.contains "when isMainModule": + result = false proc quoted(a: string): string = # todo: consider moving to system.nim result.addQuoted(a) -proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = +proc runJoinedTest(r: var TResults, cat: Category, testsDir: string, options: string) = ## returns a list of tests that have problems + #[ + xxx create a reusable megatest API after abstracting out testament specific code, + refs https://github.com/timotheecour/Nim/issues/655 + and https://github.com/nim-lang/gtk2/pull/28; it's useful in other contexts. + ]# var specs: seq[TSpec] = @[] for kind, dir in walkDir(testsDir): - assert testsDir.startsWith(testsDir) + assert dir.startsWith(testsDir) let cat = dir[testsDir.len .. ^1] if kind == pcDir and cat notin specialCategories: for file in walkDirRec(testsDir / cat): if isTestFile(file): - let spec = parseSpec(file) + var spec: TSpec + try: + spec = parseSpec(file) + except ValueError: + # e.g. for `tests/navigator/tincludefile.nim` which have multiple + # specs; this will be handled elsewhere + echo "parseSpec raised ValueError for: '$1', assuming this will be handled outside of megatest" % file + continue if isJoinableSpec(spec): specs.add spec - proc cmp(a: TSpec, b:TSpec): auto = cmp(a.file, b.file) - sort(specs, cmp=cmp) # reproducible order + proc cmp(a: TSpec, b: TSpec): auto = cmp(a.file, b.file) + sort(specs, cmp = cmp) # reproducible order echo "joinable specs: ", specs.len if simulate: @@ -606,142 +619,156 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = return var megatest: string - #[ - TODO(minor): - get from Nim cmd - put outputGotten.txt, outputGotten.txt, megatest.nim there too - delete upon completion, maybe - ]# + # xxx (minor) put outputExceptedFile, outputGottenFile, megatestFile under here or `buildDir` var outDir = nimcacheDir(testsDir / "megatest", "", targetC) - const marker = "megatest:processing: " - + template toMarker(file, i): string = + "megatest:processing: [$1] $2" % [$i, file] for i, runSpec in specs: let file = runSpec.file - let file2 = outDir / ("megatest_" & $i & ".nim") + let file2 = outDir / ("megatest_a_$1.nim" % $i) # `include` didn't work with `trecmod2.nim`, so using `import` - let code = "echo \"" & marker & "\", " & quoted(file) & "\n" + let code = "echo $1\nstatic: echo \"CT:\", $1\n" % [toMarker(file, i).quoted] createDir(file2.parentDir) writeFile(file2, code) - megatest.add "import " & quoted(file2) & "\n" - megatest.add "import " & quoted(file) & "\n" + megatest.add "import $1\nimport $2 as megatest_b_$3\n" % [file2.quoted, file.quoted, $i] - writeFile("megatest.nim", megatest) + let megatestFile = testsDir / "megatest.nim" # so it uses testsDir / "config.nims" + writeFile(megatestFile, megatest) - let args = ["c", "--nimCache:" & outDir, "-d:testing", "--listCmd", - "--listFullPaths:off", "--excessiveStackTrace:off", "megatest.nim"] + let root = getCurrentDir() + var args = @["c", "--nimCache:" & outDir, "-d:testing", "-d:nimMegatest", "--listCmd", + "--path:" & root] + args.add options.parseCmdLine + args.add megatestFile var (cmdLine, buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, input = "") if exitCode != 0: - echo "$ ", cmdLine - echo buf.string - quit("megatest compilation failed") + echo "$ " & cmdLine & "\n" & buf + quit(failString & "megatest compilation failed") - (buf, exitCode) = execCmdEx("./megatest") + (buf, exitCode) = execCmdEx(megatestFile.changeFileExt(ExeExt).dup normalizeExe) if exitCode != 0: - echo buf.string - quit("megatest execution failed") + echo buf + quit(failString & "megatest execution failed") - norm buf.string - writeFile("outputGotten.txt", buf.string) + const outputExceptedFile = "outputExpected.txt" + const outputGottenFile = "outputGotten.txt" + writeFile(outputGottenFile, buf) var outputExpected = "" for i, runSpec in specs: - outputExpected.add marker & runSpec.file & "\n" - outputExpected.add runSpec.output.strip - outputExpected.add '\n' - norm outputExpected - - if buf.string != outputExpected: - writeFile("outputExpected.txt", outputExpected) - discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt") - echo "output different!" - # outputGotten.txt, outputExpected.txt not removed on purpose for debugging. + outputExpected.add toMarker(runSpec.file, i) & "\n" + if runSpec.output.len > 0: + outputExpected.add runSpec.output + if not runSpec.output.endsWith "\n": + outputExpected.add '\n' + + if buf != outputExpected: + writeFile(outputExceptedFile, outputExpected) + echo diffFiles(outputGottenFile, outputExceptedFile).output + echo failString & "megatest output different, see $1 vs $2" % [outputGottenFile, outputExceptedFile] + # outputGottenFile, outputExceptedFile not removed on purpose for debugging. quit 1 else: - echo "output OK" - removeFile("outputGotten.txt") - removeFile("megatest.nim") - #testSpec r, makeTest("megatest", options, cat) + echo "megatest output OK" + # --------------------------------------------------------------------------- proc processCategory(r: var TResults, cat: Category, options, testsDir: string, runJoinableTests: bool) = - case cat.string.normalize - of "rodfiles": - when false: - compileRodFiles(r, cat, options) - runRodFiles(r, cat, options) - of "ic": - when false: - icTests(r, testsDir, cat, options) - of "js": - # only run the JS tests on Windows or Linux because Travis is bad - # and other OSes like Haiku might lack nodejs: - if not defined(linux) and isTravis: + let cat2 = cat.string.normalize + var handled = false + if isNimRepoTests(): + handled = true + case cat2 + of "js": + # only run the JS tests on Windows or Linux because Travis is bad + # and other OSes like Haiku might lack nodejs: + if not defined(linux) and isTravis: + discard + else: + jsTests(r, cat, options) + of "dll": + dllTests(r, cat, options & " -d:nimDebugDlOpen") + of "gc": + gcTests(r, cat, options) + of "debugger": + debuggerTests(r, cat, options) + of "manyloc": + manyLoc r, cat, options + of "threads": + threadTests r, cat, options & " --threads:on" + of "io": + ioTests r, cat, options + of "async": + asyncTests r, cat, options + of "lib": + testStdlib(r, "lib/pure/", options, cat) + testStdlib(r, "lib/packages/docutils/", options, cat) + of "examples": + compileExample(r, "examples/*.nim", options, cat) + compileExample(r, "examples/gtk/*.nim", options, cat) + compileExample(r, "examples/talk/*.nim", options, cat) + of "nimble-packages": + testNimblePackages(r, cat, options) + of "niminaction": + testNimInAction(r, cat, options) + of "ic": + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=false) + of "navigator": + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=true) + of "untestable": + # These require special treatment e.g. because they depend on a third party + # dependency; see `trunner_special` which runs some of those. discard else: - jsTests(r, cat, options) - of "dll": - dllTests(r, cat, options) - of "flags": - flagTests(r, cat, options) - of "gc": - gcTests(r, cat, options) - of "longgc": - longGCTests(r, cat, options) - of "debugger": - debuggerTests(r, cat, options) - of "manyloc": - manyLoc r, cat, options - of "threads": - threadTests r, cat, options & " --threads:on" - of "io": - ioTests r, cat, options - of "async": - asyncTests r, cat, options - of "lib": - testStdlib(r, "lib/pure/", options, cat) - testStdlib(r, "lib/packages/docutils/", options, cat) - of "examples": - compileExample(r, "examples/*.nim", options, cat) - compileExample(r, "examples/gtk/*.nim", options, cat) - compileExample(r, "examples/talk/*.nim", options, cat) - of "nimble-packages": - testNimblePackages(r, cat) - of "niminaction": - testNimInAction(r, cat, options) - of "untestable": - # We can't test it because it depends on a third party. - discard # TODO: Move untestable tests to someplace else, i.e. nimble repo. - of "megatest": - runJoinedTest(r, cat, testsDir) - else: - var testsRun = 0 - var files: seq[string] - for file in walkDirRec(testsDir &.? cat.string): - if isTestFile(file): files.add file - files.sort # give reproducible order - - for i, name in files: - var test = makeTest(name, options, cat) - if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories: - discard "run the test" - else: - test.spec.err = reJoined - testSpec r, test - inc testsRun - if testsRun == 0: - echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run" + handled = false + if not handled: + case cat2 + of "megatest": + runJoinedTest(r, cat, testsDir, options) + if isNimRepoTests(): + runJoinedTest(r, cat, testsDir, options & " --mm:refc") + else: + var testsRun = 0 + var files: seq[string] + for file in walkDirRec(testsDir &.? cat.string): + if isTestFile(file): files.add file + files.sort # give reproducible order + for i, name in files: + var test = makeTest(name, options, cat) + if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories: + discard "run the test" + else: + test.spec.err = reJoined + testSpec r, test + inc testsRun + if testsRun == 0: + const whiteListedDirs = ["deps", "htmldocs", "pkgs"] + # `pkgs` because bug #16556 creates `pkgs` dirs and this can affect some users + # that try an old version of choosenim. + doAssert cat.string in whiteListedDirs, + "Invalid category specified: '$#' not in whilelist: $#" % [cat.string, $whiteListedDirs] proc processPattern(r: var TResults, pattern, options: string; simulate: bool) = var testsRun = 0 - for name in walkPattern(pattern): - if simulate: - echo "Detected test: ", name - else: - var test = makeTest(name, options, Category"pattern") - testSpec r, test - inc testsRun + if dirExists(pattern): + for k, name in walkDir(pattern): + if k in {pcFile, pcLinkToFile} and name.endsWith(".nim"): + if simulate: + echo "Detected test: ", name + else: + var test = makeTest(name, options, Category"pattern") + testSpec r, test + inc testsRun + else: + for name in walkPattern(pattern): + if simulate: + echo "Detected test: ", name + else: + var test = makeTest(name, options, Category"pattern") + testSpec r, test + inc testsRun if testsRun == 0: echo "no tests were found for pattern: ", pattern |