diff options
Diffstat (limited to 'testament')
| -rw-r--r-- | testament/categories.nim | 57 | ||||
| -rw-r--r-- | testament/tester.nim | 9 |
2 files changed, 46 insertions, 20 deletions
diff --git a/testament/categories.nim b/testament/categories.nim index 55b9e5ed2..4ff50af97 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -97,7 +97,7 @@ proc compileRodFiles(r: var TResults, cat: Category, options: string) = proc flagTests(r: var TResults, cat: Category, options: string) = # --genscript - const filename = "tests"/"flags"/"tgenscript" + const filename = testsDir/"flags"/"tgenscript" const genopts = " --genscript" let nimcache = nimcacheDir(filename, genopts, targetC) testSpec r, makeTest(filename, genopts, cat) @@ -356,7 +356,7 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) = ] for i, test in tests: - let filename = "tests" / test.addFileExt("nim") + let filename = testsDir / test.addFileExt("nim") let testHash = getMD5(readFile(filename).string) doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed." # Run the tests. @@ -425,15 +425,12 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = let contents = readFile(testFile).string var testObj = makeTest(testFile, options, cat) #[ - TODO: + todo: this logic is fragile: false positives (if appears in a comment), or false negatives, eg `when defined(osx) and isMainModule`. - Instead of fixing this, a much better way, is to extend - https://github.com/nim-lang/Nim/issues/9581 to stdlib modules as follows: - * add these to megatest - * patch compiler so `isMainModule` is true when -d:isMainModuleIsAlwaysTrue - That'll give speedup benefit, and we don't have to patch stdlib files. + Instead of fixing this, see https://github.com/nim-lang/Nim/issues/10045 + for a much better way. ]# if "when isMainModule" notin contents: testObj.spec.action = actionCompile @@ -534,7 +531,7 @@ proc `&.?`(a, b: string): string = result = if b.startswith(a): b else: a & b proc processSingleTest(r: var TResults, cat: Category, options, test: string) = - let test = "tests" & DirSep &.? cat.string / test + let test = testsDir &.? cat.string / test let target = if cat.string.normalize == "js": targetJS else: targetC if existsFile(test): testSpec r, makeTest(test, options, cat), {target} @@ -559,6 +556,7 @@ proc isJoinableSpec(spec: TSpec): bool = (spec.targets == {} or spec.targets == {targetC}) proc norm(s: var string) = + # equivalent of s/\n+/\n/g (could use a single pass over input if needed) while true: let tmp = s.replace("\n\n", "\n") if tmp == s: break @@ -566,9 +564,13 @@ proc norm(s: var string) = s = s.strip proc isTestFile*(file: string): bool = - let (dir, name, ext) = splitFile(file) + let (_, name, ext) = splitFile(file) result = ext == ".nim" and name.startsWith("t") +proc quoted(a: string): string = + # todo: consider moving to system.nim + result.addQuoted(a) + proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = ## returs a list of tests that have problems var specs: seq[TSpec] = @[] @@ -582,6 +584,8 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = if isJoinableSpec(spec): specs.add spec + proc cmp(a: TSpec, b:TSpec): auto = cmp(a.file, b.file) + sort(specs, cmp=cmp) # reproducible order echo "joinable specs: ", specs.len if simulate: @@ -591,19 +595,36 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = return var megatest: string - for runSpec in specs: - megatest.add "import r\"" - megatest.add runSpec.file - megatest.add "\"\n" + #[ + TODO(minor): + get from Nim cmd + put outputGotten.txt, outputGotten.txt, megatest.nim there too + delete upon completion, maybe + ]# + var outDir = nimcacheDir(testsDir / "megatest", "", targetC) + const marker = "megatest:processing: " + + for i, runSpec in specs: + let file = runSpec.file + let file2 = outDir / ("megatest_" & $i & ".nim") + # `include` didn't work with `trecmod2.nim`, so using `import` + let code = "echo \"" & marker & "\", " & quoted(file) & "\n" + createDir(file2.parentDir) + writeFile(file2, code) + megatest.add "import " & quoted(file2) & "\n" + megatest.add "import " & quoted(file) & "\n" writeFile("megatest.nim", megatest) - const args = ["c", "-d:testing", "--listCmd", "megatest.nim"] - var (buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, options = {poStdErrToStdOut, poUsePath}, input = "") + let args = ["c", "--nimCache:" & outDir, "-d:testing", "--listCmd", "megatest.nim"] + proc onStdout(line: string) = echo line + var (buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, options = {poStdErrToStdOut, poUsePath}, input = "", + onStdout = if verboseMegatest: onStdout else: nil) if exitCode != 0: echo buf quit("megatest compilation failed") + # Could also use onStdout here. (buf, exitCode) = execCmdEx("./megatest") if exitCode != 0: echo buf @@ -613,6 +634,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = writeFile("outputGotten.txt", buf) var outputExpected = "" for i, runSpec in specs: + outputExpected.add marker & runSpec.file & "\n" outputExpected.add runSpec.output.strip outputExpected.add '\n' norm outputExpected @@ -621,6 +643,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = writeFile("outputExpected.txt", outputExpected) discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt") echo "output different!" + # outputGotten.txt, outputExpected.txt not removed on purpose for debugging. quit 1 else: echo "output OK" @@ -686,7 +709,7 @@ proc processCategory(r: var TResults, cat: Category, options, testsDir: string, var testsRun = 0 var files: seq[string] - for file in walkDirRec("tests" & DirSep &.? cat.string): + for file in walkDirRec(testsDir &.? cat.string): if isTestFile(file): files.add file files.sort # give reproducible order diff --git a/testament/tester.nim b/testament/tester.nim index 9e5fe5830..1867871c5 100644 --- a/testament/tester.nim +++ b/testament/tester.nim @@ -17,8 +17,10 @@ import var useColors = true var backendLogging = true var simulate = false +var verboseMegatest = false # very verbose but can be useful const + testsDir = "tests" & DirSep resultsFile = "testresults.html" #jsonFile = "testresults.json" # not used Usage = """Usage: @@ -34,6 +36,7 @@ Arguments: arguments are passed to the compiler Options: --print also print results to the console + --verboseMegatest log to stdout megatetest compilation --simulate see what tests would be run but don't run them (for debugging) --failing only show failing/ignored tests --targets:"c c++ js objc" run tests for specified targets (default: all) @@ -83,7 +86,7 @@ proc getFileDir(filename: string): string = if not result.isAbsolute(): result = getCurrentDir() / result -proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOption], input: string): tuple[ +proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOption], input: string, onStdout: proc(line: string) = nil): tuple[ output: TaintedString, exitCode: int] {.tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = @@ -103,6 +106,7 @@ proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOp if outp.readLine(line): result[0].string.add(line.string) result[0].string.add("\n") + if onStdout != nil: onStdout(line.string) else: result[1] = peekExitCode(p) if result[1] != -1: break @@ -517,8 +521,6 @@ else: include categories -const testsDir = "tests" & DirSep - proc main() = os.putenv "NIMTEST_COLOR", "never" os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES" @@ -533,6 +535,7 @@ proc main() = while p.kind == cmdLongoption: case p.key.string.normalize of "print", "verbose": optPrintResults = true + of "verbosemegatest": verboseMegatest = true of "failing": optFailing = true of "pedantic": discard "now always enabled" of "targets": |