diff options
Diffstat (limited to 'testament')
| -rw-r--r-- | testament/categories.nim | 288 | ||||
| -rw-r--r-- | testament/specs.nim | 13 | ||||
| -rw-r--r-- | testament/tester.nim | 25 |
3 files changed, 120 insertions, 206 deletions
diff --git a/testament/categories.nim b/testament/categories.nim index d9089e941..4c34eb1d1 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -10,6 +10,58 @@ ## Include for the tester that contains test suites that test special features ## of the compiler. +const + specialCategories = [ + "assert", + "async", + "debugger", + "dll", + "examples", + "flags", + "gc", + "io", + "js", + "lib", + "longgc", + "manyloc", + "nimble-all", + "nimble-core", + "nimble-extra", + "niminaction", + "rodfiles", + "threads", + "untestable", + "stdlib", + "testdata", + "nimcache", + "coroutines", + "osproc" + ] + + +# these tests still have bugs. At some point when the bugs are fixd +# this should become empty. + +# exclude for various reasons +const + specialDisabedTests = [ + "tests/dir with space/tspace.nim", # can't import dir with spaces. + "tests/method/tmultim.nim", # (77, 8) Error: method is not a base + "tests/system/talloc2.nim", # too much memory + "tests/collections/ttables.nim", # takes too long + "tests/system/tparams.nim", # executes itself with parameters + "tests/stdlib/tquit.nim", # not testing for obvious reasons + "tests/system/trealloc.nim", # out of memory + "tests/system/t7894.nim", # causes out of memory in later tests + "tests/types/tissues_types.nim", # causes out of memory with --gc:boehm + "tests/pragmas/tused.nim", # paths in nimout differ when imported + "tests/generics/trtree.nim", # very very ugly test + "tests/array/tarray.nim", # + "tests/destructor/turn_destroy_into_finalizer.nim", # fails when imported + "tests/misc/tnew.nim", + "tests/misc/tcmdline.nim" + ] + # included from tester.nim # ---------------- ROD file tests --------------------------------------------- @@ -327,24 +379,15 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) = let filename = "tests" / test.addFileExt("nim") let testHash = getMD5(readFile(filename).string) doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed." - - - # Run the tests. for testfile in tests: test "tests/" & testfile & ".nim" - let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim" testJS jsFile - let cppFile = "tests/niminaction/Chapter8/sfml/sfml_test.nim" testCPP cppFile - # ------------------------- manyloc ------------------------------------------- -#proc runSpecialTests(r: var TResults, options: string) = -# for t in ["lib/packages/docutils/highlite"]: -# testSpec(r, t, options) proc findMainFile(dir: string): string = # finds the file belonging to ".nim.cfg"; if there is no such file @@ -382,20 +425,18 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = for testFile in os.walkFiles(pattern): let name = extractFilename(testFile) if name notin disabledFiles: - - let contents = readFile(testFile).string - var testObj = makeTest(testFile, options, cat) if "when isMainModule" notin contents: testObj.spec.action = actionCompile testSpec r, testObj # ----------------------------- nimble ---------------------------------------- -type PackageFilter = enum - pfCoreOnly - pfExtraOnly - pfAll +type + PackageFilter = enum + pfCoreOnly + pfExtraOnly + pfAll var nimbleDir = getEnv("NIMBLE_DIR").string if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" @@ -425,7 +466,6 @@ proc getPackageDir(package: string): string = iterator listPackages(filter: PackageFilter): tuple[name, url: string] = let packageList = parseFile(packageIndex) - for package in packageList.items(): let name = package["name"].str @@ -485,19 +525,28 @@ proc `&.?`(a, b: string): string = # candidate for the stdlib? result = if b.startswith(a): b else: a & b -#proc `&?.`(a, b: string): string = # not used - # candidate for the stdlib? - #result = if a.endswith(b): a else: a & b - proc processSingleTest(r: var TResults, cat: Category, options, test: string) = let test = "tests" & DirSep &.? cat.string / test let target = if cat.string.normalize == "js": targetJS else: targetC - if existsFile(test): testSpec r, makeTest(test, options, cat), {target} - else: echo "[Warning] - ", test, " test does not exist" + else: + echo "[Warning] - ", test, " test does not exist" -proc isJoinableSpec(spec: TSpec): bool +proc isJoinableSpec(spec: TSpec): bool = + result = not spec.sortoutput and + spec.action == actionRun and + spec.file.replace('\\', '/') notin specialDisabedTests and + not fileExists(spec.file.changeFileExt("cfg")) and + not fileExists(parentDir(spec.file) / "nim.cfg") and + spec.cmd.len == 0 and + spec.err != reIgnored and + spec.exitCode == 0 and + spec.input.len == 0 and + spec.nimout.len == 0 and + spec.outputCheck != ocSubstr and + spec.ccodeCheck.len == 0 and + (spec.targets == {} or spec.targets == {targetC}) proc processCategory(r: var TResults, cat: Category, options: string, runJoinableTests: bool) = case cat.string.normalize @@ -560,191 +609,64 @@ proc processCategory(r: var TResults, cat: Category, options: string, runJoinabl if testsRun == 0: echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run" +proc norm(s: var string) = + while true: + let tmp = s.replace("\n\n", "\n") + if tmp == s: break + s = tmp + s = s.strip -const specialCategories = [ - "async", - "debugger", - "dll", - "examples", - "flags", - "gc", - "io", - "js", - "lib", - "longgc", - "manyloc", - "nimble-all", - "nimble-core", - "nimble-extra", - "niminaction", - "rodfiles", - "threads", - "untestable", - "stdlib", -] - - -# these tests still have bugs. At some point when the bugs are fixd -# this should become empty. - -# exclude for various reasons -const specialDisabedTests = [ - "tests/dir with space/tspace.nim", # can't import dir with spaces. - "tests/method/tmultim.nim", # (77, 8) Error: method is not a base - "tests/system/talloc2.nim", # too much memory - "tests/collections/ttables.nim", # takes too long - "tests/system/tparams.nim", # executes itself with parameters - "tests/stdlib/tquit.nim", # not testing for obvious reasons - "tests/system/trealloc.nim", # out of memory - "tests/system/t7894.nim", # causes out of memory in later tests - "tests/types/tissues_types.nim", # causes out of memory with --gc:boehm - "tests/pragmas/tused.nim", # paths in nimout differ when imported - "tests/generics/trtree.nim", # very very ugly test - "tests/array/tarray.nim", # - "tests/osproc/texecps.nim", # uses getAppFileName() to start itself with arguments - "tests/destructor/turn_destroy_into_finalizer.nim", # fails when imported - "tests/osproc/texitsignal.nim", # uses getAppFileName() to start itself with arguments -] - -proc isJoinableSpec(spec: TSpec): bool = - - if spec.sortoutput: - return false - - if spec.action != actionRun: - return false - - if spec.file in specialDisabedTests: - return false - - if fileExists(spec.file & ".cfg"): - return false - - if fileExists(parentDir(spec.file) / "nim.cfg"): - return false - - if spec.cmd != cmdTemplate(): - return false - - if spec.err == reIgnored: - return false - - if spec.exitCode != 0: - return false - - if spec.input != "": - return false - - if spec.targets != {} and spec.targets != {targetC}: - return false - - return true - - -proc runJoinedTest(): bool = +proc runJoinedTest(testsDir: string): bool = ## returs a list of tests that have problems - var specs:seq[TSpec] + var specs: seq[TSpec] = @[] - for file in os.walkFiles("tests/*/t*.nim"): - let a = find(file, '/') + 1 - let b = find(file, '/', a) - let cat = file[a ..< b] - - if cat in specialCategories: - continue - - let spec = parseSpec(file) - - if isJoinableSpec(spec): - specs.add spec + for kind, dir in walkDir(testsDir): + assert testsDir.startsWith(testsDir) + let cat = dir[testsDir.len .. ^1] + if kind == pcDir and cat notin specialCategories: + for file in os.walkFiles(testsDir / cat / "t*.nim"): + let spec = parseSpec(file) + if isJoinableSpec(spec): + specs.add spec echo "joinable specs: ", specs.len var megatest: string for runSpec in specs: - megatest.add "import \"" + megatest.add "import r\"" megatest.add runSpec.file megatest.add "\"\n" writeFile("megatest.nim", megatest) - let args = ["c", "-d:testing", "--gc:boehm", "megatest.nim"] + const args = ["c", "-d:testing", "--listCmd", "megatest.nim"] var (buf, exitCode) = execCmdEx2(command = "nim", args = args, options = {poStdErrToStdOut, poUsePath}, input = "") if exitCode != 0: + echo buf quit("megatest compilation failed") echo "compilation ok" - var nimoutOK = true - for runSpec in specs: - for line in runSpec.nimout.splitLines: - if buf.find(line) < 0: - echo "could not find: ", line - echo runSpec.file - nimoutOK = false - - if nimoutOK: - echo "nimout OK" - else: - echo "nimout FAIL" - (buf, exitCode) = execCmdEx2("./megatest", [], {poStdErrToStdOut}, "") if exitCode != 0: quit("megatest execution failed") echo "run ok" - + norm buf writeFile("outputGotten.txt", buf) var outputExpected = "" - - var outputErrorCount = 0 - var currentPos = 0 - - var lastLine = "" - - # when a lot of output is skipped, this can be the cause why a later test fails. - var warnings = "" - for i, runSpec in specs: - outputExpected.add runSpec.output - if outputExpected[^1] != '\n': - outputExpected.add '\n' - - for line in runSpec.output.splitLines: - if line != "": - #if line == "2": - # echo "found the test: ", runSpec.file - let newPos = buf.find(line, currentPos) - if newPos < 0: - if outputErrorCount < 5: - echo "could not find: ", line - echo "it could be, because the test failed, or too much output is discarded by a previous search in the output." - echo warnings - warnings.setLen 0 - - # don't spam too much of this - if outputErrorCount == 0: - echo "############" - echo buf[currentPos-200 ..< currentPos] - echo "| (", current_pos, ")" - echo buf[currentPos ..< min(currentPos+200, buf.len)] - echo "############" - - inc outputErrorCount - else: - if currentPos + lastLine.len * 2 < newPos: - warnings.addLine "Warning long skip in search for: ", line - warnings.addLine "in test: ", runSpec.file - currentPos = newPos + line.len - - lastLine = line - if outputErrorCount == 0: - echo "output OK" + outputExpected.add runSpec.output.strip + outputExpected.add '\n' + norm outputExpected + + if buf != outputExpected: + writeFile("outputExpected.txt", outputExpected) + discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt") + echo "output different!" + result = false else: - echo "output FAIL (", outputErrorCount, " errors)" - - writeFile("outputExpected.txt", outputExpected) - - # removeFile("megatest.nim") - return nimoutOK and outputErrorCount == 0 + echo "output OK" + removeFile("megatest.nim") + result = true diff --git a/testament/specs.nim b/testament/specs.nim index 9cbf2a9f6..6283d18d7 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -14,9 +14,6 @@ var compilerPrefix* = "compiler" / "nim" let isTravis* = existsEnv("TRAVIS") let isAppVeyor* = existsEnv("APPVEYOR") -proc cmdTemplate*(): string = - compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file" - type TTestAction* = enum actionRun = "run" @@ -70,6 +67,12 @@ type nimout*: string parseErrors*: string # when the spec definition is invalid, this is not empty. +proc getCmd*(s: TSpec): string = + if s.cmd.len == 0: + result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file" + else: + result = s.cmd + const targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"] targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"] @@ -97,9 +100,6 @@ proc extractSpec(filename: string): string = when not defined(nimhygiene): {.pragma: inject.} -proc defaultSpec*(): TSpec = - result.cmd = cmdTemplate() - proc parseTargets*(value: string): set[TTarget] = for v in value.normalize.splitWhitespace: case v @@ -119,7 +119,6 @@ proc addLine*(self: var string; a,b: string) = self.add "\n" proc parseSpec*(filename: string): TSpec = - result = defaultSpec() result.file = filename let specStr = extractSpec(filename) var ss = newStringStream(specStr) diff --git a/testament/tester.nim b/testament/tester.nim index d82d5ec0c..b290bb3b0 100644 --- a/testament/tester.nim +++ b/testament/tester.nim @@ -378,13 +378,13 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = case expected.action of actionCompile: - var given = callCompiler(expected.cmd, test.name, test.options, target, + var given = callCompiler(expected.getCmd, test.name, test.options, target, extraOptions=" --stdout --hint[Path]:off --hint[Processing]:off") compilerOutputTests(test, target, given, expected, r) of actionRun: # In this branch of code "early return" pattern is clearer than deep # nested conditionals - the empty rows in between to clarify the "danger" - var given = callCompiler(expected.cmd, test.name, test.options, + var given = callCompiler(expected.getCmd, test.name, test.options, target) if given.err != reSuccess: @@ -438,7 +438,7 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = bufB, reExitCodesDiffer) continue - if (expected.outputCheck == ocEqual and expected.output != bufB) or + if (expected.outputCheck == ocEqual and expected.output != bufB) or (expected.outputCheck == ocSubstr and expected.output notin bufB): given.err = reOutputsDiffer r.addResult(test, target, expected.output, bufB, reOutputsDiffer) @@ -448,7 +448,7 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = continue of actionReject: - var given = callCompiler(expected.cmd, test.name, test.options, + var given = callCompiler(expected.getCmd, test.name, test.options, target) cmpMsgs(r, expected, given, test, target) continue @@ -458,7 +458,7 @@ proc testC(r: var TResults, test: TTest, action: TTestAction) = let tname = test.name.addFileExt(".c") inc(r.total) maybeStyledEcho "Processing ", fgCyan, extractFilename(tname) - var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC) + var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC) if given.err != reSuccess: r.addResult(test, targetC, "", given.msg, given.err) elif action == actionRun: @@ -471,7 +471,7 @@ proc testExec(r: var TResults, test: TTest) = # runs executable or script, just goes by exit code inc(r.total) let (outp, errC) = execCmdEx(test.options.strip()) - var given: TSpec = defaultSpec() + var given: TSpec if errC == 0: given.err = reSuccess else: @@ -497,15 +497,9 @@ else: # array of modules disabled from compilation test of stdlib. disabledFiles = ["-"] - - - include categories -# proc runCaasTests(r: var TResults) = -# for test, output, status, mode in caasTestsRunner(): -# r.addResult(test, "", output & "-> " & $mode, -# if status: reSuccess else: reOutputsDiffer) +const testsDir = "tests" & DirSep proc main() = os.putenv "NIMTEST_COLOR", "never" @@ -556,9 +550,8 @@ proc main() = var r = initResults() case action of "all": - doAssert runJoinedTest() + doAssert runJoinedTest(testsDir) - let testsDir = "tests" & DirSep var myself = quoteShell(findExe("testament" / "tester")) if targetsStr.len > 0: myself &= " " & quoteShell("--targets:" & targetsStr) @@ -587,7 +580,7 @@ proc main() = of "html": generateHtml(resultsFile, optFailing) of "stats": - discard runJoinedTest() + discard runJoinedTest(testsDir) else: quit Usage |