diff options
Diffstat (limited to 'testament')
| -rw-r--r-- | testament/backend.nim | 16 | ||||
| -rw-r--r-- | testament/categories.nim | 267 | ||||
| -rw-r--r-- | testament/specs.nim | 257 | ||||
| -rw-r--r-- | testament/tester.nim | 205 | ||||
| -rw-r--r-- | testament/tests/shouldfail/tcolumn.nim | 2 | ||||
| -rw-r--r-- | testament/tests/shouldfail/terrormsg.nim | 2 | ||||
| -rw-r--r-- | testament/tests/shouldfail/tfile.nim | 2 | ||||
| -rw-r--r-- | testament/tests/shouldfail/tline.nim | 2 |
8 files changed, 448 insertions, 305 deletions
diff --git a/testament/backend.nim b/testament/backend.nim index 385f1171c..c2658d6a0 100644 --- a/testament/backend.nim +++ b/testament/backend.nim @@ -20,14 +20,11 @@ var thisCommit: CommitId thisBranch: string -{.experimental.} -proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip - proc getMachine*(): MachineId = - var name = "hostname"() + var name = execProcess("hostname").string.strip if name.len == 0: - name = when defined(posix): getenv"HOSTNAME".string - else: getenv"COMPUTERNAME".string + name = when defined(posix): getenv("HOSTNAME").string + else: getenv("COMPUTERNAME").string if name.len == 0: quit "cannot determine the machine name" @@ -35,8 +32,8 @@ proc getMachine*(): MachineId = proc getCommit(): CommitId = const commLen = "commit ".len - let hash = "git log -n 1"()[commLen..commLen+10] - thisBranch = "git symbolic-ref --short HEAD"() + let hash = execProcess("git log -n 1").string.strip[commLen..commLen+10] + thisBranch = execProcess("git symbolic-ref --short HEAD").string.strip if hash.len == 0 or thisBranch.len == 0: quit "cannot determine git HEAD" result = CommitId(hash) @@ -45,8 +42,7 @@ var currentCategory: string entries: int -proc writeTestResult*(name, category, target, - action, result, expected, given: string) = +proc writeTestResult*(name, category, target, action, result, expected, given: string) = createDir("testresults") if currentCategory != category: if currentCategory.len > 0: diff --git a/testament/categories.nim b/testament/categories.nim index e1f173c26..a72602217 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -10,6 +10,36 @@ ## Include for the tester that contains test suites that test special features ## of the compiler. +const + specialCategories = [ + "assert", + "async", + "debugger", + "dll", + "examples", + "flags", + "gc", + "io", + "js", + "lib", + "longgc", + "manyloc", + "nimble-all", + "nimble-core", + "nimble-extra", + "niminaction", + "rodfiles", + "threads", + "untestable", + "stdlib", + "testdata", + "nimcache", + "coroutines", + "osproc", + "shouldfail", + "dir with space" + ] + # included from tester.nim # ---------------- ROD file tests --------------------------------------------- @@ -27,7 +57,7 @@ proc delNimCache(filename, options: string) = proc runRodFiles(r: var TResults, cat: Category, options: string) = template test(filename: string, clearCacheFirst=false) = if clearCacheFirst: delNimCache(filename, options) - testSpec r, makeTest(rodfilesDir / filename, options, cat, actionRun) + testSpec r, makeTest(rodfilesDir / filename, options, cat) # test basic recompilation scheme: @@ -97,10 +127,12 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = else: "" - testNoSpec c, makeTest("lib/nimrtl.nim", - options & " --app:lib -d:createNimRtl --threads:on", cat, actionCompile) - testNoSpec c, makeTest("tests/dll/server.nim", - options & " --app:lib -d:useNimRtl --threads:on" & rpath, cat, actionCompile) + var test1 = makeTest("lib/nimrtl.nim", options & " --app:lib -d:createNimRtl --threads:on", cat) + test1.spec.action = actionCompile + testSpec c, test1 + var test2 = makeTest("tests/dll/server.nim", options & " --app:lib -d:useNimRtl --threads:on" & rpath, cat) + test2.spec.action = actionCompile + testSpec c, test2 when defined(Windows): # windows looks in the dir of the exe (yay!): @@ -120,7 +152,7 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll) testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl --threads:on" & rpath, - cat, actionRun) + cat) proc dllTests(r: var TResults, cat: Category, options: string) = # dummy compile result: @@ -138,32 +170,32 @@ proc dllTests(r: var TResults, cat: Category, options: string) = proc gcTests(r: var TResults, cat: Category, options: string) = template testWithNone(filename: untyped) = testSpec r, makeTest("tests/gc" / filename, options & - " --gc:none", cat, actionRun) + " --gc:none", cat) testSpec r, makeTest("tests/gc" / filename, options & - " -d:release --gc:none", cat, actionRun) + " -d:release --gc:none", cat) template testWithoutMs(filename: untyped) = - testSpec r, makeTest("tests/gc" / filename, options, cat, actionRun) + testSpec r, makeTest("tests/gc" / filename, options, cat) testSpec r, makeTest("tests/gc" / filename, options & - " -d:release", cat, actionRun) + " -d:release", cat) testSpec r, makeTest("tests/gc" / filename, options & - " -d:release -d:useRealtimeGC", cat, actionRun) + " -d:release -d:useRealtimeGC", cat) template testWithoutBoehm(filename: untyped) = testWithoutMs filename testSpec r, makeTest("tests/gc" / filename, options & - " --gc:markAndSweep", cat, actionRun) + " --gc:markAndSweep", cat) testSpec r, makeTest("tests/gc" / filename, options & - " -d:release --gc:markAndSweep", cat, actionRun) + " -d:release --gc:markAndSweep", cat) template test(filename: untyped) = testWithoutBoehm filename when not defined(windows) and not defined(android): # AR: cannot find any boehm.dll on the net, right now, so disabled # for windows: testSpec r, makeTest("tests/gc" / filename, options & - " --gc:boehm", cat, actionRun) + " --gc:boehm", cat) testSpec r, makeTest("tests/gc" / filename, options & - " -d:release --gc:boehm", cat, actionRun) + " -d:release --gc:boehm", cat) testWithoutBoehm "foreign_thr" test "gcemscripten" @@ -196,17 +228,18 @@ proc longGCTests(r: var TResults, cat: Category, options: string) = var c = initResults() # According to ioTests, this should compile the file - testNoSpec c, makeTest("tests/realtimeGC/shared", options, cat, actionCompile) - testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat, actionRun) - testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat, actionRun) + testSpec c, makeTest("tests/realtimeGC/shared", options, cat) + # ^- why is this not appended to r? Should this be discarded? + testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun + testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat) # ------------------------- threading tests ----------------------------------- proc threadTests(r: var TResults, cat: Category, options: string) = template test(filename: untyped) = - testSpec r, makeTest(filename, options, cat, actionRun) - testSpec r, makeTest(filename, options & " -d:release", cat, actionRun) - testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat, actionRun) + testSpec r, makeTest(filename, options, cat) + testSpec r, makeTest(filename, options & " -d:release", cat) + testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat) for t in os.walkFiles("tests/threads/t*.nim"): test(t) @@ -229,16 +262,16 @@ proc asyncTests(r: var TResults, cat: Category, options: string) = # ------------------------- debugger tests ------------------------------------ proc debuggerTests(r: var TResults, cat: Category, options: string) = - testNoSpec r, makeTest("tools/nimgrep", options & " --debugger:on", cat) + var t = makeTest("tools/nimgrep", options & " --debugger:on", cat) + t.spec.action = actionCompile + testSpec r, t # ------------------------- JS tests ------------------------------------------ proc jsTests(r: var TResults, cat: Category, options: string) = template test(filename: untyped) = - testSpec r, makeTest(filename, options & " -d:nodejs", cat, - actionRun), targetJS - testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat, - actionRun), targetJS + testSpec r, makeTest(filename, options & " -d:nodejs", cat), {targetJS} + testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat), {targetJS} for t in os.walkFiles("tests/js/t*.nim"): test(t) @@ -259,14 +292,14 @@ proc jsTests(r: var TResults, cat: Category, options: string) = proc testNimInAction(r: var TResults, cat: Category, options: string) = let options = options & " --nilseqs:on" - template test(filename: untyped, action: untyped) = - testSpec r, makeTest(filename, options, cat, action) + template test(filename: untyped) = + testSpec r, makeTest(filename, options, cat) template testJS(filename: untyped) = - testSpec r, makeTest(filename, options, cat, actionCompile), targetJS + testSpec r, makeTest(filename, options, cat), {targetJS} template testCPP(filename: untyped) = - testSpec r, makeTest(filename, options, cat, actionCompile), targetCPP + testSpec r, makeTest(filename, options, cat), {targetCPP} let tests = [ "niminaction/Chapter1/various1", @@ -298,39 +331,43 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) = # edit when making a conscious breaking change, also please try to make your # commit message clear and notify me so I can easily compile an errata later. const refHashes = @[ - "51afdfa84b3ca3d810809d6c4e5037ba", "30f07e4cd5eaec981f67868d4e91cfcf", - "d14e7c032de36d219c9548066a97e846", "2e40bfd5daadb268268727da91bb4e81", - "c5d3853ed0aba04bf6d35ba28a98dca0", "058603145ff92d46c009006b06e5b228", - "7b94a029b94ddb7efafddd546c965ff6", "586d74514394e49f2370dfc01dd9e830", - "13febc363ed82585f2a60de40ddfefda", "c11a013db35e798f44077bc0763cc86d", - "3e32e2c5e9a24bd13375e1cd0467079c", "0b9fe7ba159623d49ae60db18a15037c", - "b2dd5293d7f784824bbf9792c6fb51ad", "4c19d8d9026bfe151b31d7007fa3c237", - "9415c6a568cfceed08da8378e95b5cd5", "da520038c153f4054cb8cc5faa617714", - "e6c6e061b6f77b2475db6fec7abfb7f4", "9a8fe78c588d08018843b64b57409a02", - "8b5d28e985c0542163927d253a3e4fc9", "783299b98179cc725f9c46b5e3b5381f", - "bc523f9a9921299090bac1af6c958e73", "80f9c3e594a798225046e8a42e990daf" + "51afdfa84b3ca3d810809d6c4e5037ba", + "30f07e4cd5eaec981f67868d4e91cfcf", + "d14e7c032de36d219c9548066a97e846", + "b335635562ff26ec0301bdd86356ac0c", + "6c4add749fbf50860e2f523f548e6b0e", + "76de5833a7cc46f96b006ce51179aeb1", + "705eff79844e219b47366bd431658961", + "a1e87b881c5eb161553d119be8b52f64", + "2d706a6ec68d2973ec7e733e6d5dce50", + "c11a013db35e798f44077bc0763cc86d", + "3e32e2c5e9a24bd13375e1cd0467079c", + "a5452722b2841f0c1db030cf17708955", + "dc6c45eb59f8814aaaf7aabdb8962294", + "69d208d281a2e7bffd3eaf4bab2309b1", + "ec05666cfb60211bedc5e81d4c1caf3d", + "da520038c153f4054cb8cc5faa617714", + "59906c8cd819cae67476baa90a36b8c1", + "9a8fe78c588d08018843b64b57409a02", + "8b5d28e985c0542163927d253a3e4fc9", + "783299b98179cc725f9c46b5e3b5381f", + "1a2b3fba1187c68d6a9bfa66854f3318", + "80f9c3e594a798225046e8a42e990daf" ] for i, test in tests: let filename = "tests" / test.addFileExt("nim") let testHash = getMD5(readFile(filename).string) doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed." - # Run the tests. for testfile in tests: - test "tests/" & testfile & ".nim", actionCompile - + test "tests/" & testfile & ".nim" let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim" testJS jsFile - let cppFile = "tests/niminaction/Chapter8/sfml/sfml_test.nim" testCPP cppFile - # ------------------------- manyloc ------------------------------------------- -#proc runSpecialTests(r: var TResults, options: string) = -# for t in ["lib/packages/docutils/highlite"]: -# testSpec(r, t, options) proc findMainFile(dir: string): string = # finds the file belonging to ".nim.cfg"; if there is no such file @@ -354,27 +391,32 @@ proc manyLoc(r: var TResults, cat: Category, options: string) = if dir.endsWith"named_argument_bug": continue let mainfile = findMainFile(dir) if mainfile != "": - testNoSpec r, makeTest(mainfile, options, cat) + var test = makeTest(mainfile, options, cat) + test.spec.action = actionCompile + testSpec r, test proc compileExample(r: var TResults, pattern, options: string, cat: Category) = for test in os.walkFiles(pattern): - testNoSpec r, makeTest(test, options, cat) + var test = makeTest(test, options, cat) + test.spec.action = actionCompile + testSpec r, test proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = - for test in os.walkFiles(pattern): - let name = extractFilename(test) + for testFile in os.walkFiles(pattern): + let name = extractFilename(testFile) if name notin disabledFiles: - let contents = readFile(test).string - if contents.contains("when isMainModule"): - testSpec r, makeTest(test, options, cat, actionRunNoSpec) - else: - testNoSpec r, makeTest(test, options, cat, actionCompile) + let contents = readFile(testFile).string + var testObj = makeTest(testFile, options, cat) + if "when isMainModule" notin contents: + testObj.spec.action = actionCompile + testSpec r, testObj # ----------------------------- nimble ---------------------------------------- -type PackageFilter = enum - pfCoreOnly - pfExtraOnly - pfAll +type + PackageFilter = enum + pfCoreOnly + pfExtraOnly + pfAll var nimbleDir = getEnv("NIMBLE_DIR").string if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" @@ -404,7 +446,6 @@ proc getPackageDir(package: string): string = iterator listPackages(filter: PackageFilter): tuple[name, url: string] = let packageList = parseFile(packageIndex) - for package in packageList.items(): let name = package["name"].str @@ -458,24 +499,99 @@ proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) = # ---------------------------------------------------------------------------- -const AdditionalCategories = ["debugger", "examples", "lib"] +const AdditionalCategories = ["debugger", "examples", "lib", "megatest"] proc `&.?`(a, b: string): string = # candidate for the stdlib? result = if b.startswith(a): b else: a & b -#proc `&?.`(a, b: string): string = # not used - # candidate for the stdlib? - #result = if a.endswith(b): a else: a & b - proc processSingleTest(r: var TResults, cat: Category, options, test: string) = let test = "tests" & DirSep &.? cat.string / test let target = if cat.string.normalize == "js": targetJS else: targetC + if existsFile(test): + testSpec r, makeTest(test, options, cat), {target} + else: + echo "[Warning] - ", test, " test does not exist" + +proc isJoinableSpec(spec: TSpec): bool = + result = not spec.sortoutput and + spec.action == actionRun and + not fileExists(spec.file.changeFileExt("cfg")) and + not fileExists(parentDir(spec.file) / "nim.cfg") and + spec.cmd.len == 0 and + spec.err != reDisabled and + not spec.unjoinable and + spec.exitCode == 0 and + spec.input.len == 0 and + spec.nimout.len == 0 and + spec.outputCheck != ocSubstr and + spec.ccodeCheck.len == 0 and + (spec.targets == {} or spec.targets == {targetC}) + +proc norm(s: var string) = + while true: + let tmp = s.replace("\n\n", "\n") + if tmp == s: break + s = tmp + s = s.strip + +proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = + ## returs a list of tests that have problems + var specs: seq[TSpec] = @[] + + for kind, dir in walkDir(testsDir): + assert testsDir.startsWith(testsDir) + let cat = dir[testsDir.len .. ^1] + if kind == pcDir and cat notin specialCategories: + for file in os.walkFiles(testsDir / cat / "t*.nim"): + let spec = parseSpec(file) + if isJoinableSpec(spec): + specs.add spec + + echo "joinable specs: ", specs.len + + var megatest: string + for runSpec in specs: + megatest.add "import r\"" + megatest.add runSpec.file + megatest.add "\"\n" + + writeFile("megatest.nim", megatest) + + const args = ["c", "-d:testing", "--listCmd", "megatest.nim"] + var (buf, exitCode) = execCmdEx2(command = "nim", args = args, options = {poStdErrToStdOut, poUsePath}, input = "") + if exitCode != 0: + echo buf + quit("megatest compilation failed") + + (buf, exitCode) = execCmdEx("./megatest") + if exitCode != 0: + echo buf + quit("megatest execution failed") + + norm buf + writeFile("outputGotten.txt", buf) + var outputExpected = "" + for i, runSpec in specs: + outputExpected.add runSpec.output.strip + outputExpected.add '\n' + norm outputExpected + + if buf != outputExpected: + writeFile("outputExpected.txt", outputExpected) + discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt") + echo "output different!" + quit 1 + else: + echo "output OK" + removeFile("outputGotten.txt") + removeFile("megatest.nim") + #testSpec r, makeTest("megatest", options, cat) - if existsFile(test): testSpec r, makeTest(test, options, cat), target - else: echo "[Warning] - ", test, " test does not exist" +# --------------------------------------------------------------------------- -proc processCategory(r: var TResults, cat: Category, options: string) = +proc processCategory(r: var TResults, cat: Category, options, testsDir: string, + runJoinableTests: bool) = case cat.string.normalize of "rodfiles": when false: @@ -524,10 +640,17 @@ proc processCategory(r: var TResults, cat: Category, options: string) = of "untestable": # We can't test it because it depends on a third party. discard # TODO: Move untestable tests to someplace else, i.e. nimble repo. + of "megatest": + runJoinedTest(r, cat, testsDir) else: var testsRun = 0 for name in os.walkFiles("tests" & DirSep &.? cat.string / "t*.nim"): - testSpec r, makeTest(name, options, cat) + var test = makeTest(name, options, cat) + if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories: + discard "run the test" + else: + test.spec.err = reJoined + testSpec r, test inc testsRun if testsRun == 0: echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run" diff --git a/testament/specs.nim b/testament/specs.nim index 86fc8bed4..df12db543 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -9,21 +9,21 @@ import parseutils, strutils, os, osproc, streams, parsecfg - var compilerPrefix* = "compiler" / "nim" let isTravis* = existsEnv("TRAVIS") let isAppVeyor* = existsEnv("APPVEYOR") -proc cmdTemplate*(): string = - compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file" - type TTestAction* = enum actionRun = "run" actionCompile = "compile" actionReject = "reject" - actionRunNoSpec = "runNoSpec" + + TOutputCheck* = enum + ocIgnore = "ignore" + ocEqual = "equal" + ocSubstr = "substr" TResultEnum* = enum reNimcCrash, # nim compiler seems to have crashed @@ -38,8 +38,10 @@ type reExeNotFound, reInstallFailed # package installation failed reBuildFailed # package building failed - reIgnored, # test is ignored + reDisabled, # test is disabled + reJoined, # test is disabled because it was joined into the megatest reSuccess # test was successful + reInvalidSpec # test had problems to parse the spec TTarget* = enum targetC = "C" @@ -51,7 +53,9 @@ type action*: TTestAction file*, cmd*: string input*: string - outp*: string + outputCheck*: TOutputCheck + sortoutput*: bool + output*: string line*, column*: int tfile*: string tline*, tcolumn*: int @@ -60,9 +64,16 @@ type ccodeCheck*: string maxCodeSize*: int err*: TResultEnum - substr*, sortoutput*: bool targets*: set[TTarget] nimout*: string + parseErrors*: string # when the spec definition is invalid, this is not empty. + unjoinable*: bool + +proc getCmd*(s: TSpec): string = + if s.cmd.len == 0: + result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file" + else: + result = s.cmd const targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"] @@ -91,33 +102,6 @@ proc extractSpec(filename: string): string = when not defined(nimhygiene): {.pragma: inject.} -template parseSpecAux(fillResult: untyped) = - var ss = newStringStream(extractSpec(filename)) - var p {.inject.}: CfgParser - open(p, ss, filename, 1) - while true: - var e {.inject.} = next(p) - case e.kind - of cfgEof: break - of cfgSectionStart, cfgOption, cfgError: - echo ignoreMsg(p, e) - of cfgKeyValuePair: - fillResult - close(p) - -proc specDefaults*(result: var TSpec) = - result.msg = "" - result.outp = "" - result.nimout = "" - result.ccodeCheck = "" - result.cmd = cmdTemplate() - result.line = 0 - result.column = 0 - result.tfile = "" - result.tline = 0 - result.tcolumn = 0 - result.maxCodeSize = 0 - proc parseTargets*(value: string): set[TTarget] = for v in value.normalize.splitWhitespace: case v @@ -127,81 +111,134 @@ proc parseTargets*(value: string): set[TTarget] = of "js": result.incl(targetJS) else: echo "target ignored: " & v +proc addLine*(self: var string; a: string) = + self.add a + self.add "\n" + +proc addLine*(self: var string; a,b: string) = + self.add a + self.add b + self.add "\n" + proc parseSpec*(filename: string): TSpec = - specDefaults(result) result.file = filename - parseSpecAux: - case normalize(e.key) - of "action": - case e.value.normalize - of "compile": result.action = actionCompile - of "run": result.action = actionRun - of "reject": result.action = actionReject - else: echo ignoreMsg(p, e) - of "file": result.file = e.value - of "line": discard parseInt(e.value, result.line) - of "column": discard parseInt(e.value, result.column) - of "tfile": result.tfile = e.value - of "tline": discard parseInt(e.value, result.tline) - of "tcolumn": discard parseInt(e.value, result.tcolumn) - of "output": - result.action = actionRun - result.outp = e.value - of "input": - result.input = e.value - of "outputsub": - result.action = actionRun - result.outp = e.value - result.substr = true - of "sortoutput": - result.sortoutput = parseCfgBool(e.value) - of "exitcode": - discard parseInt(e.value, result.exitCode) - result.action = actionRun - of "msg": - result.msg = e.value - if result.action != actionRun: - result.action = actionCompile - of "errormsg", "errmsg": - result.msg = e.value - result.action = actionReject - of "nimout": - result.nimout = e.value - of "disabled": - case e.value.normalize - of "y", "yes", "true", "1", "on": result.err = reIgnored - of "n", "no", "false", "0", "off": discard - of "win", "windows": - when defined(windows): result.err = reIgnored - of "linux": - when defined(linux): result.err = reIgnored - of "bsd": - when defined(bsd): result.err = reIgnored - of "macosx": - when defined(macosx): result.err = reIgnored - of "unix": - when defined(unix): result.err = reIgnored - of "posix": - when defined(posix): result.err = reIgnored - of "travis": - if isTravis: result.err = reIgnored - of "appveyor": - if isAppVeyor: result.err = reIgnored - else: - raise newException(ValueError, "cannot interpret as a bool: " & e.value) - of "cmd": - if e.value.startsWith("nim "): - result.cmd = compilerPrefix & e.value[3..^1] + let specStr = extractSpec(filename) + var ss = newStringStream(specStr) + var p: CfgParser + open(p, ss, filename, 1) + while true: + var e = next(p) + case e.kind + of cfgKeyValuePair: + case normalize(e.key) + of "action": + case e.value.normalize + of "compile": + result.action = actionCompile + of "run": + result.action = actionRun + of "reject": + result.action = actionReject + else: + result.parseErrors.addLine "cannot interpret as action: ", e.value + of "file": + if result.msg.len == 0 and result.nimout.len == 0: + result.parseErrors.addLine "errormsg or msg needs to be specified before file" + result.file = e.value + of "line": + if result.msg.len == 0 and result.nimout.len == 0: + result.parseErrors.addLine "errormsg, msg or nimout needs to be specified before line" + discard parseInt(e.value, result.line) + of "column": + if result.msg.len == 0 and result.nimout.len == 0: + result.parseErrors.addLine "errormsg or msg needs to be specified before column" + discard parseInt(e.value, result.column) + of "tfile": + result.tfile = e.value + of "tline": + discard parseInt(e.value, result.tline) + of "tcolumn": + discard parseInt(e.value, result.tcolumn) + of "output": + result.outputCheck = ocEqual + result.output = strip(e.value) + of "input": + result.input = e.value + of "outputsub": + result.outputCheck = ocSubstr + result.output = strip(e.value) + of "sortoutput": + try: + result.sortoutput = parseCfgBool(e.value) + except: + result.parseErrors.addLine getCurrentExceptionMsg() + of "exitcode": + discard parseInt(e.value, result.exitCode) + result.action = actionRun + of "msg": + result.msg = e.value + if result.action != actionRun: + result.action = actionCompile + of "errormsg", "errmsg": + result.msg = e.value + result.action = actionReject + of "nimout": + result.nimout = e.value + of "joinable": + result.unjoinable = not parseCfgBool(e.value) + of "disabled": + case e.value.normalize + of "y", "yes", "true", "1", "on": result.err = reDisabled + of "n", "no", "false", "0", "off": discard + of "win", "windows": + when defined(windows): result.err = reDisabled + of "linux": + when defined(linux): result.err = reDisabled + of "bsd": + when defined(bsd): result.err = reDisabled + of "macosx": + when defined(macosx): result.err = reDisabled + of "unix": + when defined(unix): result.err = reDisabled + of "posix": + when defined(posix): result.err = reDisabled + of "travis": + if isTravis: result.err = reDisabled + of "appveyor": + if isAppVeyor: result.err = reDisabled + else: + result.parseErrors.addLine "cannot interpret as a bool: ", e.value + of "cmd": + if e.value.startsWith("nim "): + result.cmd = compilerPrefix & e.value[3..^1] + else: + result.cmd = e.value + of "ccodecheck": + result.ccodeCheck = e.value + of "maxcodesize": + discard parseInt(e.value, result.maxCodeSize) + of "target", "targets": + for v in e.value.normalize.splitWhitespace: + case v + of "c": + result.targets.incl(targetC) + of "cpp", "c++": + result.targets.incl(targetCpp) + of "objc": + result.targets.incl(targetObjC) + of "js": + result.targets.incl(targetJS) + else: + result.parseErrors.addLine "cannot interpret as a target: ", e.value else: - result.cmd = e.value - of "ccodecheck": result.ccodeCheck = e.value - of "maxcodesize": discard parseInt(e.value, result.maxCodeSize) - of "target", "targets": - for v in e.value.normalize.splitWhitespace: - case v - of "c": result.targets.incl(targetC) - of "cpp", "c++": result.targets.incl(targetCpp) - of "objc": result.targets.incl(targetObjC) - of "js": result.targets.incl(targetJS) - else: echo ignoreMsg(p, e) - else: echo ignoreMsg(p, e) + result.parseErrors.addLine "invalid key for test spec: ", e.key + + of cfgSectionStart: + result.parseErrors.addLine "section ignored: ", e.section + of cfgOption: + result.parseErrors.addLine "command ignored: ", e.key & ": " & e.value + of cfgError: + result.parseErrors.addLine e.msg + of cfgEof: + break + close(p) diff --git a/testament/tester.nim b/testament/tester.nim index 59c0171b4..6d9e05aa9 100644 --- a/testament/tester.nim +++ b/testament/tester.nim @@ -15,6 +15,7 @@ import algorithm, compiler/nodejs, times, sets, md5 var useColors = true +var backendLogging = true const resultsFile = "testresults.html" @@ -27,6 +28,7 @@ Command: c|cat|category <category> run all the tests of a certain category r|run <test> run single test file html generate $1 from the database + stats generate statistics about test cases Arguments: arguments are passed to the compiler Options: @@ -35,7 +37,8 @@ Options: --targets:"c c++ js objc" run tests for specified targets (default: all) --nim:path use a particular nim executable (default: compiler/nim) --directory:dir Change to directory dir before reading the tests or doing anything else. - --colors:on|off turn messagescoloring on|off + --colors:on|off Turn messagescoloring on|off. + --backendLogging:on|off Disable or enable backend logging. By default turned on. """ % resultsFile type @@ -48,7 +51,7 @@ type name: string cat: Category options: string - action: TTestAction + spec: TSpec startTime: float # ---------------------------------------------------------------------------- @@ -113,7 +116,7 @@ proc nimcacheDir(filename, options: string, target: TTarget): string = proc callCompiler(cmdTemplate, filename, options: string, target: TTarget, extraOptions=""): TSpec = let nimcache = nimcacheDir(filename, options, target) - let options = options & " " & ("--nimCache:" & nimcache).quoteShell & extraOptions + let options = options & " " & quoteShell("--nimCache:" & nimcache) & extraOptions let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], "options", options, "file", filename.quoteShell, "filedir", filename.getFileDir()]) @@ -138,7 +141,7 @@ proc callCompiler(cmdTemplate, filename, options: string, close(p) result.msg = "" result.file = "" - result.outp = "" + result.output = "" result.line = 0 result.column = 0 result.tfile = "" @@ -170,7 +173,7 @@ proc callCCompiler(cmdTemplate, filename, options: string, result.nimout = "" result.msg = "" result.file = "" - result.outp = "" + result.output = "" result.line = -1 while outp.readLine(x.TaintedString) or running(p): result.nimout.add(x & "\n") @@ -219,18 +222,21 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, let name = test.name.extractFilename & " " & $target & test.options let duration = epochTime() - test.startTime let durationStr = duration.formatFloat(ffDecimal, precision = 8).align(11) - backend.writeTestResult(name = name, - category = test.cat.string, - target = $target, - action = $test.action, - result = $success, - expected = expected, - given = given) + if backendLogging: + backend.writeTestResult(name = name, + category = test.cat.string, + target = $target, + action = $test.spec.action, + result = $success, + expected = expected, + given = given) r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) if success == reSuccess: maybeStyledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)" - elif success == reIgnored: + elif success == reDisabled: maybeStyledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name + elif success == reJoined: + maybeStyledEcho styleDim, fgYellow, "JOINED: ", styleBright, fgCyan, name else: maybeStyledEcho styleBright, fgRed, "FAIL: ", fgCyan, name maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" @@ -240,11 +246,11 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, maybeStyledEcho fgYellow, "Gotten:" maybeStyledEcho styleBright, given, "\n" - if existsEnv("APPVEYOR"): + if backendLogging and existsEnv("APPVEYOR"): let (outcome, msg) = if success == reSuccess: ("Passed", "") - elif success == reIgnored: + elif success in {reDisabled, reJoined}: ("Skipped", "") else: ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given) @@ -328,11 +334,6 @@ proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) = given.err = reMsgsDiffer return -proc makeDeterministic(s: string): string = - var x = splitLines(s) - sort(x, system.cmp) - result = join(x, "\n") - proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, expected: TSpec; r: var TResults) = var expectedmsg: string = "" @@ -350,75 +351,58 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, if given.err == reSuccess: inc(r.passed) r.addResult(test, target, expectedmsg, givenmsg, given.err) -proc testSpec(r: var TResults, test: TTest, target = targetC) = - let tname = test.name.addFileExt(".nim") - var expected: TSpec - if test.action != actionRunNoSpec: - expected = parseSpec(tname) - if test.action == actionRun and expected.action == actionCompile: - expected.action = actionRun - else: - specDefaults expected - expected.action = actionRunNoSpec - - if expected.err == reIgnored: - r.addResult(test, target, "", "", reIgnored) +proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = + var expected = test.spec + if expected.parseErrors.len > 0: + # targetC is a lie, but parameter is required + r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec) + inc(r.total) + return + if expected.err in {reDisabled, reJoined}: + # targetC is a lie, but parameter is required + r.addResult(test, targetC, "", "", expected.err) inc(r.skipped) inc(r.total) return - - if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true" and target == targetC and expected.targets == {}: - expected.targets.incl(targetCpp) - elif expected.targets == {}: - expected.targets.incl(target) - + expected.targets.incl targets + # still no target specified at all + if expected.targets == {}: + if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true": + expected.targets = {targetCpp} + else: + expected.targets = {targetC} for target in expected.targets: inc(r.total) - if target notin targets: - r.addResult(test, target, "", "", reIgnored) - inc(r.skipped) - continue - case expected.action of actionCompile: - var given = callCompiler(expected.cmd, test.name, test.options, target, + var given = callCompiler(expected.getCmd, test.name, test.options, target, extraOptions=" --stdout --hint[Path]:off --hint[Processing]:off") compilerOutputTests(test, target, given, expected, r) - of actionRun, actionRunNoSpec: + of actionRun: # In this branch of code "early return" pattern is clearer than deep # nested conditionals - the empty rows in between to clarify the "danger" - var given = callCompiler(expected.cmd, test.name, test.options, + var given = callCompiler(expected.getCmd, test.name, test.options, target) - - # echo "expected.cmd: ", expected.cmd - # echo "nimout: ", given.nimout - # echo "outp: ", given.outp - # echo "msg: ", given.msg - # echo "err: ", given.err - if given.err != reSuccess: r.addResult(test, target, "", given.msg, given.err) continue - let isJsTarget = target == targetJS var exeFile: string if isJsTarget: - let (_, file, _) = splitFile(tname) - exeFile = nimcacheDir(test.name, test.options, target) / file & ".js" + let file = test.name.lastPathPart.changeFileExt("js") + exeFile = nimcacheDir(test.name, test.options, target) / file else: - exeFile = changeFileExt(tname, ExeExt) + exeFile = changeFileExt(test.name, ExeExt) if not existsFile(exeFile): - r.addResult(test, target, expected.outp, "executable not found", reExeNotFound) + r.addResult(test, target, expected.output, "executable not found", reExeNotFound) continue let nodejs = if isJsTarget: findNodeJs() else: "" if isJsTarget and nodejs == "": - r.addResult(test, target, expected.outp, "nodejs binary not in PATH", + r.addResult(test, target, expected.output, "nodejs binary not in PATH", reExeNotFound) continue - - var exeCmd: string var args: seq[string] if isJsTarget: @@ -426,55 +410,44 @@ proc testSpec(r: var TResults, test: TTest, target = targetC) = args.add exeFile else: exeCmd = exeFile - var (buf, exitCode) = execCmdEx2(exeCmd, args, options = {poStdErrToStdOut}, input = expected.input) - # Treat all failure codes from nodejs as 1. Older versions of nodejs used # to return other codes, but for us it is sufficient to know that it's not 0. if exitCode != 0: exitCode = 1 - - let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string)) - else: strip(buf.string) - let expectedOut = strip(expected.outp) - + let bufB = + if expected.sortoutput: + var x = splitLines(strip(buf.string)) + sort(x, system.cmp) + join(x, "\n") + else: + strip(buf.string) if exitCode != expected.exitCode: r.addResult(test, target, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode & "\n\nOutput:\n" & bufB, reExitCodesDiffer) continue - - if bufB != expectedOut and expected.action != actionRunNoSpec: - if not (expected.substr and expectedOut in bufB): - given.err = reOutputsDiffer - r.addResult(test, target, expected.outp, bufB, reOutputsDiffer) - continue - + if (expected.outputCheck == ocEqual and expected.output != bufB) or + (expected.outputCheck == ocSubstr and expected.output notin bufB): + given.err = reOutputsDiffer + r.addResult(test, target, expected.output, bufB, reOutputsDiffer) + continue compilerOutputTests(test, target, given, expected, r) continue - of actionReject: - var given = callCompiler(expected.cmd, test.name, test.options, + var given = callCompiler(expected.getCmd, test.name, test.options, target) cmpMsgs(r, expected, given, test, target) continue -proc testNoSpec(r: var TResults, test: TTest, target = targetC) = - # does not extract the spec because the file is not supposed to have any - #let tname = test.name.addFileExt(".nim") - inc(r.total) - let given = callCompiler(cmdTemplate(), test.name, test.options, target) - r.addResult(test, target, "", given.msg, given.err) - if given.err == reSuccess: inc(r.passed) - -proc testC(r: var TResults, test: TTest) = +proc testC(r: var TResults, test: TTest, action: TTestAction) = # runs C code. Doesn't support any specs, just goes by exit code. let tname = test.name.addFileExt(".c") inc(r.total) maybeStyledEcho "Processing ", fgCyan, extractFilename(tname) - var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC) + var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC) if given.err != reSuccess: r.addResult(test, targetC, "", given.msg, given.err) - elif test.action == actionRun: + elif action == actionRun: let exeFile = changeFileExt(test.name, ExeExt) var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath}) if exitCode != 0: given.err = reExitCodesDiffer @@ -485,7 +458,6 @@ proc testExec(r: var TResults, test: TTest) = inc(r.total) let (outp, errC) = execCmdEx(test.options.strip()) var given: TSpec - specDefaults(given) if errC == 0: given.err = reSuccess else: @@ -495,11 +467,12 @@ proc testExec(r: var TResults, test: TTest) = if given.err == reSuccess: inc(r.passed) r.addResult(test, targetC, "", given.msg, given.err) -proc makeTest(test, options: string, cat: Category, action = actionCompile, - env: string = ""): TTest = - # start with 'actionCompile', will be overwritten in the spec: - result = TTest(cat: cat, name: test, options: options, - action: action, startTime: epochTime()) +proc makeTest(test, options: string, cat: Category): TTest = + result.cat = cat + result.name = test + result.options = options + result.spec = parseSpec(addFileExt(test, ".nim")) + result.startTime = epochTime() when defined(windows): const @@ -512,10 +485,7 @@ else: include categories -# proc runCaasTests(r: var TResults) = -# for test, output, status, mode in caasTestsRunner(): -# r.addResult(test, "", output & "-> " & $mode, -# if status: reSuccess else: reOutputsDiffer) +const testsDir = "tests" & DirSep proc main() = os.putenv "NIMTEST_COLOR", "never" @@ -524,10 +494,8 @@ proc main() = backend.open() var optPrintResults = false var optFailing = false - var targetsStr = "" - var p = initOptParser() p.next() while p.kind == cmdLongoption: @@ -538,26 +506,38 @@ proc main() = of "targets": targetsStr = p.val.string targets = parseTargets(targetsStr) - of "nim": compilerPrefix = p.val.string + of "nim": + compilerPrefix = addFileExt(p.val.string, ExeExt) of "directory": setCurrentDir(p.val.string) of "colors": - if p.val.string == "on": + case p.val.string: + of "on": useColors = true - elif p.val.string == "off": + of "off": useColors = false else: quit Usage + of "backendlogging": + case p.val.string: + of "on": + backendLogging = true + of "off": + backendLogging = false + else: + quit Usage else: quit Usage p.next() - if p.kind != cmdArgument: quit Usage + if p.kind != cmdArgument: + quit Usage var action = p.key.string.normalize p.next() var r = initResults() case action of "all": - let testsDir = "tests" & DirSep + #processCategory(r, Category"megatest", p.cmdLineRest.string, testsDir, runJoinableTests = false) + var myself = quoteShell(findExe("testament" / "tester")) if targetsStr.len > 0: myself &= " " & quoteShell("--targets:" & targetsStr) @@ -570,14 +550,21 @@ proc main() = assert testsDir.startsWith(testsDir) let cat = dir[testsDir.len .. ^1] if kind == pcDir and cat notin ["testdata", "nimcache"]: - cmds.add(myself & " cat " & quoteShell(cat) & rest) + cmds.add(myself & " pcat " & quoteShell(cat) & rest) for cat in AdditionalCategories: - cmds.add(myself & " cat " & quoteShell(cat) & rest) + cmds.add(myself & " pcat " & quoteShell(cat) & rest) quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}) of "c", "cat", "category": var cat = Category(p.key) p.next - processCategory(r, cat, p.cmdLineRest.string) + processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true) + of "pcat": + # 'pcat' is used for running a category in parallel. Currently the only + # difference is that we don't want to run joinable tests here as they + # are covered by the 'megatest' category. + var cat = Category(p.key) + p.next + processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = false) of "r", "run": let (dir, file) = splitPath(p.key.string) let (_, subdir) = splitPath(dir) diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim index f4046d58d..89482e673 100644 --- a/testament/tests/shouldfail/tcolumn.nim +++ b/testament/tests/shouldfail/tcolumn.nim @@ -1,7 +1,7 @@ discard """ +errormsg: "undeclared identifier: 'undeclared'" line: 8 column: 7 -errormsg: "undeclared identifier: 'undeclared'" """ # test should fail because the line directive is wrong diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim index 61c08d93d..dbbdf5021 100644 --- a/testament/tests/shouldfail/terrormsg.nim +++ b/testament/tests/shouldfail/terrormsg.nim @@ -1,7 +1,7 @@ discard """ +errormsg: "wrong error message" line: 8 column: 6 -errormsg: "wrong error message" """ # test should fail because the line directive is wrong diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim index 07a526c68..a6c2ad359 100644 --- a/testament/tests/shouldfail/tfile.nim +++ b/testament/tests/shouldfail/tfile.nim @@ -1,6 +1,6 @@ discard """ -file: "notthisfile.nim" errmsg: "undeclared identifier: 'undefined'" +file: "notthisfile.nim" """ echo undefined diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim index 963e44fc7..f7a09875c 100644 --- a/testament/tests/shouldfail/tline.nim +++ b/testament/tests/shouldfail/tline.nim @@ -1,7 +1,7 @@ discard """ +errormsg: "undeclared identifier: 'undeclared'" line: 9 column: 6 -errormsg: "undeclared identifier: 'undeclared'" """ # test should fail because the line directive is wrong |