diff options
| -rw-r--r-- | testament/categories.nim | 2 | ||||
| -rw-r--r-- | testament/specs.nim | 17 | ||||
| -rw-r--r-- | testament/tester.nim | 50 |
3 files changed, 39 insertions, 30 deletions
diff --git a/testament/categories.nim b/testament/categories.nim index e1f173c26..6694e3f0e 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -521,6 +521,8 @@ proc processCategory(r: var TResults, cat: Category, options: string) = testNimblePackages(r, cat, pfAll) of "niminaction": testNimInAction(r, cat, options) + of "testament": + testTestament(r, cat, options) of "untestable": # We can't test it because it depends on a third party. discard # TODO: Move untestable tests to someplace else, i.e. nimble repo. diff --git a/testament/specs.nim b/testament/specs.nim index 86fc8bed4..a86544c27 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -25,6 +25,11 @@ type actionReject = "reject" actionRunNoSpec = "runNoSpec" + TOutputCheck* = enum + ocIgnore = "ignore" + ocEqual = "equal" + ocSubstr = "substr" + TResultEnum* = enum reNimcCrash, # nim compiler seems to have crashed reMsgsDiffer, # error messages differ @@ -51,6 +56,8 @@ type action*: TTestAction file*, cmd*: string input*: string + outputCheck*: TOutputCheck + sortoutput*: bool outp*: string line*, column*: int tfile*: string @@ -60,7 +67,6 @@ type ccodeCheck*: string maxCodeSize*: int err*: TResultEnum - substr*, sortoutput*: bool targets*: set[TTarget] nimout*: string @@ -145,14 +151,13 @@ proc parseSpec*(filename: string): TSpec = of "tline": discard parseInt(e.value, result.tline) of "tcolumn": discard parseInt(e.value, result.tcolumn) of "output": - result.action = actionRun - result.outp = e.value + result.outputCheck = ocEqual + result.outp = strip(e.value) of "input": result.input = e.value of "outputsub": - result.action = actionRun - result.outp = e.value - result.substr = true + result.outputCheck = ocSubstr + result.outp = strip(e.value) of "sortoutput": result.sortoutput = parseCfgBool(e.value) of "exitcode": diff --git a/testament/tester.nim b/testament/tester.nim index 59c0171b4..1c105c97f 100644 --- a/testament/tester.nim +++ b/testament/tester.nim @@ -328,11 +328,6 @@ proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) = given.err = reMsgsDiffer return -proc makeDeterministic(s: string): string = - var x = splitLines(s) - sort(x, system.cmp) - result = join(x, "\n") - proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, expected: TSpec; r: var TResults) = var expectedmsg: string = "" @@ -350,27 +345,31 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, if given.err == reSuccess: inc(r.passed) r.addResult(test, target, expectedmsg, givenmsg, given.err) -proc testSpec(r: var TResults, test: TTest, target = targetC) = +proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = let tname = test.name.addFileExt(".nim") var expected: TSpec if test.action != actionRunNoSpec: expected = parseSpec(tname) - if test.action == actionRun and expected.action == actionCompile: - expected.action = actionRun else: specDefaults expected expected.action = actionRunNoSpec + expected.outputCheck = ocIgnore # this is default so it is unnecessary if expected.err == reIgnored: - r.addResult(test, target, "", "", reIgnored) + # targetC is a lie + r.addResult(test, targetC, "", "", reIgnored) inc(r.skipped) inc(r.total) return - if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true" and target == targetC and expected.targets == {}: - expected.targets.incl(targetCpp) - elif expected.targets == {}: - expected.targets.incl(target) + expected.targets.incl targets + + # still no target specified at all + if expected.targets == {}: + if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true": + expected.targets = {targetCpp} + else: + expected.targets = {targetC} for target in expected.targets: inc(r.total) @@ -433,9 +432,13 @@ proc testSpec(r: var TResults, test: TTest, target = targetC) = # to return other codes, but for us it is sufficient to know that it's not 0. if exitCode != 0: exitCode = 1 - let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string)) - else: strip(buf.string) - let expectedOut = strip(expected.outp) + let bufB = + if expected.sortoutput: + var x = splitLines(strip(buf.string)) + sort(x, system.cmp) + join(x, "\n") + else: + strip(buf.string) if exitCode != expected.exitCode: r.addResult(test, target, "exitcode: " & $expected.exitCode, @@ -443,11 +446,11 @@ proc testSpec(r: var TResults, test: TTest, target = targetC) = bufB, reExitCodesDiffer) continue - if bufB != expectedOut and expected.action != actionRunNoSpec: - if not (expected.substr and expectedOut in bufB): - given.err = reOutputsDiffer - r.addResult(test, target, expected.outp, bufB, reOutputsDiffer) - continue + if (expected.outputCheck == ocEqual and bufB != expected.outp) or + (expected.outputCheck == ocSubstr and bufB notin expected.outp): + given.err = reOutputsDiffer + r.addResult(test, target, expected.outp, bufB, reOutputsDiffer) + continue compilerOutputTests(test, target, given, expected, r) continue @@ -495,11 +498,10 @@ proc testExec(r: var TResults, test: TTest) = if given.err == reSuccess: inc(r.passed) r.addResult(test, targetC, "", given.msg, given.err) -proc makeTest(test, options: string, cat: Category, action = actionCompile, +proc makeTest(test, options: string, cat: Category, env: string = ""): TTest = # start with 'actionCompile', will be overwritten in the spec: - result = TTest(cat: cat, name: test, options: options, - action: action, startTime: epochTime()) + result = TTest(cat: cat, name: test, options: options, startTime: epochTime()) when defined(windows): const |