diff options
Diffstat (limited to 'testament/specs.nim')
| -rw-r--r-- | testament/specs.nim | 429 |
1 files changed, 334 insertions, 95 deletions
diff --git a/testament/specs.nim b/testament/specs.nim index 70be6990c..c3040c1d8 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -7,7 +7,17 @@ # distribution, for details about the copyright. # -import sequtils, parseutils, strutils, os, streams, parsecfg +import sequtils, parseutils, strutils, os, streams, parsecfg, + tables, hashes, sets +import compiler/platform + +type TestamentData* = ref object + # better to group globals under 1 object; could group the other ones here too + batchArg*: string + testamentNumBatch*: int + testamentBatch*: int + +let testamentData0* = TestamentData() var compilerPrefix* = findExe("nim") @@ -25,60 +35,77 @@ type TOutputCheck* = enum ocIgnore = "ignore" - ocEqual = "equal" + ocEqual = "equal" ocSubstr = "substr" TResultEnum* = enum - reNimcCrash, # nim compiler seems to have crashed - reMsgsDiffer, # error messages differ - reFilesDiffer, # expected and given filenames differ - reLinesDiffer, # expected and given line numbers differ + reNimcCrash, # nim compiler seems to have crashed + reMsgsDiffer, # error messages differ + reFilesDiffer, # expected and given filenames differ + reLinesDiffer, # expected and given line numbers differ reOutputsDiffer, - reExitcodesDiffer, + reExitcodesDiffer, # exit codes of program or of valgrind differ reTimeout, reInvalidPeg, reCodegenFailure, reCodeNotFound, reExeNotFound, - reInstallFailed # package installation failed - reBuildFailed # package building failed - reDisabled, # test is disabled - reJoined, # test is disabled because it was joined into the megatest - reSuccess # test was successful - reInvalidSpec # test had problems to parse the spec + reInstallFailed # package installation failed + reBuildFailed # package building failed + reDisabled, # test is disabled + reJoined, # test is disabled because it was joined into the megatest + reSuccess # test was successful + reInvalidSpec # test had problems to parse the spec TTarget* = enum - targetC = "C" - targetCpp = "C++" - targetObjC = "ObjC" - targetJS = "JS" + targetC = "c" + targetCpp = "cpp" + targetObjC = "objc" + targetJS = "js" + + InlineError* = object + kind*: string + msg*: string + line*, col*: int + + ValgrindSpec* = enum + disabled, enabled, leaking TSpec* = object + # xxx make sure `isJoinableSpec` takes into account each field here. action*: TTestAction file*, cmd*: string + filename*: string ## Test filename (without path). input*: string outputCheck*: TOutputCheck sortoutput*: bool output*: string line*, column*: int - tfile*: string - tline*, tcolumn*: int exitCode*: int msg*: string - ccodeCheck*: string + ccodeCheck*: seq[string] maxCodeSize*: int err*: TResultEnum + inCurrentBatch*: bool targets*: set[TTarget] + matrix*: seq[string] nimout*: string - parseErrors*: string # when the spec definition is invalid, this is not empty. + nimoutFull*: bool # whether nimout is all compiler output or a subset + parseErrors*: string # when the spec definition is invalid, this is not empty. unjoinable*: bool - useValgrind*: bool + unbatchable*: bool + # whether this test can be batchable via `NIM_TESTAMENT_BATCH`; only very + # few tests are not batchable; the ones that are not could be turned batchable + # by making the dependencies explicit + useValgrind*: ValgrindSpec timeout*: float # in seconds, fractions possible, - # but don't rely on much precision + # but don't rely on much precision + inlineErrors*: seq[InlineError] # line information to error message + debugInfo*: string # debug info to give more context proc getCmd*(s: TSpec): string = if s.cmd.len == 0: - result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:tests/deps $options $file" + result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:build/deps/pkgs2 $options $file" else: result = s.cmd @@ -86,6 +113,13 @@ const targetToExt*: array[TTarget, string] = ["nim.c", "nim.cpp", "nim.m", "js"] targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"] +proc defaultOptions*(a: TTarget): string = + case a + of targetJS: "-d:nodejs" + # once we start testing for `nim js -d:nimbrowser` (eg selenium or similar), + # we can adapt this logic; or a given js test can override with `-u:nodejs`. + else: "" + when not declared(parseCfgBool): # candidate for the stdlib: proc parseCfgBool(s: string): bool = @@ -94,21 +128,174 @@ when not declared(parseCfgBool): of "n", "no", "false", "0", "off": result = false else: raise newException(ValueError, "cannot interpret as a bool: " & s) -proc extractSpec(filename: string): string = - const tripleQuote = "\"\"\"" - var x = readFile(filename).string - var a = x.find(tripleQuote) - var b = x.find(tripleQuote, a+3) - # look for """ only in the first section - if a >= 0 and b > a and a < 40: - result = x.substr(a+3, b-1).replace("'''", tripleQuote) +proc addLine*(self: var string; pieces: varargs[string]) = + for piece in pieces: + self.add piece + self.add "\n" + + +const + inlineErrorKindMarker = "tt." + inlineErrorMarker = "#[" & inlineErrorKindMarker + +proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int = + ## Extract inline error messages. + ## + ## Can parse a single message for a line: + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name] ]# + ## ``` + ## + ## Can parse multiple messages for a line when they are separated by ';': + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error + ## ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error + ## ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name]; + ## tt.Error ^ 'no_destroy' should be: 'nodestroy' [Name]; + ## tt.Error ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` + result = i + len(inlineErrorMarker) + inc col, len(inlineErrorMarker) + let msgLine = line + var msgCol = -1 + var msg = "" + var kind = "" + + template parseKind = + while result < s.len and s[result] in IdentChars: + kind.add s[result] + inc result + inc col + if kind notin ["Hint", "Warning", "Error"]: + spec.parseErrors.addLine "expected inline message kind: Hint, Warning, Error" + + template skipWhitespace = + while result < s.len and s[result] in Whitespace: + if s[result] == '\n': + col = 1 + inc line + else: + inc col + inc result + + template parseCaret = + if result < s.len and s[result] == '^': + msgCol = col + inc result + inc col + skipWhitespace() + else: + spec.parseErrors.addLine "expected column marker ('^') for inline message" + + template isMsgDelimiter: bool = + s[result] == ';' and + (block: + let nextTokenIdx = result + 1 + parseutils.skipWhitespace(s, result + 1) + if s.len > nextTokenIdx + len(inlineErrorKindMarker) and + s[nextTokenIdx..(nextTokenIdx + len(inlineErrorKindMarker) - 1)] == inlineErrorKindMarker: + true + else: + false) + + template trimTrailingMsgWhitespace = + while msg.len > 0 and msg[^1] in Whitespace: + setLen msg, msg.len - 1 + + template addInlineError = + doAssert msg[^1] notin Whitespace + if kind == "Error": spec.action = actionReject + spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: msgLine, col: msgCol) + + parseKind() + skipWhitespace() + parseCaret() + + while result < s.len-1: + if s[result] == '\n': + if result > 0 and s[result - 1] == '\r': + msg[^1] = '\n' + else: + msg.add '\n' + inc result + inc line + col = 1 + elif isMsgDelimiter(): + trimTrailingMsgWhitespace() + inc result + skipWhitespace() + addInlineError() + inc result, len(inlineErrorKindMarker) + inc col, 1 + len(inlineErrorKindMarker) + kind.setLen 0 + msg.setLen 0 + parseKind() + skipWhitespace() + parseCaret() + elif s[result] == ']' and s[result+1] == '#': + trimTrailingMsgWhitespace() + inc result, 2 + inc col, 2 + addInlineError() + break + else: + msg.add s[result] + inc result + inc col + + if spec.inlineErrors.len > 0: + spec.unjoinable = true + +proc extractSpec(filename: string; spec: var TSpec): string = + const + tripleQuote = "\"\"\"" + specStart = "discard " & tripleQuote + var s = readFile(filename) + + var i = 0 + var a = -1 + var b = -1 + var line = 1 + var col = 1 + while i < s.len: + if (i == 0 or s[i-1] != ' ') and s.continuesWith(specStart, i): + # `s[i-1] == '\n'` would not work because of `tests/stdlib/tbase64.nim` which contains BOM (https://en.wikipedia.org/wiki/Byte_order_mark) + const lineMax = 10 + if a != -1: + raise newException(ValueError, "testament spec violation: duplicate `specStart` found: " & $(filename, a, b, line)) + elif line > lineMax: + # not overly restrictive, but prevents mistaking some `specStart` as spec if deeep inside a test file + raise newException(ValueError, "testament spec violation: `specStart` should be before line $1, or be indented; info: $2" % [$lineMax, $(filename, a, b, line)]) + i += specStart.len + a = i + elif a > -1 and b == -1 and s.continuesWith(tripleQuote, i): + b = i + i += tripleQuote.len + elif s[i] == '\n': + inc line + inc i + col = 1 + elif s.continuesWith(inlineErrorMarker, i): + i = extractErrorMsg(s, i, line, col, spec) + else: + inc col + inc i + + if a >= 0 and b > a: + result = s.substr(a, b-1).multiReplace({"'''": tripleQuote, "\\31": "\31"}) + elif a >= 0: + raise newException(ValueError, "testament spec violation: `specStart` found but not trailing `tripleQuote`: $1" % $(filename, a, b, line)) else: - #echo "warning: file does not contain spec: " & filename result = "" -when not defined(nimhygiene): - {.pragma: inject.} - proc parseTargets*(value: string): set[TTarget] = for v in value.normalize.splitWhitespace: case v @@ -116,31 +303,38 @@ proc parseTargets*(value: string): set[TTarget] = of "cpp", "c++": result.incl(targetCpp) of "objc": result.incl(targetObjC) of "js": result.incl(targetJS) - else: echo "target ignored: " & v - -proc addLine*(self: var string; a: string) = - self.add a - self.add "\n" - -proc addLine*(self: var string; a,b: string) = - self.add a - self.add b - self.add "\n" + else: raise newException(ValueError, "invalid target: '$#'" % v) proc initSpec*(filename: string): TSpec = result.file = filename +proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool = + if testamentData.testamentNumBatch != 0: + hash(filename) mod testamentData.testamentNumBatch == testamentData.testamentBatch + else: + true + proc parseSpec*(filename: string): TSpec = result.file = filename - let specStr = extractSpec(filename) + result.filename = extractFilename(filename) + let specStr = extractSpec(filename, result) var ss = newStringStream(specStr) var p: CfgParser open(p, ss, filename, 1) + var flags: HashSet[string] + var nimoutFound = false while true: var e = next(p) case e.kind of cfgKeyValuePair: - case normalize(e.key) + let key = e.key.normalize + const whiteListMulti = ["disabled", "ccodecheck"] + ## list of flags that are correctly handled when passed multiple times + ## (instead of being overwritten) + if key notin whiteListMulti: + doAssert key notin flags, $(key, filename) + flags.incl key + case key of "action": case e.value.normalize of "compile": @@ -163,16 +357,10 @@ proc parseSpec*(filename: string): TSpec = if result.msg.len == 0 and result.nimout.len == 0: result.parseErrors.addLine "errormsg or msg needs to be specified before column" discard parseInt(e.value, result.column) - of "tfile": - result.tfile = e.value - of "tline": - discard parseInt(e.value, result.tline) - of "tcolumn": - discard parseInt(e.value, result.tcolumn) of "output": if result.outputCheck != ocSubstr: result.outputCheck = ocEqual - result.output = strip(e.value) + result.output = e.value of "input": result.input = e.value of "outputsub": @@ -180,71 +368,108 @@ proc parseSpec*(filename: string): TSpec = result.output = strip(e.value) of "sortoutput": try: - result.sortoutput = parseCfgBool(e.value) + result.sortoutput = parseCfgBool(e.value) except: result.parseErrors.addLine getCurrentExceptionMsg() of "exitcode": discard parseInt(e.value, result.exitCode) result.action = actionRun - of "msg": - result.msg = e.value - if result.action != actionRun: - result.action = actionCompile - of "errormsg", "errmsg": + of "errormsg": result.msg = e.value result.action = actionReject of "nimout": result.nimout = e.value + nimoutFound = true + of "nimoutfull": + result.nimoutFull = parseCfgBool(e.value) + of "batchable": + result.unbatchable = not parseCfgBool(e.value) of "joinable": result.unjoinable = not parseCfgBool(e.value) of "valgrind": when defined(linux) and sizeof(int) == 8: - result.useValgrind = parseCfgBool(e.value) + result.useValgrind = if e.value.normalize == "leaks": leaking + else: ValgrindSpec(parseCfgBool(e.value)) result.unjoinable = true - if result.useValgrind: + if result.useValgrind != disabled: result.outputCheck = ocSubstr else: # Windows lacks valgrind. Silly OS. # Valgrind only supports OSX <= 17.x - result.useValgrind = false + result.useValgrind = disabled of "disabled": - case e.value.normalize + let value = e.value.normalize + case value of "y", "yes", "true", "1", "on": result.err = reDisabled of "n", "no", "false", "0", "off": discard - of "win", "windows": + # These values are defined in `compiler/options.isDefined` + of "win": when defined(windows): result.err = reDisabled of "linux": when defined(linux): result.err = reDisabled of "bsd": when defined(bsd): result.err = reDisabled - of "macosx": - when defined(macosx): result.err = reDisabled - of "unix": - when defined(unix): result.err = reDisabled - of "posix": + of "osx": + when defined(osx): result.err = reDisabled + of "unix", "posix": when defined(posix): result.err = reDisabled - of "travis": + of "freebsd": + when defined(freebsd): result.err = reDisabled + of "littleendian": + when defined(littleendian): result.err = reDisabled + of "bigendian": + when defined(bigendian): result.err = reDisabled + of "cpu8", "8bit": + when defined(cpu8): result.err = reDisabled + of "cpu16", "16bit": + when defined(cpu16): result.err = reDisabled + of "cpu32", "32bit": + when defined(cpu32): result.err = reDisabled + of "cpu64", "64bit": + when defined(cpu64): result.err = reDisabled + # These values are for CI environments + of "travis": # deprecated if isTravis: result.err = reDisabled - of "appveyor": + of "appveyor": # deprecated if isAppVeyor: result.err = reDisabled of "azure": if isAzure: result.err = reDisabled - of "32bit": - if sizeof(int) == 4: - result.err = reDisabled - of "freebsd": - when defined(freebsd): result.err = reDisabled - of "arm64": - when defined(arm64): result.err = reDisabled else: - result.parseErrors.addLine "cannot interpret as a bool: ", e.value + # Check whether the value exists as an OS or CPU that is + # defined in `compiler/platform`. + block checkHost: + for os in platform.OS: + # Check if the value exists as OS. + if value == os.name.normalize: + # The value exists; is it the same as the current host? + if value == hostOS.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + for cpu in platform.CPU: + # Check if the value exists as CPU. + if value == cpu.name.normalize: + # The value exists; is it the same as the current host? + if value == hostCPU.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + # The value doesn't exist as an OS, CPU, or any previous value + # defined in this case statement, so raise an error. + result.parseErrors.addLine "cannot interpret as a bool: ", e.value of "cmd": if e.value.startsWith("nim "): result.cmd = compilerPrefix & e.value[3..^1] else: result.cmd = e.value of "ccodecheck": - result.ccodeCheck = e.value + result.ccodeCheck.add e.value of "maxcodesize": discard parseInt(e.value, result.maxCodeSize) of "timeout": @@ -252,19 +477,14 @@ proc parseSpec*(filename: string): TSpec = result.timeout = parseFloat(e.value) except ValueError: result.parseErrors.addLine "cannot interpret as a float: ", e.value - of "target", "targets": - for v in e.value.normalize.splitWhitespace: - case v - of "c": - result.targets.incl(targetC) - of "cpp", "c++": - result.targets.incl(targetCpp) - of "objc": - result.targets.incl(targetObjC) - of "js": - result.targets.incl(targetJS) - else: - result.parseErrors.addLine "cannot interpret as a target: ", e.value + of "targets", "target": + try: + result.targets.incl parseTargets(e.value) + except ValueError as e: + result.parseErrors.addLine e.msg + of "matrix": + for v in e.value.split(';'): + result.matrix.add(v.strip) else: result.parseErrors.addLine "invalid key for test spec: ", e.key @@ -280,3 +500,22 @@ proc parseSpec*(filename: string): TSpec = if skips.anyIt(it in result.file): result.err = reDisabled + + if nimoutFound and result.nimout.len == 0 and not result.nimoutFull: + result.parseErrors.addLine "empty `nimout` is vacuously true, use `nimoutFull:true` if intentional" + + result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable + if not result.inCurrentBatch: + result.err = reDisabled + + # Interpolate variables in msgs: + template varSub(msg: string): string = + try: + msg % ["/", $DirSep, "file", result.filename] + except ValueError: + result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unit-tests-output-message-variable-interpolation')" + msg + result.nimout = result.nimout.varSub + result.msg = result.msg.varSub + for inlineError in result.inlineErrors.mitems: + inlineError.msg = inlineError.msg.varSub |