diff options
Diffstat (limited to 'tests/caasdriver.nim')
| -rw-r--r-- | tests/caasdriver.nim | 158 |
1 files changed, 130 insertions, 28 deletions
diff --git a/tests/caasdriver.nim b/tests/caasdriver.nim index 3f3d3671e..2de046a11 100644 --- a/tests/caasdriver.nim +++ b/tests/caasdriver.nim @@ -1,26 +1,75 @@ import osproc, streams, os, strutils, re +## Compiler as a service tester. +## +## This test cases uses the txt files in the caas/ subdirectory. +## +## Each of the text files inside encodes a session with the compiler: +## +## The first line indicates the main project file. +## +## Lines starting with '>' indicate a command to be sent to the compiler and +## the lines following a command include checks for expected or forbidden +## output (! for forbidden). +## +## If a line starts with '#' it will be ignored completely, so you can use that +## for comments. +## +## All the tests are run both in ProcRun (each command creates a separate +## process) and CaasRun (first command starts up a server and it is reused for +## the rest) modes. Since some cases are specific to either ProcRun or CaasRun +## modes, you can prefix a line with the mode and the line will be processed +## only in that mode. +## +## The rest of the line is treated as a regular expression, so be careful +## escaping metacharacters like parenthesis. +## +## You can optionally pass parameters at the command line to modify the +## behaviour of the test suite. By default only tests which fail will be echoed +## to stdout. If you want to see all the output pass the word "verbose" as a +## parameter. +## +## If you don't want to run all the test case files, you can pass any substring +## as a parameter. Only files matching the passed substring will be run. The +## filtering doesn't use any globbing metacharacters, it's a plain match. +## +## Example to run only "*-compile*.txt" tests in verbose mode: +## +## ./caasdriver verbose -compile + + type + TRunMode = enum + ProcRun, CaasRun + TNimrodSession* = object - nim: PProcess + nim: PProcess # Holds the open process for CaasRun sessions, nil otherwise. + mode: TRunMode # Stores the type of run mode the session was started with. + lastOutput: string # Preserves the last output, needed for ProcRun mode. + filename: string # Appended to each command starting with '>'. -proc dirname(path: string): string = path.splitPath()[0] +const modes = [CaasRun, ProcRun] var TesterDir = getAppDir() NimrodBin = TesterDir / "../bin/nimrod" -proc startNimrodSession*(project: string): TNimrodSession = - result.nim = startProcess(NimrodBin, - workingDir = project.dirname, - args = ["serve", "--server.type:stdin", project]) +proc startNimrodSession(project: string, mode: TRunMode): TNimrodSession = + let (dir, name) = project.SplitPath + result.mode = mode + result.lastOutput = "" + result.filename = name + if mode == CaasRun: + result.nim = startProcess(NimrodBin, workingDir = dir, + args = ["serve", "--server.type:stdin", name]) -proc doCommand*(session: var TNimrodSession, command: string): string = +proc doCaasCommand(session: var TNimrodSession, command: string): string = + assert session.mode == CaasRun session.nim.inputStream.write(command & "\n") session.nim.inputStream.flush - + result = "" - + while true: var line = TaintedString("") if session.nim.outputStream.readLine(line): @@ -30,30 +79,66 @@ proc doCommand*(session: var TNimrodSession, command: string): string = result = "FAILED TO EXECUTE: " & command & "\n" & result break +proc doProcCommand(session: var TNimrodSession, command: string): string = + assert session.mode == ProcRun + except: result = "FAILED TO EXECUTE: " & command & "\n" & result + var + process = startProcess(NimrodBin, args = command.split) + stream = outputStream(process) + line = TaintedString("") + + result = "" + while stream.readLine(line): + if result.len > 0: result &= "\n" + result &= line.string + + process.close() + +proc doCommand(session: var TNimrodSession, command: string) = + if session.mode == CaasRun: + session.lastOutput = doCaasCommand(session, + command & " " & session.filename) + else: + session.lastOutput = doProcCommand(session, + command & " " & session.filename) + proc close(session: var TNimrodSession) {.destructor.} = - session.nim.close + if session.mode == CaasRun: + session.nim.close -proc doScenario(script: string, output: PStream): bool = +proc doScenario(script: string, output: PStream, mode: TRunMode): bool = result = true var f = open(script) var project = TaintedString("") - + if f.readLine(project): var - s = startNimrodSession(script.dirname / project.string) + s = startNimrodSession(script.parentDir / project.string, mode) tline = TaintedString("") - lastOutput = "" ln = 1 while f.readLine(tline): var line = tline.string inc ln + + # Filter lines by run mode, removing the prefix if the mode is current. + for testMode in modes: + if line.startsWith($testMode): + if testMode != mode: + line = "" + else: + line = line[len($testMode)..len(line) - 1].strip + break + if line.strip.len == 0: continue - if line.startsWith(">"): - lastOutput = s.doCommand(line.substr(1).strip) - output.writeln line, "\n", lastOutput + if line.startsWith("#"): + output.writeln line + continue + elif line.startsWith(">"): + s.doCommand(line.substr(1).strip) + output.writeln line, "\n", s.lastOutput else: var expectMatch = true var pattern = line @@ -61,7 +146,8 @@ proc doScenario(script: string, output: PStream): bool = pattern = line.substr(1).strip expectMatch = false - var actualMatch = lastOutput.find(re(pattern)) != -1 + let actualMatch = + s.lastOutput.find(re(pattern, flags = {reStudy})) != -1 if expectMatch == actualMatch: output.writeln "SUCCESS ", line @@ -70,17 +156,33 @@ proc doScenario(script: string, output: PStream): bool = result = false iterator caasTestsRunner*(filter = ""): tuple[test, output: string, - status: bool] = + status: bool, mode: TRunMode] = for scenario in os.walkFiles(TesterDir / "caas/*.txt"): if filter.len > 0 and find(scenario, filter) == -1: continue - var outStream = newStringStream() - let r = doScenario(scenario, outStream) - yield (scenario, outStream.data, r) + for mode in modes: + var outStream = newStringStream() + let r = doScenario(scenario, outStream, mode) + yield (scenario, outStream.data, r, mode) when isMainModule: - var filter = "" - if paramCount() > 0: filter = paramStr(1) - - for t, o, r in caasTestsRunner(filter): - echo t, "\n", o - + var + filter = "" + failures = 0 + verbose = false + + for i in 0..ParamCount() - 1: + let param = paramStr(i + 1) + case param + of "verbose": verbose = true + else: filter = param + + if verbose and len(filter) > 0: + echo "Running only test cases matching filter '$1'" % [filter] + + for test, output, result, mode in caasTestsRunner(filter): + if not result or verbose: + echo test, "\n", output, "-> ", $mode, ":", $result, "\n-----" + if not result: + failures += 1 + + quit(failures) |