Runs all caas tests also in process mode.

This requires changing a little bit the test cases to avoid specifying
the filename as the last parameter. The trackDirty switch still doesn't
work for process mode. Also revers some SuccessX which I think were
hidden in the amount of output verbosity.

6 files changed, 67 insertions, 32 deletions

diff --git a/tests/caas/compile-suggest.txt b/tests/caas/compile-suggest.txt
index 4e2ab9729..3cc0ee0cb 100644
--- a/tests/caas/compile-suggest.txt
+++ b/tests/caas/compile-suggest.txt
@@ -1,7 +1,7 @@
 main.nim
 > c
 SuccessX
-> idetools --trackDirty:main_dirty.nim,main.nim,12,7 --suggest main.nim
+> idetools --trackDirty:main_dirty.nim,main.nim,12,7 --suggest
 skField\tx
 skField\ty
 
diff --git a/tests/caas/compile-then-def.txt b/tests/caas/compile-then-def.txt
index a5080c50d..839ca1532 100644
--- a/tests/caas/compile-then-def.txt
+++ b/tests/caas/compile-then-def.txt
@@ -1,9 +1,9 @@
 main.nim
 > c
 SuccessX
-> idetools --track:main.nim,5,18 --def main.nim
+> idetools --track:main.nim,5,18 --def
 strutils.toUpper
-SuccessX
-> idetools --track:main.nim,5,18 --def main.nim
+! SuccessX
+> idetools --track:main.nim,5,18 --def
 strutils.toUpper
-SuccessX
+! SuccessX
diff --git a/tests/caas/def-def-compile.txt b/tests/caas/def-def-compile.txt
index 64002aff1..05ea5a1aa 100644
--- a/tests/caas/def-def-compile.txt
+++ b/tests/caas/def-def-compile.txt
@@ -1,10 +1,10 @@
 main.nim
-> idetools --track:main.nim,5,18 --def main.nim
+> idetools --track:main.nim,5,18 --def
 strutils.toUpper
-SuccessX
-> idetools --track:main.nim,5,18 --def main.nim
+! SuccessX
+> idetools --track:main.nim,5,18 --def
 strutils.toUpper
-SuccessX
+! SuccessX
 > c
 SuccessX
 
diff --git a/tests/caas/def-then-compile.txt b/tests/caas/def-then-compile.txt
index efa4dcbb3..7e755cff2 100644
--- a/tests/caas/def-then-compile.txt
+++ b/tests/caas/def-then-compile.txt
@@ -1,7 +1,7 @@
 main.nim
-> idetools --track:main.nim,5,18 --def main.nim
+> idetools --track:main.nim,5,18 --def
 strutils.toUpper
-SuccessX
+! SuccessX
 > c
 SuccessX
 
diff --git a/tests/caas/suggest-compile.txt b/tests/caas/suggest-compile.txt
index 49d0dc431..76756c86e 100644
--- a/tests/caas/suggest-compile.txt
+++ b/tests/caas/suggest-compile.txt
@@ -1,5 +1,5 @@
 main.nim
-> idetools --trackDirty:main_dirty.nim,main.nim,12,7 --suggest main.nim
+> idetools --trackDirty:main_dirty.nim,main.nim,12,7 --suggest
 skField\tx
 skField\ty
 > c
diff --git a/tests/caasdriver.nim b/tests/caasdriver.nim
index 9d3e91b22..2f482a810 100644
--- a/tests/caasdriver.nim
+++ b/tests/caasdriver.nim
@@ -23,19 +23,30 @@ import osproc, streams, os, strutils, re
 
 
 type
+  TRunMode = enum
+    ProcRun, CaasRun
+
   TNimrodSession* = object
-    nim: PProcess
+    nim: PProcess # Holds the open process for CaasRun sessions, nil otherwise.
+    mode: TRunMode # Stores the type of run mode the session was started with.
+    lastOutput: string # Preserves the last output, needed for ProcRun mode.
+    filename: string # Appended to each command starting with '>'.
 
 var
   TesterDir = getAppDir()
   NimrodBin = TesterDir / "../bin/nimrod"
 
-proc startNimrodSession*(project: string): TNimrodSession =
-  result.nim = startProcess(NimrodBin,
-    workingDir = project.parentDir,
-    args = ["serve", "--server.type:stdin", project])
-
-proc doCommand*(session: var TNimrodSession, command: string): string =
+proc startNimrodSession(project: string, mode: TRunMode): TNimrodSession =
+  let (dir, name) = project.SplitPath
+  result.mode = mode
+  result.lastOutput = ""
+  result.filename = name
+  if mode == CaasRun:
+    result.nim = startProcess(NimrodBin, workingDir = dir,
+      args = ["serve", "--server.type:stdin", name])
+
+proc doCaasCommand(session: var TNimrodSession, command: string): string =
+  assert session.mode == CaasRun
   session.nim.inputStream.write(command & "\n")
   session.nim.inputStream.flush
 
@@ -50,10 +61,34 @@ proc doCommand*(session: var TNimrodSession, command: string): string =
       result = "FAILED TO EXECUTE: " & command & "\n" & result
       break
 
+proc doProcCommand(session: var TNimrodSession, command: string): string =
+  assert session.mode == ProcRun
+  except: result = "FAILED TO EXECUTE: " & command & "\n" & result
+  var
+    process = startProcess(NimrodBin, args = command.split)
+    stream = outputStream(process)
+    line = TaintedString("")
+
+  result = ""
+  while stream.readLine(line):
+    if result.len > 0: result &= "\n"
+    result &= line.string
+
+  process.close()
+
+proc doCommand(session: var TNimrodSession, command: string) =
+  if session.mode == CaasRun:
+    session.lastOutput = doCaasCommand(session,
+                                       command & " " & session.filename)
+  else:
+    session.lastOutput = doProcCommand(session,
+                                       command & " " & session.filename)
+
 proc close(session: var TNimrodSession) {.destructor.} =
-  session.nim.close
+  if session.mode == CaasRun:
+    session.nim.close
 
-proc doScenario(script: string, output: PStream): bool =
+proc doScenario(script: string, output: PStream, mode: TRunMode): bool =
   result = true
 
   var f = open(script)
@@ -61,9 +96,8 @@ proc doScenario(script: string, output: PStream): bool =
 
   if f.readLine(project):
     var
-      s = startNimrodSession(script.parentDir / project.string)
+      s = startNimrodSession(script.parentDir / project.string, mode)
       tline = TaintedString("")
-      lastOutput = ""
       ln = 1
 
     while f.readLine(tline):
@@ -72,8 +106,8 @@ proc doScenario(script: string, output: PStream): bool =
       if line.strip.len == 0: continue
 
       if line.startsWith(">"):
-        lastOutput = s.doCommand(line.substr(1).strip)
-        output.writeln line, "\n", lastOutput
+        s.doCommand(line.substr(1).strip)
+        output.writeln line, "\n", s.lastOutput
       else:
         var expectMatch = true
         var pattern = line
@@ -81,7 +115,7 @@ proc doScenario(script: string, output: PStream): bool =
           pattern = line.substr(1).strip
           expectMatch = false
 
-        var actualMatch = lastOutput.find(re(pattern)) != -1
+        var actualMatch = s.lastOutput.find(re(pattern)) != -1
 
         if expectMatch == actualMatch:
           output.writeln "SUCCESS ", line
@@ -90,12 +124,13 @@ proc doScenario(script: string, output: PStream): bool =
           result = false
 
 iterator caasTestsRunner*(filter = ""): tuple[test, output: string,
-                                              status: bool] =
+                                              status: bool, mode: TRunMode] =
   for scenario in os.walkFiles(TesterDir / "caas/*.txt"):
     if filter.len > 0 and find(scenario, filter) == -1: continue
-    var outStream = newStringStream()
-    let r = doScenario(scenario, outStream)
-    yield (scenario, outStream.data, r)
+    for mode in [CaasRun, ProcRun]:
+      var outStream = newStringStream()
+      let r = doScenario(scenario, outStream, mode)
+      yield (scenario, outStream.data, r, mode)
 
 when isMainModule:
   var
@@ -112,9 +147,9 @@ when isMainModule:
   if verbose and len(filter) > 0:
     echo "Running only test cases matching filter '$1'" % [filter]
 
-  for test, output, result in caasTestsRunner(filter):
+  for test, output, result, mode in caasTestsRunner(filter):
     if not result or verbose:
-      echo test, "\n", output, "-> ", $result, "\n-----"
+      echo test, "\n", output, "-> ", $mode, ":", $result, "\n-----"
     if not result:
       failures += 1