Testament pre parallel (#9137)

* testament: move to root dir (it's not a test)

* osproc: fix process index passed to afterRunEvent for parallel runs

it was passing the index of the process, not index of all commands

* testament: complete file move

1 files changed, 195 insertions, 0 deletions

diff --git a/testament/caasdriver.nim b/testament/caasdriver.nim
new file mode 100644
index 000000000..30383bddb
--- /dev/null
+++ b/testament/caasdriver.nim
@@ -0,0 +1,195 @@
+import osproc, streams, os, strutils, re
+{.experimental.}
+
+## Compiler as a service tester.
+##
+## Please read docs/idetools.txt for information about this.
+
+
+type
+  TRunMode = enum
+    ProcRun, CaasRun, SymbolProcRun
+
+  NimSession* = object
+    nim: Process # Holds the open process for CaasRun sessions, nil otherwise.
+    mode: TRunMode # Stores the type of run mode the session was started with.
+    lastOutput: string # Preserves the last output, needed for ProcRun mode.
+    filename: string # Appended to each command starting with '>'. Also a var.
+    modname: string # Like filename but without extension.
+    nimcache: string # Input script based name for the nimcache dir.
+
+const
+  modes = [CaasRun, ProcRun, SymbolProcRun]
+  filenameReplaceVar = "$TESTNIM"
+  moduleReplaceVar = "$MODULE"
+  silentReplaceVar = "$SILENT"
+  silentReplaceText = "--verbosity:0 --hints:off"
+
+var
+  TesterDir = getAppDir() / ".."
+  NimBin = TesterDir / "../bin/nim"
+
+proc replaceVars(session: var NimSession, text: string): string =
+  result = text.replace(filenameReplaceVar, session.filename)
+  result = result.replace(moduleReplaceVar, session.modname)
+  result = result.replace(silentReplaceVar, silentReplaceText)
+
+proc startNimSession(project, script: string, mode: TRunMode):
+                        NimSession =
+  let (dir, name, ext) = project.splitFile
+  result.mode = mode
+  result.lastOutput = ""
+  result.filename = name & ext
+  result.modname = name
+
+  let (nimcacheDir, nimcacheName, nimcacheExt) = script.splitFile
+  result.nimcache = "SymbolProcRun." & nimcacheName
+
+  if mode == SymbolProcRun:
+    removeDir(nimcacheDir / result.nimcache)
+  else:
+    removeDir(nimcacheDir / "nimcache")
+
+  if mode == CaasRun:
+    result.nim = startProcess(NimBin, workingDir = dir,
+      args = ["serve", "--server.type:stdin", name])
+
+proc doCaasCommand(session: var NimSession, command: string): string =
+  assert session.mode == CaasRun
+  session.nim.inputStream.write(session.replaceVars(command) & "\n")
+  session.nim.inputStream.flush
+
+  result = ""
+
+  while true:
+    var line = TaintedString("")
+    if session.nim.outputStream.readLine(line):
+      if line.string == "": break
+      result.add(line.string & "\n")
+    else:
+      result = "FAILED TO EXECUTE: " & command & "\n" & result
+      break
+
+proc doProcCommand(session: var NimSession, command: string): string =
+  try:
+    assert session.mode == ProcRun or session.mode == SymbolProcRun
+  except:
+    result = "FAILED TO EXECUTE: " & command & "\n" & result
+  var
+    process = startProcess(NimBin, args = session.replaceVars(command).split)
+    stream = outputStream(process)
+    line = TaintedString("")
+
+  result = ""
+  while stream.readLine(line):
+    if result.len > 0: result &= "\n"
+    result &= line.string
+
+  process.close()
+
+proc doCommand(session: var NimSession, command: string) =
+  if session.mode == CaasRun:
+    if not session.nim.running:
+      session.lastOutput = "FAILED TO EXECUTE: " & command & "\n" &
+          "Exit code " & $session.nim.peekExitCode
+      return
+    session.lastOutput = doCaasCommand(session,
+                                       command & " " & session.filename)
+  else:
+    var command = command
+    # For symbol runs we prepend the necessary parameters to avoid clobbering
+    # the normal nimcache.
+    if session.mode == SymbolProcRun:
+      command = "--symbolFiles:on --nimcache:" & session.nimcache &
+                " " & command
+    session.lastOutput = doProcCommand(session,
+                                       command & " " & session.filename)
+
+proc destroy(session: var NimSession) {.destructor.} =
+  if session.mode == CaasRun:
+    session.nim.close
+
+proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): bool =
+  result = true
+
+  var f = open(script)
+  var project = TaintedString("")
+
+  if f.readLine(project):
+    var
+      s = startNimSession(script.parentDir / project.string, script, mode)
+      tline = TaintedString("")
+      ln = 1
+
+    while f.readLine(tline):
+      var line = tline.string
+      inc ln
+
+      # Filter lines by run mode, removing the prefix if the mode is current.
+      for testMode in modes:
+        if line.startsWith($testMode):
+          if testMode != mode:
+            line = ""
+          else:
+            line = line[len($testMode)..len(line) - 1].strip
+          break
+
+      if line.strip.len == 0: continue
+
+      if line.startsWith("#"):
+        output.writeLine line
+        continue
+      elif line.startsWith(">"):
+        s.doCommand(line.substr(1).strip)
+        output.writeLine line, "\n", if verbose: s.lastOutput else: ""
+      else:
+        var expectMatch = true
+        var pattern = s.replaceVars(line)
+        if line.startsWith("!"):
+          pattern = pattern.substr(1).strip
+          expectMatch = false
+
+        let actualMatch =
+          s.lastOutput.find(re(pattern, flags = {reStudy})) != -1
+
+        if expectMatch == actualMatch:
+          output.writeLine "SUCCESS ", line
+        else:
+          output.writeLine "FAILURE ", line
+          result = false
+
+iterator caasTestsRunner*(filter = "", verbose = false): tuple[test,
+                                              output: string, status: bool,
+                                              mode: TRunMode] =
+  for scenario in os.walkFiles(TesterDir / "caas/*.txt"):
+    if filter.len > 0 and find(scenario, filter) == -1: continue
+    for mode in modes:
+      var outStream = newStringStream()
+      let r = doScenario(scenario, outStream, mode, verbose)
+      yield (scenario, outStream.data, r, mode)
+
+when isMainModule:
+  var
+    filter = ""
+    failures = 0
+    verbose = false
+
+  for i in 0..paramCount() - 1:
+    let param = string(paramStr(i + 1))
+    case param
+    of "verbose": verbose = true
+    else: filter = param
+
+  if verbose and len(filter) > 0:
+    echo "Running only test cases matching filter '$1'" % [filter]
+
+  for test, output, result, mode in caasTestsRunner(filter, verbose):
+    if not result or verbose:
+      echo "Mode ", $mode, " (", if result: "succeeded)" else: "failed)"
+      echo test
+      echo output
+      echo "---------\n"
+    if not result:
+      failures += 1
+
+  quit(failures)