summary refs log tree commit diff stats
path: root/testament/testament.nim
diff options
context:
space:
mode:
Diffstat (limited to 'testament/testament.nim')
-rw-r--r--testament/testament.nim806
1 files changed, 806 insertions, 0 deletions
diff --git a/testament/testament.nim b/testament/testament.nim
new file mode 100644
index 000000000..1e892e636
--- /dev/null
+++ b/testament/testament.nim
@@ -0,0 +1,806 @@
+#
+#
+#            Nim Testament
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This program verifies Nim against the testcases.
+
+import
+  std/[strutils, pegs, os, osproc, streams, json,
+    parseopt, browsers, terminal, exitprocs,
+    algorithm, times, intsets, macros]
+
+import backend, specs, azure, htmlgen
+
+from std/sugar import dup
+import compiler/nodejs
+import lib/stdtest/testutils
+from lib/stdtest/specialpaths import splitTestFile
+from std/private/gitutils import diffStrings
+
+import ../dist/checksums/src/checksums/md5
+
+proc trimUnitSep(x: var string) =
+  let L = x.len
+  if L > 0 and x[^1] == '\31':
+    setLen x, L-1
+
+var useColors = true
+var backendLogging = true
+var simulate = false
+var optVerbose = false
+var useMegatest = true
+var valgrindEnabled = true
+
+proc verboseCmd(cmd: string) =
+  if optVerbose:
+    echo "executing: ", cmd
+
+const
+  failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs
+  testsDir = "tests" & DirSep
+  resultsFile = "testresults.html"
+  Usage = """Usage:
+  testament [options] command [arguments]
+
+Command:
+  p|pat|pattern <glob>        run all the tests matching the given pattern
+  all                         run all tests in category folders
+  c|cat|category <category>   run all the tests of a certain category
+  r|run <test>                run single test file
+  html                        generate $1 from the database
+Arguments:
+  arguments are passed to the compiler
+Options:
+  --print                   print results to the console
+  --verbose                 print commands (compiling and running tests)
+  --simulate                see what tests would be run but don't run them (for debugging)
+  --failing                 only show failing/ignored tests
+  --targets:"c cpp js objc" run tests for specified targets (default: c)
+  --nim:path                use a particular nim executable (default: $$PATH/nim)
+  --directory:dir           Change to directory dir before reading the tests or doing anything else.
+  --colors:on|off           Turn messages coloring on|off.
+  --backendLogging:on|off   Disable or enable backend logging. By default turned on.
+  --megatest:on|off         Enable or disable megatest. Default is on.
+  --valgrind:on|off         Enable or disable valgrind support. Default is on.
+  --skipFrom:file           Read tests to skip from `file` - one test per line, # comments ignored
+
+On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API
+provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN.
+
+Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables
+tests with remote networking (as in CI).
+""" % resultsFile
+
+proc isNimRepoTests(): bool =
+  # this logic could either be specific to cwd, or to some file derived from
+  # the input file, eg testament r /pathto/tests/foo/tmain.nim; we choose
+  # the former since it's simpler and also works with `testament all`.
+  let file = "testament"/"testament.nim.cfg"
+  result = file.fileExists
+
+type
+  Category = distinct string
+  TResults = object
+    total, passed, failedButAllowed, skipped: int
+      ## xxx rename passed to passedOrAllowedFailure
+    data: string
+  TTest = object
+    name: string
+    cat: Category
+    options: string
+    testArgs: seq[string]
+    spec: TSpec
+    startTime: float
+    debugInfo: string
+
+# ----------------------------------------------------------------------------
+
+let
+  pegLineError =
+    peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}"
+  pegOtherError = peg"'Error:' \s* {.*}"
+  pegOfInterest = pegLineError / pegOtherError
+
+var gTargets = {low(TTarget)..high(TTarget)}
+var targetsSet = false
+
+proc isSuccess(input: string): bool =
+  # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs
+  # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we
+  # could handle colors, there would be other issues such as handling other flags
+  # that may appear in user config (eg: `--filenames`).
+  # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config
+  # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory
+  input.startsWith("Hint: ") and input.endsWith("[SuccessX]")
+
+proc getFileDir(filename: string): string =
+  result = filename.splitFile().dir
+  if not result.isAbsolute():
+    result = getCurrentDir() / result
+
+proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: string = ""): tuple[
+                cmdLine: string,
+                output: string,
+                exitCode: int] {.tags:
+                [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
+
+  result.cmdLine.add quoteShell(command)
+  for arg in args:
+    result.cmdLine.add ' '
+    result.cmdLine.add quoteShell(arg)
+  verboseCmd(result.cmdLine)
+  var p = startProcess(command, workingDir = workingDir, args = args,
+                       options = {poStdErrToStdOut, poUsePath})
+  var outp = outputStream(p)
+
+  # There is no way to provide input for the child process
+  # anymore. Closing it will create EOF on stdin instead of eternal
+  # blocking.
+  let instream = inputStream(p)
+  instream.write(input)
+  close instream
+
+  result.exitCode = -1
+  var line = newStringOfCap(120)
+  while true:
+    if outp.readLine(line):
+      result.output.add line
+      result.output.add '\n'
+    else:
+      result.exitCode = peekExitCode(p)
+      if result.exitCode != -1: break
+  close(p)
+
+proc nimcacheDir(filename, options: string, target: TTarget): string =
+  ## Give each test a private nimcache dir so they don't clobber each other's.
+  let hashInput = options & $target
+  result = "nimcache" / (filename & '_' & hashInput.getMD5)
+
+proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string,
+                     target: TTarget, extraOptions = ""): string =
+  var options = target.defaultOptions & ' ' & options
+  if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell)
+  options.add ' ' & extraOptions
+  # we avoid using `parseCmdLine` which is buggy, refs bug #14343
+  result = cmdTemplate % ["target", targetToCmd[target],
+                      "options", options, "file", filename.quoteShell,
+                      "filedir", filename.getFileDir(), "nim", compilerPrefix]
+
+proc callNimCompiler(cmdTemplate, filename, options, nimcache: string,
+                     target: TTarget, extraOptions = ""): TSpec =
+  result.cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache, target,
+                          extraOptions)
+  verboseCmd(result.cmd)
+  var p = startProcess(command = result.cmd,
+                       options = {poStdErrToStdOut, poUsePath, poEvalCommand})
+  let outp = p.outputStream
+  var foundSuccessMsg = false
+  var foundErrorMsg = false
+  var err = ""
+  var x = newStringOfCap(120)
+  result.nimout = ""
+  while true:
+    if outp.readLine(x):
+      trimUnitSep x
+      result.nimout.add(x & '\n')
+      if x =~ pegOfInterest:
+        # `err` should contain the last error message
+        err = x
+        foundErrorMsg = true
+      elif x.isSuccess:
+        foundSuccessMsg = true
+    elif not running(p):
+      break
+  result.msg = ""
+  result.file = ""
+  result.output = ""
+  result.line = 0
+  result.column = 0
+
+  result.err = reNimcCrash
+  result.exitCode = p.peekExitCode
+  close p
+  case result.exitCode
+  of 0:
+    if foundErrorMsg:
+      result.debugInfo.add " compiler exit code was 0 but some Error's were found."
+    else:
+      result.err = reSuccess
+  of 1:
+    if not foundErrorMsg:
+      result.debugInfo.add " compiler exit code was 1 but no Error's were found."
+    if foundSuccessMsg:
+      result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true."
+  else:
+    result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode
+
+  if err =~ pegLineError:
+    result.file = extractFilename(matches[0])
+    result.line = parseInt(matches[1])
+    result.column = parseInt(matches[2])
+    result.msg = matches[3]
+  elif err =~ pegOtherError:
+    result.msg = matches[0]
+  trimUnitSep result.msg
+
+proc initResults: TResults =
+  result.total = 0
+  result.passed = 0
+  result.failedButAllowed = 0
+  result.skipped = 0
+  result.data = ""
+
+macro ignoreStyleEcho(args: varargs[typed]): untyped =
+  let typForegroundColor = bindSym"ForegroundColor".getType
+  let typBackgroundColor = bindSym"BackgroundColor".getType
+  let typStyle = bindSym"Style".getType
+  let typTerminalCmd = bindSym"TerminalCmd".getType
+  result = newCall(bindSym"echo")
+  for arg in children(args):
+    if arg.kind == nnkNilLit: continue
+    let typ = arg.getType
+    if typ.kind != nnkEnumTy or
+       typ != typForegroundColor and
+       typ != typBackgroundColor and
+       typ != typStyle and
+       typ != typTerminalCmd:
+      result.add(arg)
+
+template maybeStyledEcho(args: varargs[untyped]): untyped =
+  if useColors:
+    styledEcho(args)
+  else:
+    ignoreStyleEcho(args)
+
+
+proc `$`(x: TResults): string =
+  result = """
+Tests passed or allowed to fail: $2 / $1 <br />
+Tests failed and allowed to fail: $3 / $1 <br />
+Tests skipped: $4 / $1 <br />
+""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped]
+
+proc testName(test: TTest, target: TTarget, extraOptions: string, allowFailure: bool): string =
+  var name = test.name.replace(DirSep, '/')
+  name.add ' ' & $target
+  if allowFailure:
+    name.add " (allowed to fail) "
+  if test.options.len > 0: name.add ' ' & test.options
+  if extraOptions.len > 0: name.add ' ' & extraOptions
+  name.strip()
+
+proc addResult(r: var TResults, test: TTest, target: TTarget,
+               extraOptions, expected, given: string, successOrig: TResultEnum,
+               allowFailure = false, givenSpec: ptr TSpec = nil) =
+  # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need
+  # instead of having to pass individual fields, or abusing existing ones like expected vs given.
+  # test.name is easier to find than test.name.extractFilename
+  # A bit hacky but simple and works with tests/testament/tshould_not_work.nim
+  let name = testName(test, target, extraOptions, allowFailure)
+  let duration = epochTime() - test.startTime
+  let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout
+                else: successOrig
+
+  let durationStr = duration.formatFloat(ffDecimal, precision = 2).align(5)
+  if backendLogging:
+    backend.writeTestResult(name = name,
+                            category = test.cat.string,
+                            target = $target,
+                            action = $test.spec.action,
+                            result = $success,
+                            expected = expected,
+                            given = given)
+  r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
+  template dispNonSkipped(color, outcome) =
+    maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)"
+  template disp(msg) =
+    maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name
+  if success == reSuccess:
+    dispNonSkipped(fgGreen, "PASS: ")
+  elif success == reDisabled:
+    if test.spec.inCurrentBatch: disp("SKIP:")
+    else: disp("NOTINBATCH:")
+  elif success == reJoined: disp("JOINED:")
+  else:
+    dispNonSkipped(fgRed, failString)
+    maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
+    maybeStyledEcho styleBright, fgRed, "Failure: ", $success
+    if givenSpec != nil and givenSpec.debugInfo.len > 0:
+      echo "debugInfo: " & givenSpec.debugInfo
+    if success in {reBuildFailed, reNimcCrash, reInstallFailed}:
+      # expected is empty, no reason to print it.
+      echo given
+    else:
+      maybeStyledEcho fgYellow, "Expected:"
+      maybeStyledEcho styleBright, expected, "\n"
+      maybeStyledEcho fgYellow, "Gotten:"
+      maybeStyledEcho styleBright, given, "\n"
+      echo diffStrings(expected, given).output
+
+  if backendLogging and (isAppVeyor or isAzure):
+    let (outcome, msg) =
+      case success
+      of reSuccess:
+        ("Passed", "")
+      of reDisabled, reJoined:
+        ("Skipped", "")
+      of reBuildFailed, reNimcCrash, reInstallFailed:
+        ("Failed", "Failure: " & $success & '\n' & given)
+      else:
+        ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given)
+    if isAzure:
+      azure.addTestResult(name, test.cat.string, int(duration * 1000), msg, success)
+    else:
+      var p = startProcess("appveyor", args = ["AddTest", test.name.replace("\\", "/") & test.options,
+                           "-Framework", "nim-testament", "-FileName",
+                           test.cat.string,
+                           "-Outcome", outcome, "-ErrorMessage", msg,
+                           "-Duration", $(duration * 1000).int],
+                           options = {poStdErrToStdOut, poUsePath, poParentStreams})
+      discard waitForExit(p)
+      close(p)
+
+proc toString(inlineError: InlineError, filename: string): string =
+  result.add "$file($line, $col) $kind: $msg" % [
+    "file", filename,
+    "line", $inlineError.line,
+    "col", $inlineError.col,
+    "kind", $inlineError.kind,
+    "msg", $inlineError.msg
+  ]
+
+proc inlineErrorsMsgs(expected: TSpec): string =
+  for inlineError in expected.inlineErrors.items:
+    result.addLine inlineError.toString(expected.filename)
+
+proc checkForInlineErrors(expected, given: TSpec): bool =
+  for inlineError in expected.inlineErrors:
+    if inlineError.toString(expected.filename) notin given.nimout:
+      return false
+  true
+
+proc nimoutCheck(expected, given: TSpec): bool =
+  result = true
+  if expected.nimoutFull:
+    if expected.nimout != given.nimout:
+      result = false
+  elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout):
+    result = false
+
+proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest,
+             target: TTarget, extraOptions: string) =
+  if not checkForInlineErrors(expected, given) or
+    (not expected.nimoutFull and not nimoutCheck(expected, given)):
+      r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer)
+  elif strip(expected.msg) notin strip(given.msg):
+    r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer)
+  elif not nimoutCheck(expected, given):
+    r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer)
+  elif extractFilename(expected.file) != extractFilename(given.file) and
+      "internal error:" notin expected.msg:
+    r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer)
+  elif expected.line != given.line and expected.line != 0 or
+       expected.column != given.column and expected.column != 0:
+    r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column,
+                      $given.line & ':' & $given.column, reLinesDiffer)
+  else:
+    r.addResult(test, target, extraOptions, expected.msg, given.msg, reSuccess)
+    inc(r.passed)
+
+proc generatedFile(test: TTest, target: TTarget): string =
+  if target == targetJS:
+    result = test.name.changeFileExt("js")
+  else:
+    let (_, name, _) = test.name.splitFile
+    let ext = targetToExt[target]
+    result = nimcacheDir(test.name, test.options, target) / "@m" & name.changeFileExt(ext)
+
+proc needsCodegenCheck(spec: TSpec): bool =
+  result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0
+
+proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var string,
+                  given: var TSpec) =
+  try:
+    let genFile = generatedFile(test, target)
+    let contents = readFile(genFile)
+    for check in spec.ccodeCheck:
+      if check.len > 0 and check[0] == '\\':
+        # little hack to get 'match' support:
+        if not contents.match(check.peg):
+          given.err = reCodegenFailure
+      elif contents.find(check.peg) < 0:
+        given.err = reCodegenFailure
+      expectedMsg = check
+    if spec.maxCodeSize > 0 and contents.len > spec.maxCodeSize:
+      given.err = reCodegenFailure
+      given.msg = "generated code size: " & $contents.len
+      expectedMsg = "max allowed size: " & $spec.maxCodeSize
+  except ValueError:
+    given.err = reInvalidPeg
+    echo getCurrentExceptionMsg()
+  except IOError:
+    given.err = reCodeNotFound
+    echo getCurrentExceptionMsg()
+
+proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string,
+                         given: var TSpec, expected: TSpec; r: var TResults) =
+  var expectedmsg: string = ""
+  var givenmsg: string = ""
+  if given.err == reSuccess:
+    if expected.needsCodegenCheck:
+      codegenCheck(test, target, expected, expectedmsg, given)
+      givenmsg = given.msg
+    if not nimoutCheck(expected, given) or
+       not checkForInlineErrors(expected, given):
+      given.err = reMsgsDiffer
+      expectedmsg = expected.nimout & inlineErrorsMsgs(expected)
+      givenmsg = given.nimout.strip
+  else:
+    givenmsg = "$ " & given.cmd & '\n' & given.nimout
+  if given.err == reSuccess: inc(r.passed)
+  r.addResult(test, target, extraOptions, expectedmsg, givenmsg, given.err)
+
+proc getTestSpecTarget(): TTarget =
+  if getEnv("NIM_COMPILE_TO_CPP", "false") == "true":
+    result = targetCpp
+  else:
+    result = targetC
+
+var count = 0
+
+proc equalModuloLastNewline(a, b: string): bool =
+  # allow lazy output spec that omits last newline, but really those should be fixed instead
+  result = a == b or b.endsWith("\n") and a == b[0 ..< ^1]
+
+proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
+                    target: TTarget, extraOptions: string, nimcache: string) =
+  test.startTime = epochTime()
+  if testName(test, target, extraOptions, false) in skips:
+    test.spec.err = reDisabled
+
+  if test.spec.err in {reDisabled, reJoined}:
+    r.addResult(test, target, extraOptions, "", "", test.spec.err)
+    inc(r.skipped)
+    return
+  var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
+  case expected.action
+  of actionCompile:
+    compilerOutputTests(test, target, extraOptions, given, expected, r)
+  of actionRun:
+    if given.err != reSuccess:
+      r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr)
+    else:
+      let isJsTarget = target == targetJS
+      var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt)
+      if not fileExists(exeFile):
+        r.addResult(test, target, extraOptions, expected.output,
+                    "executable not found: " & exeFile, reExeNotFound)
+      else:
+        let nodejs = if isJsTarget: findNodeJs() else: ""
+        if isJsTarget and nodejs == "":
+          r.addResult(test, target, extraOptions, expected.output, "nodejs binary not in PATH",
+                      reExeNotFound)
+        else:
+          var exeCmd: string
+          var args = test.testArgs
+          if isJsTarget:
+            exeCmd = nodejs
+            # see D20210217T215950
+            args = @["--unhandled-rejections=strict", exeFile] & args
+          else:
+            exeCmd = exeFile.dup(normalizeExe)
+            if valgrindEnabled and expected.useValgrind != disabled:
+              var valgrindOptions = @["--error-exitcode=1"]
+              if expected.useValgrind != leaking:
+                valgrindOptions.add "--leak-check=yes"
+              args = valgrindOptions & exeCmd & args
+              exeCmd = "valgrind"
+          var (_, buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input)
+          # Treat all failure codes from nodejs as 1. Older versions of nodejs used
+          # to return other codes, but for us it is sufficient to know that it's not 0.
+          if exitCode != 0: exitCode = 1
+          let bufB =
+            if expected.sortoutput:
+              var buf2 = buf
+              buf2.stripLineEnd
+              var x = splitLines(buf2)
+              sort(x, system.cmp)
+              join(x, "\n") & '\n'
+            else:
+              buf
+          if exitCode != expected.exitCode:
+            given.err = reExitcodesDiffer
+            r.addResult(test, target, extraOptions, "exitcode: " & $expected.exitCode,
+                              "exitcode: " & $exitCode & "\n\nOutput:\n" &
+                              bufB, reExitcodesDiffer)
+          elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or
+              (expected.outputCheck == ocSubstr and expected.output notin bufB):
+            given.err = reOutputsDiffer
+            r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer)
+          compilerOutputTests(test, target, extraOptions, given, expected, r)
+  of actionReject:
+    # Make sure its the compiler rejecting and not the system (e.g. segfault)
+    cmpMsgs(r, expected, given, test, target, extraOptions)
+    if given.exitCode != QuitFailure:
+      r.addResult(test, target, extraOptions, "exitcode: " & $QuitFailure,
+                        "exitcode: " & $given.exitCode & "\n\nOutput:\n" &
+                        given.nimout, reExitcodesDiffer)
+
+
+
+proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget =
+  result = defaultTarget
+  var p = parseopt.initOptParser(extraOptions)
+
+  while true:
+    parseopt.next(p)
+    case p.kind
+    of cmdEnd: break
+    of cmdLongOption, cmdShortOption:
+      if p.key == "b" or p.key == "backend":
+        result = parseEnum[TTarget](p.val.normalize)
+        # chooses the last one
+    else:
+      discard
+
+proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) =
+  for target in expected.targets:
+    inc(r.total)
+    if target notin gTargets:
+      r.addResult(test, target, extraOptions, "", "", reDisabled)
+      inc(r.skipped)
+    elif simulate:
+      inc count
+      echo "testSpec count: ", count, " expected: ", expected
+    else:
+      let nimcache = nimcacheDir(test.name, test.options, target)
+      var testClone = test
+      let target = changeTarget(extraOptions, target)
+      testSpecHelper(r, testClone, expected, target, extraOptions, nimcache)
+
+proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) =
+  var expected = test.spec
+  if expected.parseErrors.len > 0:
+    # targetC is a lie, but a parameter is required
+    r.addResult(test, targetC, "", "", expected.parseErrors, reInvalidSpec)
+    inc(r.total)
+    return
+
+  expected.targets.incl targets
+  # still no target specified at all
+  if expected.targets == {}:
+    expected.targets = {getTestSpecTarget()}
+  if test.spec.matrix.len > 0:
+    for m in test.spec.matrix:
+      targetHelper(r, test, expected, m)
+  else:
+    targetHelper(r, test, expected, "")
+
+proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} =
+  for target in test.spec.targets:
+    inc(r.total)
+    var testClone = test
+    testSpecHelper(r, testClone, test.spec, target, "", nimcache)
+
+proc makeTest(test, options: string, cat: Category): TTest =
+  result.cat = cat
+  result.name = test
+  result.options = options
+  result.spec = parseSpec(addFileExt(test, ".nim"))
+  result.startTime = epochTime()
+
+proc makeRawTest(test, options: string, cat: Category): TTest {.used.} =
+  result.cat = cat
+  result.name = test
+  result.options = options
+  result.spec = initSpec(addFileExt(test, ".nim"))
+  result.spec.action = actionCompile
+  result.spec.targets = {getTestSpecTarget()}
+  result.startTime = epochTime()
+
+# TODO: fix these files
+const disabledFilesDefault = @[
+  "tableimpl.nim",
+  "setimpl.nim",
+  "hashcommon.nim",
+
+  # Requires compiling with '--threads:on`
+  "sharedlist.nim",
+  "sharedtables.nim",
+
+  # Error: undeclared identifier: 'hasThreadSupport'
+  "ioselectors_epoll.nim",
+  "ioselectors_kqueue.nim",
+  "ioselectors_poll.nim",
+
+  # Error: undeclared identifier: 'Timeval'
+  "ioselectors_select.nim",
+]
+
+when defined(windows):
+  const
+    # array of modules disabled from compilation test of stdlib.
+    disabledFiles = disabledFilesDefault & @["coro.nim"]
+else:
+  const
+    # array of modules disabled from compilation test of stdlib.
+    disabledFiles = disabledFilesDefault
+
+include categories
+
+proc loadSkipFrom(name: string): seq[string] =
+  if name.len == 0: return
+  # One skip per line, comments start with #
+  # used by `nlvm` (at least)
+  for line in lines(name):
+    let sline = line.strip()
+    if sline.len > 0 and not sline.startsWith('#'):
+      result.add sline
+
+proc main() =
+  azure.init()
+  backend.open()
+  var optPrintResults = false
+  var optFailing = false
+  var targetsStr = ""
+  var isMainProcess = true
+  var skipFrom = ""
+
+  var p = initOptParser()
+  p.next()
+  while p.kind in {cmdLongOption, cmdShortOption}:
+    case p.key.normalize
+    of "print": optPrintResults = true
+    of "verbose": optVerbose = true
+    of "failing": optFailing = true
+    of "pedantic": discard # deadcode refs https://github.com/nim-lang/Nim/issues/16731
+    of "targets":
+      targetsStr = p.val
+      gTargets = parseTargets(targetsStr)
+      targetsSet = true
+    of "nim":
+      compilerPrefix = addFileExt(p.val.absolutePath, ExeExt)
+    of "directory":
+      setCurrentDir(p.val)
+    of "colors":
+      case p.val:
+      of "on":
+        useColors = true
+      of "off":
+        useColors = false
+      else:
+        quit Usage
+    of "batch":
+      testamentData0.batchArg = p.val
+      if p.val != "_" and p.val.len > 0 and p.val[0] in {'0'..'9'}:
+        let s = p.val.split("_")
+        doAssert s.len == 2, $(p.val, s)
+        testamentData0.testamentBatch = s[0].parseInt
+        testamentData0.testamentNumBatch = s[1].parseInt
+        doAssert testamentData0.testamentNumBatch > 0
+        doAssert testamentData0.testamentBatch >= 0 and testamentData0.testamentBatch < testamentData0.testamentNumBatch
+    of "simulate":
+      simulate = true
+    of "megatest":
+      case p.val:
+      of "on":
+        useMegatest = true
+      of "off":
+        useMegatest = false
+      else:
+        quit Usage
+    of "valgrind":
+      case p.val:
+      of "on":
+        valgrindEnabled = true
+      of "off":
+        valgrindEnabled = false
+      else:
+        quit Usage
+    of "backendlogging":
+      case p.val:
+      of "on":
+        backendLogging = true
+      of "off":
+        backendLogging = false
+      else:
+        quit Usage
+    of "skipfrom":
+      skipFrom = p.val
+    else:
+      quit Usage
+    p.next()
+  if p.kind != cmdArgument:
+    quit Usage
+  var action = p.key.normalize
+  p.next()
+  var r = initResults()
+  case action
+  of "all":
+    #processCategory(r, Category"megatest", p.cmdLineRest, testsDir, runJoinableTests = false)
+
+    var myself = quoteShell(getAppFilename())
+    if targetsStr.len > 0:
+      myself &= " " & quoteShell("--targets:" & targetsStr)
+
+    myself &= " " & quoteShell("--nim:" & compilerPrefix)
+    if testamentData0.batchArg.len > 0:
+      myself &= " --batch:" & testamentData0.batchArg
+
+    if skipFrom.len > 0:
+      myself &= " " & quoteShell("--skipFrom:" & skipFrom)
+
+    var cats: seq[string]
+    let rest = if p.cmdLineRest.len > 0: " " & p.cmdLineRest else: ""
+    for kind, dir in walkDir(testsDir):
+      assert testsDir.startsWith(testsDir)
+      let cat = dir[testsDir.len .. ^1]
+      if kind == pcDir and cat notin ["testdata", "nimcache"]:
+        cats.add cat
+    if isNimRepoTests():
+      cats.add AdditionalCategories
+    if useMegatest: cats.add MegaTestCat
+
+    var cmds: seq[string]
+    for cat in cats:
+      let runtype = if useMegatest: " pcat " else: " cat "
+      cmds.add(myself & runtype & quoteShell(cat) & rest)
+
+    proc progressStatus(idx: int) =
+      echo "progress[all]: $1/$2 starting: cat: $3" % [$idx, $cats.len, cats[idx]]
+
+    if simulate:
+      skips = loadSkipFrom(skipFrom)
+      for i, cati in cats:
+        progressStatus(i)
+        processCategory(r, Category(cati), p.cmdLineRest, testsDir, runJoinableTests = false)
+    else:
+      addExitProc azure.finalize
+      quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus)
+  of "c", "cat", "category":
+    skips = loadSkipFrom(skipFrom)
+    var cat = Category(p.key)
+    processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = true)
+  of "pcat":
+    skips = loadSkipFrom(skipFrom)
+    # 'pcat' is used for running a category in parallel. Currently the only
+    # difference is that we don't want to run joinable tests here as they
+    # are covered by the 'megatest' category.
+    isMainProcess = false
+    var cat = Category(p.key)
+    p.next
+    processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = false)
+  of "p", "pat", "pattern":
+    skips = loadSkipFrom(skipFrom)
+    let pattern = p.key
+    p.next
+    processPattern(r, pattern, p.cmdLineRest, simulate)
+  of "r", "run":
+    let (cat, path) = splitTestFile(p.key)
+    processSingleTest(r, cat.Category, p.cmdLineRest, path, gTargets, targetsSet)
+  of "html":
+    generateHtml(resultsFile, optFailing)
+  else:
+    quit Usage
+
+  if optPrintResults:
+    if action == "html": openDefaultBrowser(resultsFile)
+    else: echo r, r.data
+  azure.finalize()
+  backend.close()
+  var failed = r.total - r.passed - r.skipped
+  if failed != 0:
+    echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ",
+      r.skipped, " failed: ", failed
+    quit(QuitFailure)
+  if isMainProcess:
+    echo "Used ", compilerPrefix, " to run the tests. Use --nim to override."
+
+if paramCount() == 0:
+  quit Usage
+main()