diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/readme.md | 2 | ||||
| -rw-r--r-- | tests/stdlib/tnre.nim | 2 | ||||
| -rw-r--r-- | tests/testament/backend.nim | 75 | ||||
| -rw-r--r-- | tests/testament/caasdriver.nim | 195 | ||||
| -rw-r--r-- | tests/testament/categories.nim | 539 | ||||
| -rw-r--r-- | tests/testament/htmlgen.nim | 148 | ||||
| -rw-r--r-- | tests/testament/specs.nim | 202 | ||||
| -rw-r--r-- | tests/testament/testamenthtml.templ | 297 | ||||
| -rw-r--r-- | tests/testament/tester.nim | 522 | ||||
| -rw-r--r-- | tests/testament/tester.nim.cfg | 1 |
10 files changed, 2 insertions, 1981 deletions
diff --git a/tests/readme.md b/tests/readme.md index 34a2c4bfb..1ee93dbcb 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -10,7 +10,7 @@ execute the compiled binary, you need to specify a spec with an ``action`` key Each test can contain a spec in a ``discard """ ... """`` block. -**Check out the [``parseSpec`` procedure](https://github.com/nim-lang/Nim/blob/devel/tests/testament/specs.nim#L124) in the ``specs`` module for a full and reliable reference** +**Check out the [``parseSpec`` procedure](https://github.com/nim-lang/Nim/blob/devel/testament/specs.nim#L124) in the ``specs`` module for a full and reliable reference** ## action diff --git a/tests/stdlib/tnre.nim b/tests/stdlib/tnre.nim index fabbb69a8..0929956cb 100644 --- a/tests/stdlib/tnre.nim +++ b/tests/stdlib/tnre.nim @@ -1,6 +1,6 @@ discard """ # Since the tests for nre are all bundled together we treat failure in one test as an nre failure -# When running 'tests/testament/tester' a failed check() in the test suite will cause the exit +# When running 'testament/tester' a failed check() in the test suite will cause the exit # codes to differ and be reported as a failure output: diff --git a/tests/testament/backend.nim b/tests/testament/backend.nim deleted file mode 100644 index 385f1171c..000000000 --- a/tests/testament/backend.nim +++ /dev/null @@ -1,75 +0,0 @@ -# -# -# The Nim Tester -# (c) Copyright 2017 Andreas Rumpf -# -# Look at license.txt for more info. -# All rights reserved. - -import strutils, os, osproc, json - -type - MachineId* = distinct string - CommitId = distinct string - -proc `$`*(id: MachineId): string {.borrow.} -#proc `$`(id: CommitId): string {.borrow.} # not used - -var - thisMachine: MachineId - thisCommit: CommitId - thisBranch: string - -{.experimental.} -proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip - -proc getMachine*(): MachineId = - var name = "hostname"() - if name.len == 0: - name = when defined(posix): getenv"HOSTNAME".string - else: getenv"COMPUTERNAME".string - if name.len == 0: - quit "cannot determine the machine name" - - result = MachineId(name) - -proc getCommit(): CommitId = - const commLen = "commit ".len - let hash = "git log -n 1"()[commLen..commLen+10] - thisBranch = "git symbolic-ref --short HEAD"() - if hash.len == 0 or thisBranch.len == 0: quit "cannot determine git HEAD" - result = CommitId(hash) - -var - results: File - currentCategory: string - entries: int - -proc writeTestResult*(name, category, target, - action, result, expected, given: string) = - createDir("testresults") - if currentCategory != category: - if currentCategory.len > 0: - results.writeLine("]") - close(results) - currentCategory = category - results = open("testresults" / category.addFileExt"json", fmWrite) - results.writeLine("[") - entries = 0 - - let jentry = %*{"name": name, "category": category, "target": target, - "action": action, "result": result, "expected": expected, "given": given, - "machine": thisMachine.string, "commit": thisCommit.string, "branch": thisBranch} - if entries > 0: - results.writeLine(",") - results.write($jentry) - inc entries - -proc open*() = - thisMachine = getMachine() - thisCommit = getCommit() - -proc close*() = - if currentCategory.len > 0: - results.writeLine("]") - close(results) diff --git a/tests/testament/caasdriver.nim b/tests/testament/caasdriver.nim deleted file mode 100644 index 30383bddb..000000000 --- a/tests/testament/caasdriver.nim +++ /dev/null @@ -1,195 +0,0 @@ -import osproc, streams, os, strutils, re -{.experimental.} - -## Compiler as a service tester. -## -## Please read docs/idetools.txt for information about this. - - -type - TRunMode = enum - ProcRun, CaasRun, SymbolProcRun - - NimSession* = object - nim: Process # Holds the open process for CaasRun sessions, nil otherwise. - mode: TRunMode # Stores the type of run mode the session was started with. - lastOutput: string # Preserves the last output, needed for ProcRun mode. - filename: string # Appended to each command starting with '>'. Also a var. - modname: string # Like filename but without extension. - nimcache: string # Input script based name for the nimcache dir. - -const - modes = [CaasRun, ProcRun, SymbolProcRun] - filenameReplaceVar = "$TESTNIM" - moduleReplaceVar = "$MODULE" - silentReplaceVar = "$SILENT" - silentReplaceText = "--verbosity:0 --hints:off" - -var - TesterDir = getAppDir() / ".." - NimBin = TesterDir / "../bin/nim" - -proc replaceVars(session: var NimSession, text: string): string = - result = text.replace(filenameReplaceVar, session.filename) - result = result.replace(moduleReplaceVar, session.modname) - result = result.replace(silentReplaceVar, silentReplaceText) - -proc startNimSession(project, script: string, mode: TRunMode): - NimSession = - let (dir, name, ext) = project.splitFile - result.mode = mode - result.lastOutput = "" - result.filename = name & ext - result.modname = name - - let (nimcacheDir, nimcacheName, nimcacheExt) = script.splitFile - result.nimcache = "SymbolProcRun." & nimcacheName - - if mode == SymbolProcRun: - removeDir(nimcacheDir / result.nimcache) - else: - removeDir(nimcacheDir / "nimcache") - - if mode == CaasRun: - result.nim = startProcess(NimBin, workingDir = dir, - args = ["serve", "--server.type:stdin", name]) - -proc doCaasCommand(session: var NimSession, command: string): string = - assert session.mode == CaasRun - session.nim.inputStream.write(session.replaceVars(command) & "\n") - session.nim.inputStream.flush - - result = "" - - while true: - var line = TaintedString("") - if session.nim.outputStream.readLine(line): - if line.string == "": break - result.add(line.string & "\n") - else: - result = "FAILED TO EXECUTE: " & command & "\n" & result - break - -proc doProcCommand(session: var NimSession, command: string): string = - try: - assert session.mode == ProcRun or session.mode == SymbolProcRun - except: - result = "FAILED TO EXECUTE: " & command & "\n" & result - var - process = startProcess(NimBin, args = session.replaceVars(command).split) - stream = outputStream(process) - line = TaintedString("") - - result = "" - while stream.readLine(line): - if result.len > 0: result &= "\n" - result &= line.string - - process.close() - -proc doCommand(session: var NimSession, command: string) = - if session.mode == CaasRun: - if not session.nim.running: - session.lastOutput = "FAILED TO EXECUTE: " & command & "\n" & - "Exit code " & $session.nim.peekExitCode - return - session.lastOutput = doCaasCommand(session, - command & " " & session.filename) - else: - var command = command - # For symbol runs we prepend the necessary parameters to avoid clobbering - # the normal nimcache. - if session.mode == SymbolProcRun: - command = "--symbolFiles:on --nimcache:" & session.nimcache & - " " & command - session.lastOutput = doProcCommand(session, - command & " " & session.filename) - -proc destroy(session: var NimSession) {.destructor.} = - if session.mode == CaasRun: - session.nim.close - -proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): bool = - result = true - - var f = open(script) - var project = TaintedString("") - - if f.readLine(project): - var - s = startNimSession(script.parentDir / project.string, script, mode) - tline = TaintedString("") - ln = 1 - - while f.readLine(tline): - var line = tline.string - inc ln - - # Filter lines by run mode, removing the prefix if the mode is current. - for testMode in modes: - if line.startsWith($testMode): - if testMode != mode: - line = "" - else: - line = line[len($testMode)..len(line) - 1].strip - break - - if line.strip.len == 0: continue - - if line.startsWith("#"): - output.writeLine line - continue - elif line.startsWith(">"): - s.doCommand(line.substr(1).strip) - output.writeLine line, "\n", if verbose: s.lastOutput else: "" - else: - var expectMatch = true - var pattern = s.replaceVars(line) - if line.startsWith("!"): - pattern = pattern.substr(1).strip - expectMatch = false - - let actualMatch = - s.lastOutput.find(re(pattern, flags = {reStudy})) != -1 - - if expectMatch == actualMatch: - output.writeLine "SUCCESS ", line - else: - output.writeLine "FAILURE ", line - result = false - -iterator caasTestsRunner*(filter = "", verbose = false): tuple[test, - output: string, status: bool, - mode: TRunMode] = - for scenario in os.walkFiles(TesterDir / "caas/*.txt"): - if filter.len > 0 and find(scenario, filter) == -1: continue - for mode in modes: - var outStream = newStringStream() - let r = doScenario(scenario, outStream, mode, verbose) - yield (scenario, outStream.data, r, mode) - -when isMainModule: - var - filter = "" - failures = 0 - verbose = false - - for i in 0..paramCount() - 1: - let param = string(paramStr(i + 1)) - case param - of "verbose": verbose = true - else: filter = param - - if verbose and len(filter) > 0: - echo "Running only test cases matching filter '$1'" % [filter] - - for test, output, result, mode in caasTestsRunner(filter, verbose): - if not result or verbose: - echo "Mode ", $mode, " (", if result: "succeeded)" else: "failed)" - echo test - echo output - echo "---------\n" - if not result: - failures += 1 - - quit(failures) diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim deleted file mode 100644 index 59ce70315..000000000 --- a/tests/testament/categories.nim +++ /dev/null @@ -1,539 +0,0 @@ -# -# -# Nim Tester -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Include for the tester that contains test suites that test special features -## of the compiler. - -# included from tester.nim -# ---------------- ROD file tests --------------------------------------------- - -const - rodfilesDir = "tests/rodfiles" - -proc delNimCache(filename, options: string) = - for target in low(TTarget)..high(TTarget): - let dir = nimcacheDir(filename, options, target) - try: - removeDir(dir) - except OSError: - echo "[Warning] could not delete: ", dir - -proc runRodFiles(r: var TResults, cat: Category, options: string) = - template test(filename: string, clearCacheFirst=false) = - if clearCacheFirst: delNimCache(filename, options) - testSpec r, makeTest(rodfilesDir / filename, options, cat, actionRun) - - - # test basic recompilation scheme: - test "hallo", true - test "hallo" - when false: - # test incremental type information: - test "hallo2" - - # test type converters: - test "aconv", true - test "bconv" - - # test G, A, B example from the documentation; test init sections: - test "deada", true - test "deada2" - - when false: - # test method generation: - test "bmethods", true - test "bmethods2" - - # test generics: - test "tgeneric1", true - test "tgeneric2" - -proc compileRodFiles(r: var TResults, cat: Category, options: string) = - template test(filename: untyped, clearCacheFirst=true) = - if clearCacheFirst: delNimCache(filename, options) - testSpec r, makeTest(rodfilesDir / filename, options, cat) - - # test DLL interfacing: - test "gtkex1", true - test "gtkex2" - -# --------------------- flags tests ------------------------------------------- - -proc flagTests(r: var TResults, cat: Category, options: string) = - # --genscript - const filename = "tests"/"flags"/"tgenscript" - const genopts = " --genscript" - let nimcache = nimcacheDir(filename, genopts, targetC) - testSpec r, makeTest(filename, genopts, cat) - - when defined(windows): - testExec r, makeTest(filename, " cmd /c cd " & nimcache & - " && compile_tgenscript.bat", cat) - - elif defined(posix): - testExec r, makeTest(filename, " sh -c \"cd " & nimcache & - " && sh compile_tgenscript.sh\"", cat) - - # Run - testExec r, makeTest(filename, " " & nimcache / "tgenscript", cat) - -# --------------------- DLL generation tests ---------------------------------- - -proc safeCopyFile(src, dest: string) = - try: - copyFile(src, dest) - except OSError: - echo "[Warning] could not copy: ", src, " to ", dest - -proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = - const rpath = when defined(macosx): - " --passL:-rpath --passL:@loader_path" - else: - "" - - testSpec c, makeTest("lib/nimrtl.nim", - options & " --app:lib -d:createNimRtl --threads:on", cat) - testSpec c, makeTest("tests/dll/server.nim", - options & " --app:lib -d:useNimRtl --threads:on" & rpath, cat) - - - when defined(Windows): - # windows looks in the dir of the exe (yay!): - var nimrtlDll = DynlibFormat % "nimrtl" - safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll) - else: - # posix relies on crappy LD_LIBRARY_PATH (ugh!): - const libpathenv = when defined(haiku): - "LIBRARY_PATH" - else: - "LD_LIBRARY_PATH" - var libpath = getEnv(libpathenv).string - # Temporarily add the lib directory to LD_LIBRARY_PATH: - putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: "")) - defer: putEnv(libpathenv, libpath) - var nimrtlDll = DynlibFormat % "nimrtl" - safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll) - - testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl --threads:on" & rpath, - cat, actionRun) - -proc dllTests(r: var TResults, cat: Category, options: string) = - # dummy compile result: - var c = initResults() - - runBasicDLLTest c, r, cat, options - runBasicDLLTest c, r, cat, options & " -d:release" - when not defined(windows): - # still cannot find a recent Windows version of boehm.dll: - runBasicDLLTest c, r, cat, options & " --gc:boehm" - runBasicDLLTest c, r, cat, options & " -d:release --gc:boehm" - -# ------------------------------ GC tests ------------------------------------- - -proc gcTests(r: var TResults, cat: Category, options: string) = - template testWithNone(filename: untyped) = - testSpec r, makeTest("tests/gc" / filename, options & - " --gc:none", cat, actionRun) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release --gc:none", cat, actionRun) - - template testWithoutMs(filename: untyped) = - testSpec r, makeTest("tests/gc" / filename, options, cat, actionRun) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release", cat, actionRun) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release -d:useRealtimeGC", cat, actionRun) - - template testWithoutBoehm(filename: untyped) = - testWithoutMs filename - testSpec r, makeTest("tests/gc" / filename, options & - " --gc:markAndSweep", cat, actionRun) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release --gc:markAndSweep", cat, actionRun) - template test(filename: untyped) = - testWithoutBoehm filename - when not defined(windows) and not defined(android): - # AR: cannot find any boehm.dll on the net, right now, so disabled - # for windows: - testSpec r, makeTest("tests/gc" / filename, options & - " --gc:boehm", cat, actionRun) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release --gc:boehm", cat, actionRun) - - testWithoutBoehm "foreign_thr" - test "gcemscripten" - test "growobjcrash" - test "gcbench" - test "gcleak" - test "gcleak2" - testWithoutBoehm "gctest" - testWithNone "gctest" - test "gcleak3" - test "gcleak4" - # Disabled because it works and takes too long to run: - #test "gcleak5" - testWithoutBoehm "weakrefs" - test "cycleleak" - testWithoutBoehm "closureleak" - testWithoutMs "refarrayleak" - - testWithoutBoehm "tlists" - testWithoutBoehm "thavlak" - - test "stackrefleak" - test "cyclecollector" - -proc longGCTests(r: var TResults, cat: Category, options: string) = - when defined(windows): - let cOptions = "-ldl -DWIN" - else: - let cOptions = "-ldl" - - var c = initResults() - # According to ioTests, this should compile the file - testNoSpec c, makeTest("tests/realtimeGC/shared", options, cat, actionCompile) - testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat, actionRun) - testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat, actionRun) - -# ------------------------- threading tests ----------------------------------- - -proc threadTests(r: var TResults, cat: Category, options: string) = - template test(filename: untyped) = - testSpec r, makeTest(filename, options, cat, actionRun) - testSpec r, makeTest(filename, options & " -d:release", cat, actionRun) - testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat, actionRun) - for t in os.walkFiles("tests/threads/t*.nim"): - test(t) - -# ------------------------- IO tests ------------------------------------------ - -proc ioTests(r: var TResults, cat: Category, options: string) = - # We need readall_echo to be compiled for this test to run. - # dummy compile result: - var c = initResults() - testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat) - testSpec r, makeTest("tests/system/tio", options, cat) - -# ------------------------- async tests --------------------------------------- -proc asyncTests(r: var TResults, cat: Category, options: string) = - template test(filename: untyped) = - testSpec r, makeTest(filename, options, cat) - for t in os.walkFiles("tests/async/t*.nim"): - test(t) - -# ------------------------- debugger tests ------------------------------------ - -proc debuggerTests(r: var TResults, cat: Category, options: string) = - testNoSpec r, makeTest("tools/nimgrep", options & " --debugger:on", cat) - -# ------------------------- JS tests ------------------------------------------ - -proc jsTests(r: var TResults, cat: Category, options: string) = - template test(filename: untyped) = - testSpec r, makeTest(filename, options & " -d:nodejs", cat, - actionRun), targetJS - testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat, - actionRun), targetJS - - for t in os.walkFiles("tests/js/t*.nim"): - test(t) - for testfile in ["exception/texceptions", "exception/texcpt1", - "exception/texcsub", "exception/tfinally", - "exception/tfinally2", "exception/tfinally3", - "exception/tunhandledexc", - "actiontable/tactiontable", "method/tmultim1", - "method/tmultim3", "method/tmultim4", - "varres/tvarres0", "varres/tvarres3", "varres/tvarres4", - "varres/tvartup", "misc/tints", "misc/tunsignedinc", - "async/tjsandnativeasync"]: - test "tests/" & testfile & ".nim" - - for testfile in ["strutils", "json", "random", "times", "logging"]: - test "lib/pure/" & testfile & ".nim" - -# ------------------------- nim in action ----------- - -proc testNimInAction(r: var TResults, cat: Category, options: string) = - let options = options & " --nilseqs:on" - - template test(filename: untyped, action: untyped) = - testSpec r, makeTest(filename, options, cat, action) - - template testJS(filename: untyped) = - testSpec r, makeTest(filename, options, cat, actionCompile), targetJS - - template testCPP(filename: untyped) = - testSpec r, makeTest(filename, options, cat, actionCompile), targetCPP - - let tests = [ - "niminaction/Chapter1/various1", - "niminaction/Chapter2/various2", - "niminaction/Chapter2/resultaccept", - "niminaction/Chapter2/resultreject", - "niminaction/Chapter2/explicit_discard", - "niminaction/Chapter2/no_def_eq", - "niminaction/Chapter2/no_iterator", - "niminaction/Chapter2/no_seq_type", - "niminaction/Chapter3/ChatApp/src/server", - "niminaction/Chapter3/ChatApp/src/client", - "niminaction/Chapter3/various3", - "niminaction/Chapter6/WikipediaStats/concurrency_regex", - "niminaction/Chapter6/WikipediaStats/concurrency", - "niminaction/Chapter6/WikipediaStats/naive", - "niminaction/Chapter6/WikipediaStats/parallel_counts", - "niminaction/Chapter6/WikipediaStats/race_condition", - "niminaction/Chapter6/WikipediaStats/sequential_counts", - "niminaction/Chapter6/WikipediaStats/unguarded_access", - "niminaction/Chapter7/Tweeter/src/tweeter", - "niminaction/Chapter7/Tweeter/src/createDatabase", - "niminaction/Chapter7/Tweeter/tests/database_test", - "niminaction/Chapter8/sdl/sdl_test" - ] - - # Verify that the files have not been modified. Death shall fall upon - # whoever edits these hashes without dom96's permission, j/k. But please only - # edit when making a conscious breaking change, also please try to make your - # commit message clear and notify me so I can easily compile an errata later. - var testHashes: seq[string] = @[] - - for test in tests: - testHashes.add(getMD5(readFile("tests" / test.addFileExt("nim")).string)) - - const refHashes = @[ - "51afdfa84b3ca3d810809d6c4e5037ba", "30f07e4cd5eaec981f67868d4e91cfcf", - "d14e7c032de36d219c9548066a97e846", "2e40bfd5daadb268268727da91bb4e81", - "c5d3853ed0aba04bf6d35ba28a98dca0", "058603145ff92d46c009006b06e5b228", - "7b94a029b94ddb7efafddd546c965ff6", "586d74514394e49f2370dfc01dd9e830", - "e1901837b757c9357dc8d259fd0ef0f6", "097670c7ae12e825debaf8ec3995227b", - "a8cb7b78cc78d28535ab467361db5d6e", "bfaec2816a1848991b530c1ad17a0184", - "47cb71bb4c1198d6d29cdbee05aa10b9", "87e4436809f9d73324cfc4f57f116770", - "7b7db5cddc8cf8fa9b6776eef1d0a31d", "e6e40219f0f2b877869b738737b7685e", - "6532ee87d819f2605a443d5e94f9422a", "9a8fe78c588d08018843b64b57409a02", - "03a801275b8b76b4170c870cd0da079d", "20bb7d3e2d38d43b0cb5fcff4909a4a8", - "af6844598f534fab6942abfa4dfe9ab2", "2a7a17f84f6503d9bc89a5ab8feea127" - ] - doAssert testHashes == refHashes, "Nim in Action tests were changed." - - # Run the tests. - for testfile in tests: - test "tests/" & testfile & ".nim", actionCompile - - let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim" - testJS jsFile - - let cppFile = "tests/niminaction/Chapter8/sfml/sfml_test.nim" - testCPP cppFile - - - - -# ------------------------- manyloc ------------------------------------------- -#proc runSpecialTests(r: var TResults, options: string) = -# for t in ["lib/packages/docutils/highlite"]: -# testSpec(r, t, options) - -proc findMainFile(dir: string): string = - # finds the file belonging to ".nim.cfg"; if there is no such file - # it returns the some ".nim" file if there is only one: - const cfgExt = ".nim.cfg" - result = "" - var nimFiles = 0 - for kind, file in os.walkDir(dir): - if kind == pcFile: - if file.endsWith(cfgExt): return file[.. ^(cfgExt.len+1)] & ".nim" - elif file.endsWith(".nim"): - if result.len == 0: result = file - inc nimFiles - if nimFiles != 1: result.setlen(0) - -proc manyLoc(r: var TResults, cat: Category, options: string) = - for kind, dir in os.walkDir("tests/manyloc"): - if kind == pcDir: - when defined(windows): - if dir.endsWith"nake": continue - if dir.endsWith"named_argument_bug": continue - let mainfile = findMainFile(dir) - if mainfile != "": - testNoSpec r, makeTest(mainfile, options, cat) - -proc compileExample(r: var TResults, pattern, options: string, cat: Category) = - for test in os.walkFiles(pattern): - testNoSpec r, makeTest(test, options, cat) - -proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = - for test in os.walkFiles(pattern): - let name = extractFilename(test) - if name notin disabledFiles: - let contents = readFile(test).string - if contents.contains("when isMainModule"): - testSpec r, makeTest(test, options, cat, actionRunNoSpec) - else: - testNoSpec r, makeTest(test, options, cat, actionCompile) - -# ----------------------------- nimble ---------------------------------------- -type PackageFilter = enum - pfCoreOnly - pfExtraOnly - pfAll - -var nimbleDir = getEnv("NIMBLE_DIR").string -if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" -let - nimbleExe = findExe("nimble") - #packageDir = nimbleDir / "pkgs" # not used - packageIndex = nimbleDir / "packages.json" - -proc waitForExitEx(p: Process): int = - var outp = outputStream(p) - var line = newStringOfCap(120).TaintedString - while true: - if outp.readLine(line): - discard - else: - result = peekExitCode(p) - if result != -1: break - close(p) - -proc getPackageDir(package: string): string = - ## TODO - Replace this with dom's version comparison magic. - var commandOutput = execCmdEx("nimble path $#" % package) - if commandOutput.exitCode != QuitSuccess: - return "" - else: - result = commandOutput[0].string - -iterator listPackages(filter: PackageFilter): tuple[name, url: string] = - let packageList = parseFile(packageIndex) - - for package in packageList.items(): - let - name = package["name"].str - url = package["url"].str - isCorePackage = "nim-lang" in normalize(url) - case filter: - of pfCoreOnly: - if isCorePackage: - yield (name, url) - of pfExtraOnly: - if not isCorePackage: - yield (name, url) - of pfAll: - yield (name, url) - -proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) = - if nimbleExe == "": - echo("[Warning] - Cannot run nimble tests: Nimble binary not found.") - return - - if execCmd("$# update" % nimbleExe) == QuitFailure: - echo("[Warning] - Cannot run nimble tests: Nimble update failed.") - return - - let packageFileTest = makeTest("PackageFileParsed", "", cat) - try: - for name, url in listPackages(filter): - var test = makeTest(name, "", cat) - echo(url) - let - installProcess = startProcess(nimbleExe, "", ["install", "-y", name]) - installStatus = waitForExitEx(installProcess) - installProcess.close - if installStatus != QuitSuccess: - r.addResult(test, targetC, "", "", reInstallFailed) - continue - - let - buildPath = getPackageDir(name).strip - buildProcess = startProcess(nimbleExe, buildPath, ["build"]) - buildStatus = waitForExitEx(buildProcess) - buildProcess.close - if buildStatus != QuitSuccess: - r.addResult(test, targetC, "", "", reBuildFailed) - r.addResult(test, targetC, "", "", reSuccess) - r.addResult(packageFileTest, targetC, "", "", reSuccess) - except JsonParsingError: - echo("[Warning] - Cannot run nimble tests: Invalid package file.") - r.addResult(packageFileTest, targetC, "", "", reBuildFailed) - - -# ---------------------------------------------------------------------------- - -const AdditionalCategories = ["debugger", "examples", "lib"] - -proc `&.?`(a, b: string): string = - # candidate for the stdlib? - result = if b.startswith(a): b else: a & b - -#proc `&?.`(a, b: string): string = # not used - # candidate for the stdlib? - #result = if a.endswith(b): a else: a & b - -proc processSingleTest(r: var TResults, cat: Category, options, test: string) = - let test = "tests" & DirSep &.? cat.string / test - let target = if cat.string.normalize == "js": targetJS else: targetC - - if existsFile(test): testSpec r, makeTest(test, options, cat), target - else: echo "[Warning] - ", test, " test does not exist" - -proc processCategory(r: var TResults, cat: Category, options: string) = - case cat.string.normalize - of "rodfiles": - when false: - compileRodFiles(r, cat, options) - runRodFiles(r, cat, options) - of "js": - # only run the JS tests on Windows or Linux because Travis is bad - # and other OSes like Haiku might lack nodejs: - if not defined(linux) and isTravis: - discard - else: - jsTests(r, cat, options) - of "dll": - dllTests(r, cat, options) - of "flags": - flagTests(r, cat, options) - of "gc": - gcTests(r, cat, options) - of "longgc": - longGCTests(r, cat, options) - of "debugger": - debuggerTests(r, cat, options) - of "manyloc": - manyLoc r, cat, options - of "threads": - threadTests r, cat, options & " --threads:on" - of "io": - ioTests r, cat, options - of "async": - asyncTests r, cat, options - of "lib": - testStdlib(r, "lib/pure/*.nim", options, cat) - testStdlib(r, "lib/packages/docutils/highlite", options, cat) - of "examples": - compileExample(r, "examples/*.nim", options, cat) - compileExample(r, "examples/gtk/*.nim", options, cat) - compileExample(r, "examples/talk/*.nim", options, cat) - of "nimble-core": - testNimblePackages(r, cat, pfCoreOnly) - of "nimble-extra": - testNimblePackages(r, cat, pfExtraOnly) - of "nimble-all": - testNimblePackages(r, cat, pfAll) - of "niminaction": - testNimInAction(r, cat, options) - of "untestable": - # We can't test it because it depends on a third party. - discard # TODO: Move untestable tests to someplace else, i.e. nimble repo. - else: - var testsRun = 0 - for name in os.walkFiles("tests" & DirSep &.? cat.string / "t*.nim"): - testSpec r, makeTest(name, options, cat) - inc testsRun - if testsRun == 0: - echo "[Warning] - Invalid category specified \"", cat.string, "\", no tests were run" diff --git a/tests/testament/htmlgen.nim b/tests/testament/htmlgen.nim deleted file mode 100644 index 4a10fe00c..000000000 --- a/tests/testament/htmlgen.nim +++ /dev/null @@ -1,148 +0,0 @@ -# -# -# Nim Tester -# (c) Copyright 2017 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## HTML generator for the tester. - -import cgi, backend, strutils, json, os, tables, times - -import "testamenthtml.templ" - -proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) = - let - trId = htmlQuote(testResultRow["category"].str & "_" & testResultRow["name"].str). - multiReplace({".": "_", " ": "_", ":": "_"}) - name = testResultRow["name"].str.htmlQuote() - category = testResultRow["category"].str.htmlQuote() - target = testResultRow["target"].str.htmlQuote() - action = testResultRow["action"].str.htmlQuote() - result = htmlQuote testResultRow["result"].str - expected = testResultRow["expected"].getStr - gotten = testResultRow["given"].getStr - timestamp = "unknown" - var - panelCtxClass, textCtxClass, bgCtxClass: string - resultSign, resultDescription: string - case result - of "reSuccess": - panelCtxClass = "success" - textCtxClass = "success" - bgCtxClass = "success" - resultSign = "ok" - resultDescription = "PASS" - of "reIgnored": - panelCtxClass = "info" - textCtxClass = "info" - bgCtxClass = "info" - resultSign = "question" - resultDescription = "SKIP" - else: - panelCtxClass = "danger" - textCtxClass = "danger" - bgCtxClass = "danger" - resultSign = "exclamation" - resultDescription = "FAIL" - - outfile.generateHtmlTestresultPanelBegin( - trId, name, target, category, action, resultDescription, - timestamp, result, resultSign, panelCtxClass, textCtxClass, bgCtxClass - ) - if expected.isNilOrWhitespace() and gotten.isNilOrWhitespace(): - outfile.generateHtmlTestresultOutputNone() - else: - outfile.generateHtmlTestresultOutputDetails( - expected.strip().htmlQuote, - gotten.strip().htmlQuote - ) - outfile.generateHtmlTestresultPanelEnd() - -type - AllTests = object - data: JSonNode - totalCount, successCount, ignoredCount, failedCount: int - successPercentage, ignoredPercentage, failedPercentage: BiggestFloat - -proc allTestResults(onlyFailing = false): AllTests = - result.data = newJArray() - for file in os.walkFiles("testresults/*.json"): - let data = parseFile(file) - if data.kind != JArray: - echo "[ERROR] ignoring json file that is not an array: ", file - else: - for elem in data: - let state = elem["result"].str - inc result.totalCount - if state.contains("reSuccess"): inc result.successCount - elif state.contains("reIgnored"): inc result.ignoredCount - if not onlyFailing or not(state.contains("reSuccess")): - result.data.add elem - result.successPercentage = 100 * - (result.successCount.toBiggestFloat / result.totalCount.toBiggestFloat) - result.ignoredPercentage = 100 * - (result.ignoredCount.toBiggestFloat / result.totalCount.toBiggestFloat) - result.failedCount = result.totalCount - - result.successCount - result.ignoredCount - result.failedPercentage = 100 * - (result.failedCount.toBiggestFloat / result.totalCount.toBiggestFloat) - -proc generateTestResultsPanelGroupPartial(outfile: File, allResults: JsonNode) = - for testresultRow in allResults: - generateTestResultPanelPartial(outfile, testresultRow) - -proc generateAllTestsContent(outfile: File, allResults: AllTests, - onlyFailing = false) = - if allResults.data.len < 1: return # Nothing to do if there is no data. - # Only results from one test run means that test run environment info is the - # same for all tests - let - firstRow = allResults.data[0] - commit = htmlQuote firstRow["commit"].str - branch = htmlQuote firstRow["branch"].str - machine = htmlQuote firstRow["machine"].str - - outfile.generateHtmlAllTestsBegin( - machine, commit, branch, - allResults.totalCount, - allResults.successCount, - formatBiggestFloat(allResults.successPercentage, ffDecimal, 2) & "%", - allResults.ignoredCount, - formatBiggestFloat(allResults.ignoredPercentage, ffDecimal, 2) & "%", - allResults.failedCount, - formatBiggestFloat(allResults.failedPercentage, ffDecimal, 2) & "%", - onlyFailing - ) - generateTestResultsPanelGroupPartial(outfile, allResults.data) - outfile.generateHtmlAllTestsEnd() - -proc generateHtml*(filename: string, onlyFailing: bool) = - let - currentTime = getTime().local() - timestring = htmlQuote format(currentTime, "yyyy-MM-dd HH:mm:ss 'UTC'zzz") - var outfile = open(filename, fmWrite) - - outfile.generateHtmlBegin() - - generateAllTestsContent(outfile, allTestResults(onlyFailing), onlyFailing) - - outfile.generateHtmlEnd(timestring) - - outfile.flushFile() - close(outfile) - -proc dumpJsonTestResults*(prettyPrint, onlyFailing: bool) = - var - outfile = stdout - jsonString: string - - let results = allTestResults(onlyFailing) - if prettyPrint: - jsonString = results.data.pretty() - else: - jsonString = $ results.data - - outfile.writeLine(jsonString) diff --git a/tests/testament/specs.nim b/tests/testament/specs.nim deleted file mode 100644 index c51a3343e..000000000 --- a/tests/testament/specs.nim +++ /dev/null @@ -1,202 +0,0 @@ -# -# -# Nim Tester -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import parseutils, strutils, os, osproc, streams, parsecfg - - -var compilerPrefix* = "compiler" / "nim " - -let isTravis* = existsEnv("TRAVIS") -let isAppVeyor* = existsEnv("APPVEYOR") - -proc cmdTemplate*(): string = - compilerPrefix & "$target --lib:lib --hints:on -d:testing --nimblePath:tests/deps $options $file" - -type - TTestAction* = enum - actionCompile = "compile" - actionRun = "run" - actionReject = "reject" - actionRunNoSpec = "runNoSpec" - TResultEnum* = enum - reNimcCrash, # nim compiler seems to have crashed - reMsgsDiffer, # error messages differ - reFilesDiffer, # expected and given filenames differ - reLinesDiffer, # expected and given line numbers differ - reOutputsDiffer, - reExitcodesDiffer, - reInvalidPeg, - reCodegenFailure, - reCodeNotFound, - reExeNotFound, - reInstallFailed # package installation failed - reBuildFailed # package building failed - reIgnored, # test is ignored - reSuccess # test was successful - TTarget* = enum - targetC = "C" - targetCpp = "C++" - targetObjC = "ObjC" - targetJS = "JS" - - TSpec* = object - action*: TTestAction - file*, cmd*: string - outp*: string - line*, column*: int - tfile*: string - tline*, tcolumn*: int - exitCode*: int - msg*: string - ccodeCheck*: string - maxCodeSize*: int - err*: TResultEnum - substr*, sortoutput*: bool - targets*: set[TTarget] - nimout*: string - -const - targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"] - targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"] - -when not declared(parseCfgBool): - # candidate for the stdlib: - proc parseCfgBool(s: string): bool = - case normalize(s) - of "y", "yes", "true", "1", "on": result = true - of "n", "no", "false", "0", "off": result = false - else: raise newException(ValueError, "cannot interpret as a bool: " & s) - -proc extractSpec(filename: string): string = - const tripleQuote = "\"\"\"" - var x = readFile(filename).string - var a = x.find(tripleQuote) - var b = x.find(tripleQuote, a+3) - # look for """ only in the first section - if a >= 0 and b > a and a < 40: - result = x.substr(a+3, b-1).replace("'''", tripleQuote) - else: - #echo "warning: file does not contain spec: " & filename - result = "" - -when not defined(nimhygiene): - {.pragma: inject.} - -template parseSpecAux(fillResult: untyped) = - var ss = newStringStream(extractSpec(filename)) - var p {.inject.}: CfgParser - open(p, ss, filename, 1) - while true: - var e {.inject.} = next(p) - case e.kind - of cfgEof: break - of cfgSectionStart, cfgOption, cfgError: - echo ignoreMsg(p, e) - of cfgKeyValuePair: - fillResult - close(p) - -proc specDefaults*(result: var TSpec) = - result.msg = "" - result.outp = "" - result.nimout = "" - result.ccodeCheck = "" - result.cmd = cmdTemplate() - result.line = 0 - result.column = 0 - result.tfile = "" - result.tline = 0 - result.tcolumn = 0 - result.maxCodeSize = 0 - -proc parseTargets*(value: string): set[TTarget] = - for v in value.normalize.splitWhitespace: - case v - of "c": result.incl(targetC) - of "cpp", "c++": result.incl(targetCpp) - of "objc": result.incl(targetObjC) - of "js": result.incl(targetJS) - else: echo "target ignored: " & v - -proc parseSpec*(filename: string): TSpec = - specDefaults(result) - result.file = filename - parseSpecAux: - case normalize(e.key) - of "action": - case e.value.normalize - of "compile": result.action = actionCompile - of "run": result.action = actionRun - of "reject": result.action = actionReject - else: echo ignoreMsg(p, e) - of "file": result.file = e.value - of "line": discard parseInt(e.value, result.line) - of "column": discard parseInt(e.value, result.column) - of "tfile": result.tfile = e.value - of "tline": discard parseInt(e.value, result.tline) - of "tcolumn": discard parseInt(e.value, result.tcolumn) - of "output": - result.action = actionRun - result.outp = e.value - of "outputsub": - result.action = actionRun - result.outp = e.value - result.substr = true - of "sortoutput": - result.sortoutput = parseCfgBool(e.value) - of "exitcode": - discard parseInt(e.value, result.exitCode) - result.action = actionRun - of "msg": - result.msg = e.value - if result.action != actionRun: - result.action = actionCompile - of "errormsg", "errmsg": - result.msg = e.value - result.action = actionReject - of "nimout": - result.nimout = e.value - of "disabled": - case e.value.normalize - of "y", "yes", "true", "1", "on": result.err = reIgnored - of "n", "no", "false", "0", "off": discard - of "win", "windows": - when defined(windows): result.err = reIgnored - of "linux": - when defined(linux): result.err = reIgnored - of "bsd": - when defined(bsd): result.err = reIgnored - of "macosx": - when defined(macosx): result.err = reIgnored - of "unix": - when defined(unix): result.err = reIgnored - of "posix": - when defined(posix): result.err = reIgnored - of "travis": - if isTravis: result.err = reIgnored - of "appveyor": - if isAppVeyor: result.err = reIgnored - else: - raise newException(ValueError, "cannot interpret as a bool: " & e.value) - of "cmd": - if e.value.startsWith("nim "): - result.cmd = compilerPrefix & e.value[4..^1] - else: - result.cmd = e.value - of "ccodecheck": result.ccodeCheck = e.value - of "maxcodesize": discard parseInt(e.value, result.maxCodeSize) - of "target", "targets": - for v in e.value.normalize.splitWhitespace: - case v - of "c": result.targets.incl(targetC) - of "cpp", "c++": result.targets.incl(targetCpp) - of "objc": result.targets.incl(targetObjC) - of "js": result.targets.incl(targetJS) - else: echo ignoreMsg(p, e) - else: echo ignoreMsg(p, e) diff --git a/tests/testament/testamenthtml.templ b/tests/testament/testamenthtml.templ deleted file mode 100644 index 9190f370e..000000000 --- a/tests/testament/testamenthtml.templ +++ /dev/null @@ -1,297 +0,0 @@ -#? stdtmpl(subsChar = '%', metaChar = '#', emit = "outfile.write") -#import strutils -# -#proc htmlQuote*(raw: string): string = -# result = raw.multiReplace( -# ("&", "&"), -# ("\"", """), -# ("'", "'"), -# ("<", "<"), -# (">", ">") -# ) -# -#end proc -#proc generateHtmlBegin*(outfile: File) = -<!DOCTYPE html> -<html> -<head> - <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <title>Testament Test Results</title> - <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.0/jquery.min.js" integrity="sha256-ihAoc6M/JPfrIiIeayPE9xjin4UWjsx2mjW/rtmxLM4=" crossorigin="anonymous"></script> - <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script> - <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha256-916EbMg70RQy9LHiGkXzG8hSg9EdNy97GazNG/aiY1w=" crossorigin="anonymous" /> - <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap-theme.min.css" integrity="sha256-ZT4HPpdCOt2lvDkXokHuhJfdOKSPFLzeAJik5U/Q+l4=" crossorigin="anonymous" /> - <script> - /** - * Callback function that is executed for each Element in an array. - * @callback executeForElement - * @param {Element} elem Element to operate on - */ - - /** - * - * @param {number} index - * @param {Element[]} elemArray - * @param {executeForElement} executeOnItem - */ - function executeAllAsync(elemArray, index, executeOnItem) { - for (var i = 0; index < elemArray.length && i < 100; i++ , index++) { - var item = elemArray[index]; - executeOnItem(item); - } - if (index < elemArray.length) { - setTimeout(executeAllAsync, 0, elemArray, index, executeOnItem); - } - } - - /** @param {Element} elem */ - function executeShowOnElement(elem) { - while (elem.classList.contains("hidden")) { - elem.classList.remove("hidden"); - } - } - - /** @param {Element} elem */ - function executeHideOnElement(elem) { - if (!elem.classList.contains("hidden")) { - elem.classList.add("hidden"); - } - } - - /** @param {Element} elem */ - function executeExpandOnElement(elem) { - $(elem).collapse("show"); - } - - /** @param {Element} elem */ - function executeCollapseOnElement(elem) { - $(elem).collapse("hide"); - } - - /** - * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) - * @param {executeForElement} executeOnEachPanel - */ - function wholePanelAll(category, executeOnEachPanel) { - var selector = "div.panel"; - if (typeof category === "string" && category) { - selector += "-" + category; - } - - var jqPanels = $(selector); - /** @type {Element[]} */ - var elemArray = jqPanels.toArray(); - - setTimeout(executeAllAsync, 0, elemArray, 0, executeOnEachPanel); - } - - /** - * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) - * @param {executeForElement} executeOnEachPanel - */ - function panelBodyAll(category, executeOnEachPanelBody) { - var selector = "div.panel"; - if (typeof category === "string" && category) { - selector += "-" + category; - } - - var jqPanels = $(selector); - - var jqPanelBodies = $("div.panel-body", jqPanels); - /** @type {Element[]} */ - var elemArray = jqPanelBodies.toArray(); - - setTimeout(executeAllAsync, 0, elemArray, 0, executeOnEachPanelBody); - } - - /** - * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) - */ - function showAll(category) { - wholePanelAll(category, executeShowOnElement); - } - - /** - * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) - */ - function hideAll(category) { - wholePanelAll(category, executeHideOnElement); - } - - /** - * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) - */ - function expandAll(category) { - panelBodyAll(category, executeExpandOnElement); - } - - /** - * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) - */ - function collapseAll(category) { - panelBodyAll(category, executeCollapseOnElement); - } - </script> -</head> -<body> - <div class="container"> - <h1>Testament Test Results <small>Nim Tester</small></h1> -#end proc -#proc generateHtmlAllTestsBegin*(outfile: File, machine, commit, branch: string, -# totalCount: BiggestInt, -# successCount: BiggestInt, successPercentage: string, -# ignoredCount: BiggestInt, ignoredPercentage: string, -# failedCount: BiggestInt, failedPercentage: string, onlyFailing = false) = - <dl class="dl-horizontal"> - <dt>Hostname</dt> - <dd>%machine</dd> - <dt>Git Commit</dt> - <dd><code>%commit</code></dd> - <dt title="Git Branch reference">Branch ref.</dt> - <dd>%branch</dd> - </dl> - <dl class="dl-horizontal"> - <dt>All Tests</dt> - <dd> - <span class="glyphicon glyphicon-th-list"></span> - %totalCount - </dd> - <dt>Successful Tests</dt> - <dd> - <span class="glyphicon glyphicon-ok-sign"></span> - %successCount (%successPercentage) - </dd> - <dt>Skipped Tests</dt> - <dd> - <span class="glyphicon glyphicon-question-sign"></span> - %ignoredCount (%ignoredPercentage) - </dd> - <dt>Failed Tests</dt> - <dd> - <span class="glyphicon glyphicon-exclamation-sign"></span> - %failedCount (%failedPercentage) - </dd> - </dl> - <div class="table-responsive"> - <table class="table table-condensed"> -# if not onlyFailing: - <tr> - <th class="text-right" style="vertical-align:middle">All Tests</th> - <td> - <div class="btn-group"> - <button class="btn btn-default" type="button" onclick="showAll();">Show All</button> - <button class="btn btn-default" type="button" onclick="hideAll();">Hide All</button> - <button class="btn btn-default" type="button" onclick="expandAll();">Expand All</button> - <button class="btn btn-default" type="button" onclick="collapseAll();">Collapse All</button> - </div> - </td> - </tr> - <tr> - <th class="text-right" style="vertical-align:middle">Successful Tests</th> - <td> - <div class="btn-group"> - <button class="btn btn-default" type="button" onclick="showAll('success');">Show All</button> - <button class="btn btn-default" type="button" onclick="hideAll('success');">Hide All</button> - <button class="btn btn-default" type="button" onclick="expandAll('success');">Expand All</button> - <button class="btn btn-default" type="button" onclick="collapseAll('success');">Collapse All</button> - </div> - </td> - </tr> -# end if - <tr> - <th class="text-right" style="vertical-align:middle">Skipped Tests</th> - <td> - <div class="btn-group"> - <button class="btn btn-default" type="button" onclick="showAll('info');">Show All</button> - <button class="btn btn-default" type="button" onclick="hideAll('info');">Hide All</button> - <button class="btn btn-default" type="button" onclick="expandAll('info');">Expand All</button> - <button class="btn btn-default" type="button" onclick="collapseAll('info');">Collapse All</button> - </div> - </td> - </tr> - <tr> - <th class="text-right" style="vertical-align:middle">Failed Tests</th> - <td> - <div class="btn-group"> - <button class="btn btn-default" type="button" onclick="showAll('danger');">Show All</button> - <button class="btn btn-default" type="button" onclick="hideAll('danger');">Hide All</button> - <button class="btn btn-default" type="button" onclick="expandAll('danger');">Expand All</button> - <button class="btn btn-default" type="button" onclick="collapseAll('danger');">Collapse All</button> - </div> - </td> - </tr> - </table> - </div> - <div class="panel-group"> -#end proc -#proc generateHtmlTestresultPanelBegin*(outfile: File, trId, name, target, category, -# action, resultDescription, timestamp, result, resultSign, -# panelCtxClass, textCtxClass, bgCtxClass: string) = - <div id="panel-testResult-%trId" class="panel panel-%panelCtxClass"> - <div class="panel-heading" style="cursor:pointer" data-toggle="collapse" data-target="#panel-body-testResult-%trId" aria-controls="panel-body-testResult-%trId" aria-expanded="false"> - <div class="row"> - <h4 class="col-xs-3 col-sm-1 panel-title"> - <span class="glyphicon glyphicon-%resultSign-sign"></span> - <strong>%resultDescription</strong> - </h4> - <h4 class="col-xs-1 panel-title"><span class="badge">%target</span></h4> - <h4 class="col-xs-5 col-sm-7 panel-title" title="%name"><code class="text-%textCtxClass">%name</code></h4> - <h4 class="col-xs-3 col-sm-3 panel-title text-right"><span class="badge">%category</span></h4> - </div> - </div> - <div id="panel-body-testResult-%trId" class="panel-body collapse bg-%bgCtxClass"> - <dl class="dl-horizontal"> - <dt>Name</dt> - <dd><code class="text-%textCtxClass">%name</code></dd> - <dt>Category</dt> - <dd><span class="badge">%category</span></dd> - <dt>Timestamp</dt> - <dd>%timestamp</dd> - <dt>Nim Action</dt> - <dd><code class="text-%textCtxClass">%action</code></dd> - <dt>Nim Backend Target</dt> - <dd><span class="badge">%target</span></dd> - <dt>Code</dt> - <dd><code class="text-%textCtxClass">%result</code></dd> - </dl> -#end proc -#proc generateHtmlTestresultOutputDetails*(outfile: File, expected, gotten: string) = - <div class="table-responsive"> - <table class="table table-condensed"> - <thead> - <tr> - <th>Expected</th> - <th>Actual</th> - </tr> - </thead> - <tbody> - <tr> - <td><pre>%expected</pre></td> - <td><pre>%gotten</pre></td> - </tr> - </tbody> - </table> - </div> -#end proc -#proc generateHtmlTestresultOutputNone*(outfile: File) = - <p class="sr-only">No output details</p> -#end proc -#proc generateHtmlTestresultPanelEnd*(outfile: File) = - </div> - </div> -#end proc -#proc generateHtmlAllTestsEnd*(outfile: File) = - </div> -#end proc -#proc generateHtmlEnd*(outfile: File, timestamp: string) = - <hr /> - <footer> - <p> - Report generated by: <code>testament</code> – Nim Tester - <br /> - Made with Nim. Generated on: %timestamp - </p> - </footer> - </div> -</body> -</html> diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim deleted file mode 100644 index 15f44f871..000000000 --- a/tests/testament/tester.nim +++ /dev/null @@ -1,522 +0,0 @@ -# -# -# Nim Tester -# (c) Copyright 2017 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This program verifies Nim against the testcases. - -import - parseutils, strutils, pegs, os, osproc, streams, parsecfg, json, - marshal, backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, compiler/nodejs, times, sets, md5 - -const - resultsFile = "testresults.html" - #jsonFile = "testresults.json" # not used - Usage = """Usage: - tester [options] command [arguments] - -Command: - all run all tests - c|cat|category <category> run all the tests of a certain category - r|run <test> run single test file - html generate $1 from the database -Arguments: - arguments are passed to the compiler -Options: - --print also print results to the console - --failing only show failing/ignored tests - --targets:"c c++ js objc" run tests for specified targets (default: all) - --nim:path use a particular nim executable (default: compiler/nim) -""" % resultsFile - -type - Category = distinct string - TResults = object - total, passed, skipped: int - data: string - - TTest = object - name: string - cat: Category - options: string - action: TTestAction - startTime: float - -# ---------------------------------------------------------------------------- - -let - pegLineError = - peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" - pegLineTemplate = - peg""" - {[^(]*} '(' {\d+} ', ' {\d+} ') ' - 'template/generic instantiation' ( ' of `' [^`]+ '`' )? ' from here' .* - """ - pegOtherError = peg"'Error:' \s* {.*}" - pegSuccess = peg"'Hint: operation successful'.*" - pegOfInterest = pegLineError / pegOtherError - -var targets = {low(TTarget)..high(TTarget)} - -proc normalizeMsg(s: string): string = - result = newStringOfCap(s.len+1) - for x in splitLines(s): - if result.len > 0: result.add '\L' - result.add x.strip - -proc getFileDir(filename: string): string = - result = filename.splitFile().dir - if not result.isAbsolute(): - result = getCurrentDir() / result - -proc nimcacheDir(filename, options: string, target: TTarget): string = - ## Give each test a private nimcache dir so they don't clobber each other's. - let hashInput = options & $target - return "nimcache" / (filename & '_' & hashInput.getMD5) - -proc callCompiler(cmdTemplate, filename, options: string, - target: TTarget, extraOptions=""): TSpec = - let nimcache = nimcacheDir(filename, options, target) - let options = options & " " & ("--nimCache:" & nimcache).quoteShell & extraOptions - let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], - "options", options, "file", filename.quoteShell, - "filedir", filename.getFileDir()]) - var p = startProcess(command=c[0], args=c[1.. ^1], - options={poStdErrToStdOut, poUsePath}) - let outp = p.outputStream - var suc = "" - var err = "" - var tmpl = "" - var x = newStringOfCap(120) - result.nimout = "" - while outp.readLine(x.TaintedString) or running(p): - result.nimout.add(x & "\n") - if x =~ pegOfInterest: - # `err` should contain the last error/warning message - err = x - elif x =~ pegLineTemplate and err == "": - # `tmpl` contains the last template expansion before the error - tmpl = x - elif x =~ pegSuccess: - suc = x - close(p) - result.msg = "" - result.file = "" - result.outp = "" - result.line = 0 - result.column = 0 - result.tfile = "" - result.tline = 0 - result.tcolumn = 0 - if tmpl =~ pegLineTemplate: - result.tfile = extractFilename(matches[0]) - result.tline = parseInt(matches[1]) - result.tcolumn = parseInt(matches[2]) - if err =~ pegLineError: - result.file = extractFilename(matches[0]) - result.line = parseInt(matches[1]) - result.column = parseInt(matches[2]) - result.msg = matches[3] - elif err =~ pegOtherError: - result.msg = matches[0] - elif suc =~ pegSuccess: - result.err = reSuccess - -proc callCCompiler(cmdTemplate, filename, options: string, - target: TTarget): TSpec = - let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], - "options", options, "file", filename.quoteShell, - "filedir", filename.getFileDir()]) - var p = startProcess(command="gcc", args=c[5 .. ^1], - options={poStdErrToStdOut, poUsePath}) - let outp = p.outputStream - var x = newStringOfCap(120) - result.nimout = "" - result.msg = "" - result.file = "" - result.outp = "" - result.line = -1 - while outp.readLine(x.TaintedString) or running(p): - result.nimout.add(x & "\n") - close(p) - if p.peekExitCode == 0: - result.err = reSuccess - -proc initResults: TResults = - result.total = 0 - result.passed = 0 - result.skipped = 0 - result.data = "" - -#proc readResults(filename: string): TResults = # not used -# result = marshal.to[TResults](readFile(filename).string) - -#proc writeResults(filename: string, r: TResults) = # not used -# writeFile(filename, $$r) - -proc `$`(x: TResults): string = - result = ("Tests passed: $1 / $3 <br />\n" & - "Tests skipped: $2 / $3 <br />\n") % - [$x.passed, $x.skipped, $x.total] - -proc addResult(r: var TResults, test: TTest, target: TTarget, - expected, given: string, success: TResultEnum) = - let name = test.name.extractFilename & " " & $target & test.options - let duration = epochTime() - test.startTime - let durationStr = duration.formatFloat(ffDecimal, precision = 8) - backend.writeTestResult(name = name, - category = test.cat.string, - target = $target, - action = $test.action, - result = $success, - expected = expected, - given = given) - r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) - if success == reSuccess: - styledEcho fgGreen, "PASS: ", fgCyan, alignLeft(name, 60), fgBlue, " (", durationStr, " secs)" - elif success == reIgnored: - styledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name - else: - styledEcho styleBright, fgRed, "FAIL: ", fgCyan, name - styledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" - styledEcho styleBright, fgRed, "Failure: ", $success - styledEcho fgYellow, "Expected:" - styledEcho styleBright, expected, "\n" - styledEcho fgYellow, "Gotten:" - styledEcho styleBright, given, "\n" - - if existsEnv("APPVEYOR"): - let (outcome, msg) = - if success == reSuccess: - ("Passed", "") - elif success == reIgnored: - ("Skipped", "") - else: - ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given) - var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options, - "-Framework", "nim-testament", "-FileName", - test.cat.string, - "-Outcome", outcome, "-ErrorMessage", msg, - "-Duration", $(duration*1000).int], - options={poStdErrToStdOut, poUsePath, poParentStreams}) - discard waitForExit(p) - close(p) - -proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = - if strip(expected.msg) notin strip(given.msg): - r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer) - elif expected.nimout.len > 0 and expected.nimout.normalizeMsg notin given.nimout.normalizeMsg: - r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer) - elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and - "internal error:" notin expected.msg: - r.addResult(test, target, expected.file, given.file, reFilesDiffer) - elif expected.line != given.line and expected.line != 0 or - expected.column != given.column and expected.column != 0: - r.addResult(test, target, $expected.line & ':' & $expected.column, - $given.line & ':' & $given.column, - reLinesDiffer) - elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and - "internal error:" notin expected.msg: - r.addResult(test, target, expected.tfile, given.tfile, reFilesDiffer) - elif expected.tline != given.tline and expected.tline != 0 or - expected.tcolumn != given.tcolumn and expected.tcolumn != 0: - r.addResult(test, target, $expected.tline & ':' & $expected.tcolumn, - $given.tline & ':' & $given.tcolumn, - reLinesDiffer) - else: - r.addResult(test, target, expected.msg, given.msg, reSuccess) - inc(r.passed) - -proc generatedFile(test: TTest, target: TTarget): string = - let (_, name, _) = test.name.splitFile - let ext = targetToExt[target] - result = nimcacheDir(test.name, test.options, target) / - (if target == targetJS: "" else: "compiler_") & - name.changeFileExt(ext) - -proc needsCodegenCheck(spec: TSpec): bool = - result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0 - -proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var string, - given: var TSpec) = - try: - let genFile = generatedFile(test, target) - let contents = readFile(genFile).string - let check = spec.ccodeCheck - if check.len > 0: - if check[0] == '\\': - # little hack to get 'match' support: - if not contents.match(check.peg): - given.err = reCodegenFailure - elif contents.find(check.peg) < 0: - given.err = reCodegenFailure - expectedMsg = check - if spec.maxCodeSize > 0 and contents.len > spec.maxCodeSize: - given.err = reCodegenFailure - given.msg = "generated code size: " & $contents.len - expectedMsg = "max allowed size: " & $spec.maxCodeSize - except ValueError: - given.err = reInvalidPeg - echo getCurrentExceptionMsg() - except IOError: - given.err = reCodeNotFound - echo getCurrentExceptionMsg() - -proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) = - let exp = expectedNimout.strip.replace("\C\L", "\L") - let giv = given.nimout.strip.replace("\C\L", "\L") - if exp notin giv: - given.err = reMsgsDiffer - -proc makeDeterministic(s: string): string = - var x = splitLines(s) - sort(x, system.cmp) - result = join(x, "\n") - -proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, - expected: TSpec; r: var TResults) = - var expectedmsg: string = "" - var givenmsg: string = "" - if given.err == reSuccess: - if expected.needsCodegenCheck: - codegenCheck(test, target, expected, expectedmsg, given) - givenmsg = given.msg - if expected.nimout.len > 0: - expectedmsg = expected.nimout - givenmsg = given.nimout.strip - nimoutCheck(test, expectedmsg, given) - else: - givenmsg = given.nimout.strip - if given.err == reSuccess: inc(r.passed) - r.addResult(test, target, expectedmsg, givenmsg, given.err) - -proc testSpec(r: var TResults, test: TTest, target = targetC) = - let tname = test.name.addFileExt(".nim") - #echo "TESTING ", tname - var expected: TSpec - if test.action != actionRunNoSpec: - expected = parseSpec(tname) - if test.action == actionRun and expected.action == actionCompile: - expected.action = actionRun - else: - specDefaults expected - expected.action = actionRunNoSpec - - if expected.err == reIgnored: - r.addResult(test, target, "", "", reIgnored) - inc(r.skipped) - inc(r.total) - return - - if expected.targets == {}: - expected.targets.incl(target) - - for target in expected.targets: - inc(r.total) - if target notin targets: - r.addResult(test, target, "", "", reIgnored) - inc(r.skipped) - continue - - case expected.action - of actionCompile: - var given = callCompiler(expected.cmd, test.name, test.options, target, - extraOptions=" --stdout --hint[Path]:off --hint[Processing]:off") - compilerOutputTests(test, target, given, expected, r) - of actionRun, actionRunNoSpec: - # In this branch of code "early return" pattern is clearer than deep - # nested conditionals - the empty rows in between to clarify the "danger" - var given = callCompiler(expected.cmd, test.name, test.options, - target) - - if given.err != reSuccess: - r.addResult(test, target, "", given.msg, given.err) - continue - - let isJsTarget = target == targetJS - var exeFile: string - if isJsTarget: - let (_, file, _) = splitFile(tname) - exeFile = nimcacheDir(test.name, test.options, target) / file & ".js" - else: - exeFile = changeFileExt(tname, ExeExt) - - if not existsFile(exeFile): - r.addResult(test, target, expected.outp, "executable not found", reExeNotFound) - continue - - let nodejs = if isJsTarget: findNodeJs() else: "" - if isJsTarget and nodejs == "": - r.addResult(test, target, expected.outp, "nodejs binary not in PATH", - reExeNotFound) - continue - - let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile - var (buf, exitCode) = execCmdEx(exeCmd, options = {poStdErrToStdOut}) - - # Treat all failure codes from nodejs as 1. Older versions of nodejs used - # to return other codes, but for us it is sufficient to know that it's not 0. - if exitCode != 0: exitCode = 1 - - let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string)) - else: strip(buf.string) - let expectedOut = strip(expected.outp) - - if exitCode != expected.exitCode: - r.addResult(test, target, "exitcode: " & $expected.exitCode, - "exitcode: " & $exitCode & "\n\nOutput:\n" & - bufB, reExitCodesDiffer) - continue - - if bufB != expectedOut and expected.action != actionRunNoSpec: - if not (expected.substr and expectedOut in bufB): - given.err = reOutputsDiffer - r.addResult(test, target, expected.outp, bufB, reOutputsDiffer) - continue - - compilerOutputTests(test, target, given, expected, r) - continue - - of actionReject: - var given = callCompiler(expected.cmd, test.name, test.options, - target) - cmpMsgs(r, expected, given, test, target) - continue - -proc testNoSpec(r: var TResults, test: TTest, target = targetC) = - # does not extract the spec because the file is not supposed to have any - #let tname = test.name.addFileExt(".nim") - inc(r.total) - let given = callCompiler(cmdTemplate(), test.name, test.options, target) - r.addResult(test, target, "", given.msg, given.err) - if given.err == reSuccess: inc(r.passed) - -proc testC(r: var TResults, test: TTest) = - # runs C code. Doesn't support any specs, just goes by exit code. - let tname = test.name.addFileExt(".c") - inc(r.total) - styledEcho "Processing ", fgCyan, extractFilename(tname) - var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, targetC) - if given.err != reSuccess: - r.addResult(test, targetC, "", given.msg, given.err) - elif test.action == actionRun: - let exeFile = changeFileExt(test.name, ExeExt) - var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath}) - if exitCode != 0: given.err = reExitCodesDiffer - if given.err == reSuccess: inc(r.passed) - -proc testExec(r: var TResults, test: TTest) = - # runs executable or script, just goes by exit code - inc(r.total) - let (outp, errC) = execCmdEx(test.options.strip()) - var given: TSpec - specDefaults(given) - if errC == 0: - given.err = reSuccess - else: - given.err = reExitCodesDiffer - given.msg = outp.string - - if given.err == reSuccess: inc(r.passed) - r.addResult(test, targetC, "", given.msg, given.err) - -proc makeTest(test, options: string, cat: Category, action = actionCompile, - env: string = ""): TTest = - # start with 'actionCompile', will be overwritten in the spec: - result = TTest(cat: cat, name: test, options: options, - action: action, startTime: epochTime()) - -when defined(windows): - const - # array of modules disabled from compilation test of stdlib. - disabledFiles = ["coro.nim", "fsmonitor.nim"] -else: - const - # array of modules disabled from compilation test of stdlib. - disabledFiles = ["-"] - -include categories - -# proc runCaasTests(r: var TResults) = -# for test, output, status, mode in caasTestsRunner(): -# r.addResult(test, "", output & "-> " & $mode, -# if status: reSuccess else: reOutputsDiffer) - -proc main() = - os.putenv "NIMTEST_COLOR", "never" - os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES" - - backend.open() - var optPrintResults = false - var optFailing = false - - var targetsStr = "" - - var p = initOptParser() - p.next() - while p.kind == cmdLongoption: - case p.key.string.normalize - of "print", "verbose": optPrintResults = true - of "failing": optFailing = true - of "pedantic": discard "now always enabled" - of "targets": - targetsStr = p.val.string - targets = parseTargets(targetsStr) - of "nim": compilerPrefix = p.val.string - else: quit Usage - p.next() - if p.kind != cmdArgument: quit Usage - var action = p.key.string.normalize - p.next() - var r = initResults() - case action - of "all": - let testsDir = "tests" & DirSep - var myself = quoteShell(findExe("tests" / "testament" / "tester")) - if targetsStr.len > 0: - myself &= " " & quoteShell("--targets:" & targetsStr) - - myself &= " " & quoteShell("--nim:" & compilerPrefix) - - var cmds: seq[string] = @[] - let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: "" - for kind, dir in walkDir(testsDir): - assert testsDir.startsWith(testsDir) - let cat = dir[testsDir.len .. ^1] - if kind == pcDir and cat notin ["testament", "testdata", "nimcache"]: - cmds.add(myself & " cat " & quoteShell(cat) & rest) - for cat in AdditionalCategories: - cmds.add(myself & " cat " & quoteShell(cat) & rest) - quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}) - of "c", "cat", "category": - var cat = Category(p.key) - p.next - processCategory(r, cat, p.cmdLineRest.string) - of "r", "run": - let (dir, file) = splitPath(p.key.string) - let (_, subdir) = splitPath(dir) - var cat = Category(subdir) - processSingleTest(r, cat, p.cmdLineRest.string, file) - of "html": - generateHtml(resultsFile, optFailing) - else: - quit Usage - - if optPrintResults: - if action == "html": openDefaultBrowser(resultsFile) - else: echo r, r.data - backend.close() - var failed = r.total - r.passed - r.skipped - if failed != 0: - echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ", - r.skipped, " failed: ", failed - quit(QuitFailure) - -if paramCount() == 0: - quit Usage -main() diff --git a/tests/testament/tester.nim.cfg b/tests/testament/tester.nim.cfg deleted file mode 100644 index 27fd67075..000000000 --- a/tests/testament/tester.nim.cfg +++ /dev/null @@ -1 +0,0 @@ -path = "$nim" # For compiler/nodejs |