diff options
Diffstat (limited to 'tests/testament')
-rw-r--r-- | tests/testament/backend.nim | 122 | ||||
-rw-r--r-- | tests/testament/caasdriver.nim | 195 | ||||
-rw-r--r-- | tests/testament/categories.nim | 408 | ||||
-rw-r--r-- | tests/testament/css/boilerplate.css | 138 | ||||
-rw-r--r-- | tests/testament/css/style.css | 114 | ||||
-rw-r--r-- | tests/testament/htmlgen.nim | 228 | ||||
-rw-r--r-- | tests/testament/specs.nim | 172 | ||||
-rw-r--r-- | tests/testament/t16576.nim | 7 | ||||
-rw-r--r-- | tests/testament/tester.nim | 458 | ||||
-rw-r--r-- | tests/testament/tester.nim.cfg | 1 | ||||
-rw-r--r-- | tests/testament/tinlinemsg.nim | 8 | ||||
-rw-r--r-- | tests/testament/tjoinable.nim | 9 | ||||
-rw-r--r-- | tests/testament/treject.nim | 6 | ||||
-rw-r--r-- | tests/testament/tshould_not_work.nim | 53 | ||||
-rw-r--r-- | tests/testament/tspecialpaths.nim | 9 |
15 files changed, 92 insertions, 1836 deletions
diff --git a/tests/testament/backend.nim b/tests/testament/backend.nim deleted file mode 100644 index 671b5c8b7..000000000 --- a/tests/testament/backend.nim +++ /dev/null @@ -1,122 +0,0 @@ -# -# -# The Nim Tester -# (c) Copyright 2015 Andreas Rumpf -# -# Look at license.txt for more info. -# All rights reserved. - -import strutils, db_sqlite, os, osproc - -var db: DbConn - -proc createDb() = - db.exec(sql""" - create table if not exists Machine( - id integer primary key, - name varchar(100) not null, - os varchar(20) not null, - cpu varchar(20) not null - );""") - - db.exec(sql""" - create table if not exists [Commit]( - id integer primary key, - hash varchar(256) not null, - branch varchar(50) not null - );""") - - db.exec(sql""" - create table if not exists TestResult( - id integer primary key, - name varchar(100) not null, - category varchar(100) not null, - target varchar(20) not null, - action varchar(10) not null, - result varchar(30) not null, - [commit] int not null, - machine int not null, - expected varchar(10000) not null, - given varchar(10000) not null, - created timestamp not null default (DATETIME('now')), - - foreign key ([commit]) references [commit](id), - foreign key (machine) references machine(id) - );""") - - #db.exec(sql""" - # --create unique index if not exists TsstNameIx on TestResult(name); - # """, []) - -type - MachineId* = distinct int64 - CommitId = distinct int64 - -proc `$`*(id: MachineId): string {.borrow.} -proc `$`(id: CommitId): string {.borrow.} - -var - thisMachine: MachineId - thisCommit: CommitId - -proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip - -proc getMachine*(db: DbConn): MachineId = - var name = "hostname"() - if name.len == 0: - name = when defined(posix): getenv"HOSTNAME".string - else: getenv"COMPUTERNAME".string - if name.len == 0: - quit "cannot determine the machine name" - - let id = db.getValue(sql"select id from Machine where name = ?", name) - if id.len > 0: - result = id.parseInt.MachineId - else: - result = db.insertId(sql"insert into Machine(name, os, cpu) values (?,?,?)", - name, system.hostOS, system.hostCPU).MachineId - -proc getCommit(db: DbConn): CommitId = - const commLen = "commit ".len - let hash = "git log -n 1"()[commLen..commLen+10] - let branch = "git symbolic-ref --short HEAD"() - if hash.len == 0 or branch.len == 0: quit "cannot determine git HEAD" - - let id = db.getValue(sql"select id from [Commit] where hash = ? and branch = ?", - hash, branch) - if id.len > 0: - result = id.parseInt.CommitId - else: - result = db.insertId(sql"insert into [Commit](hash, branch) values (?, ?)", - hash, branch).CommitId - -proc writeTestResult*(name, category, target, - action, result, expected, given: string) = - let id = db.getValue(sql"""select id from TestResult - where name = ? and category = ? and target = ? and - machine = ? and [commit] = ?""", - name, category, target, - thisMachine, thisCommit) - if id.len > 0: - db.exec(sql"""update TestResult - set action = ?, result = ?, expected = ?, given = ? - where id = ?""", action, result, expected, given, id) - else: - db.exec(sql"""insert into TestResult(name, category, target, - action, - result, expected, given, - [commit], machine) - values (?,?,?,?,?,?,?,?,?) """, name, category, target, - action, - result, expected, given, - thisCommit, thisMachine) - -proc open*() = - let dbFile = if existsEnv("TRAVIS") or existsEnv("APPVEYOR"): ":memory:" else: "testament.db" - db = open(connection=dbFile, user="testament", password="", - database="testament") - createDb() - thisMachine = getMachine(db) - thisCommit = getCommit(db) - -proc close*() = close(db) diff --git a/tests/testament/caasdriver.nim b/tests/testament/caasdriver.nim deleted file mode 100644 index 30383bddb..000000000 --- a/tests/testament/caasdriver.nim +++ /dev/null @@ -1,195 +0,0 @@ -import osproc, streams, os, strutils, re -{.experimental.} - -## Compiler as a service tester. -## -## Please read docs/idetools.txt for information about this. - - -type - TRunMode = enum - ProcRun, CaasRun, SymbolProcRun - - NimSession* = object - nim: Process # Holds the open process for CaasRun sessions, nil otherwise. - mode: TRunMode # Stores the type of run mode the session was started with. - lastOutput: string # Preserves the last output, needed for ProcRun mode. - filename: string # Appended to each command starting with '>'. Also a var. - modname: string # Like filename but without extension. - nimcache: string # Input script based name for the nimcache dir. - -const - modes = [CaasRun, ProcRun, SymbolProcRun] - filenameReplaceVar = "$TESTNIM" - moduleReplaceVar = "$MODULE" - silentReplaceVar = "$SILENT" - silentReplaceText = "--verbosity:0 --hints:off" - -var - TesterDir = getAppDir() / ".." - NimBin = TesterDir / "../bin/nim" - -proc replaceVars(session: var NimSession, text: string): string = - result = text.replace(filenameReplaceVar, session.filename) - result = result.replace(moduleReplaceVar, session.modname) - result = result.replace(silentReplaceVar, silentReplaceText) - -proc startNimSession(project, script: string, mode: TRunMode): - NimSession = - let (dir, name, ext) = project.splitFile - result.mode = mode - result.lastOutput = "" - result.filename = name & ext - result.modname = name - - let (nimcacheDir, nimcacheName, nimcacheExt) = script.splitFile - result.nimcache = "SymbolProcRun." & nimcacheName - - if mode == SymbolProcRun: - removeDir(nimcacheDir / result.nimcache) - else: - removeDir(nimcacheDir / "nimcache") - - if mode == CaasRun: - result.nim = startProcess(NimBin, workingDir = dir, - args = ["serve", "--server.type:stdin", name]) - -proc doCaasCommand(session: var NimSession, command: string): string = - assert session.mode == CaasRun - session.nim.inputStream.write(session.replaceVars(command) & "\n") - session.nim.inputStream.flush - - result = "" - - while true: - var line = TaintedString("") - if session.nim.outputStream.readLine(line): - if line.string == "": break - result.add(line.string & "\n") - else: - result = "FAILED TO EXECUTE: " & command & "\n" & result - break - -proc doProcCommand(session: var NimSession, command: string): string = - try: - assert session.mode == ProcRun or session.mode == SymbolProcRun - except: - result = "FAILED TO EXECUTE: " & command & "\n" & result - var - process = startProcess(NimBin, args = session.replaceVars(command).split) - stream = outputStream(process) - line = TaintedString("") - - result = "" - while stream.readLine(line): - if result.len > 0: result &= "\n" - result &= line.string - - process.close() - -proc doCommand(session: var NimSession, command: string) = - if session.mode == CaasRun: - if not session.nim.running: - session.lastOutput = "FAILED TO EXECUTE: " & command & "\n" & - "Exit code " & $session.nim.peekExitCode - return - session.lastOutput = doCaasCommand(session, - command & " " & session.filename) - else: - var command = command - # For symbol runs we prepend the necessary parameters to avoid clobbering - # the normal nimcache. - if session.mode == SymbolProcRun: - command = "--symbolFiles:on --nimcache:" & session.nimcache & - " " & command - session.lastOutput = doProcCommand(session, - command & " " & session.filename) - -proc destroy(session: var NimSession) {.destructor.} = - if session.mode == CaasRun: - session.nim.close - -proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): bool = - result = true - - var f = open(script) - var project = TaintedString("") - - if f.readLine(project): - var - s = startNimSession(script.parentDir / project.string, script, mode) - tline = TaintedString("") - ln = 1 - - while f.readLine(tline): - var line = tline.string - inc ln - - # Filter lines by run mode, removing the prefix if the mode is current. - for testMode in modes: - if line.startsWith($testMode): - if testMode != mode: - line = "" - else: - line = line[len($testMode)..len(line) - 1].strip - break - - if line.strip.len == 0: continue - - if line.startsWith("#"): - output.writeLine line - continue - elif line.startsWith(">"): - s.doCommand(line.substr(1).strip) - output.writeLine line, "\n", if verbose: s.lastOutput else: "" - else: - var expectMatch = true - var pattern = s.replaceVars(line) - if line.startsWith("!"): - pattern = pattern.substr(1).strip - expectMatch = false - - let actualMatch = - s.lastOutput.find(re(pattern, flags = {reStudy})) != -1 - - if expectMatch == actualMatch: - output.writeLine "SUCCESS ", line - else: - output.writeLine "FAILURE ", line - result = false - -iterator caasTestsRunner*(filter = "", verbose = false): tuple[test, - output: string, status: bool, - mode: TRunMode] = - for scenario in os.walkFiles(TesterDir / "caas/*.txt"): - if filter.len > 0 and find(scenario, filter) == -1: continue - for mode in modes: - var outStream = newStringStream() - let r = doScenario(scenario, outStream, mode, verbose) - yield (scenario, outStream.data, r, mode) - -when isMainModule: - var - filter = "" - failures = 0 - verbose = false - - for i in 0..paramCount() - 1: - let param = string(paramStr(i + 1)) - case param - of "verbose": verbose = true - else: filter = param - - if verbose and len(filter) > 0: - echo "Running only test cases matching filter '$1'" % [filter] - - for test, output, result, mode in caasTestsRunner(filter, verbose): - if not result or verbose: - echo "Mode ", $mode, " (", if result: "succeeded)" else: "failed)" - echo test - echo output - echo "---------\n" - if not result: - failures += 1 - - quit(failures) diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim deleted file mode 100644 index 125643d5a..000000000 --- a/tests/testament/categories.nim +++ /dev/null @@ -1,408 +0,0 @@ -# -# -# Nim Tester -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Include for the tester that contains test suites that test special features -## of the compiler. - -# included from tester.nim -# ---------------- ROD file tests --------------------------------------------- - -const - rodfilesDir = "tests/rodfiles" - nimcacheDir = rodfilesDir / "nimcache" - -proc delNimCache() = - try: - removeDir(nimcacheDir) - except OSError: - echo "[Warning] could not delete: ", nimcacheDir - -proc runRodFiles(r: var TResults, cat: Category, options: string) = - template test(filename: expr): stmt = - testSpec r, makeTest(rodfilesDir / filename, options, cat, actionRun) - - delNimCache() - - # test basic recompilation scheme: - test "hallo" - test "hallo" - # test incremental type information: - test "hallo2" - delNimCache() - - # test type converters: - test "aconv" - test "bconv" - delNimCache() - - # test G, A, B example from the documentation; test init sections: - test "deada" - test "deada2" - delNimCache() - - # test method generation: - test "bmethods" - test "bmethods2" - delNimCache() - - # test generics: - test "tgeneric1" - test "tgeneric2" - delNimCache() - -proc compileRodFiles(r: var TResults, cat: Category, options: string) = - template test(filename: expr): stmt = - testSpec r, makeTest(rodfilesDir / filename, options, cat) - - delNimCache() - # test DLL interfacing: - test "gtkex1" - test "gtkex2" - delNimCache() - -# --------------------- DLL generation tests ---------------------------------- - -proc safeCopyFile(src, dest: string) = - try: - copyFile(src, dest) - except OSError: - echo "[Warning] could not copy: ", src, " to ", dest - -proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = - const rpath = when defined(macosx): - " --passL:-rpath --passL:@loader_path" - else: - "" - - testSpec c, makeTest("lib/nimrtl.nim", - options & " --app:lib -d:createNimRtl", cat) - testSpec c, makeTest("tests/dll/server.nim", - options & " --app:lib -d:useNimRtl" & rpath, cat) - - - when defined(Windows): - # windows looks in the dir of the exe (yay!): - var nimrtlDll = DynlibFormat % "nimrtl" - safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll) - else: - # posix relies on crappy LD_LIBRARY_PATH (ugh!): - var libpath = getEnv"LD_LIBRARY_PATH".string - # Temporarily add the lib directory to LD_LIBRARY_PATH: - putEnv("LD_LIBRARY_PATH", "tests/dll:" & libpath) - defer: putEnv("LD_LIBRARY_PATH", libpath) - var nimrtlDll = DynlibFormat % "nimrtl" - safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll) - - testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl" & rpath, - cat, actionRun) - -proc dllTests(r: var TResults, cat: Category, options: string) = - # dummy compile result: - var c = initResults() - - runBasicDLLTest c, r, cat, options - runBasicDLLTest c, r, cat, options & " -d:release" - runBasicDLLTest c, r, cat, options & " --gc:boehm" - runBasicDLLTest c, r, cat, options & " -d:release --gc:boehm" - -# ------------------------------ GC tests ------------------------------------- - -proc gcTests(r: var TResults, cat: Category, options: string) = - template testWithoutMs(filename: expr): stmt = - testSpec r, makeTest("tests/gc" / filename, options, cat, actionRun) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release", cat, actionRun) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release -d:useRealtimeGC", cat, actionRun) - - template testWithoutBoehm(filename: expr): stmt = - testWithoutMs filename - testSpec r, makeTest("tests/gc" / filename, options & - " --gc:markAndSweep", cat, actionRun) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release --gc:markAndSweep", cat, actionRun) - template test(filename: expr): stmt = - testWithoutBoehm filename - when not defined(windows): - # AR: cannot find any boehm.dll on the net, right now, so disabled - # for windows: - testSpec r, makeTest("tests/gc" / filename, options & - " --gc:boehm", cat, actionRun) - testSpec r, makeTest("tests/gc" / filename, options & - " -d:release --gc:boehm", cat, actionRun) - - test "gcemscripten" - test "growobjcrash" - test "gcbench" - test "gcleak" - test "gcleak2" - test "gctest" - test "gcleak3" - test "gcleak4" - # Disabled because it works and takes too long to run: - #test "gcleak5" - testWithoutBoehm "weakrefs" - test "cycleleak" - testWithoutBoehm "closureleak" - testWithoutMs "refarrayleak" - - testWithoutBoehm "tlists" - testWithoutBoehm "thavlak" - - test "stackrefleak" - test "cyclecollector" - -proc longGCTests(r: var TResults, cat: Category, options: string) = - when defined(windows): - let cOptions = "gcc -ldl -DWIN" - else: - let cOptions = "gcc -ldl" - - var c = initResults() - # According to ioTests, this should compile the file - testNoSpec c, makeTest("tests/realtimeGC/shared", options, cat, actionCompile) - testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat, actionRun) - testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat, actionRun) - -# ------------------------- threading tests ----------------------------------- - -proc threadTests(r: var TResults, cat: Category, options: string) = - template test(filename: expr): stmt = - testSpec r, makeTest("tests/threads" / filename, options, cat, actionRun) - testSpec r, makeTest("tests/threads" / filename, options & - " -d:release", cat, actionRun) - testSpec r, makeTest("tests/threads" / filename, options & - " --tlsEmulation:on", cat, actionRun) - - test "tactors" - test "tactors2" - test "threadex" - # deactivated because output capturing still causes problems sometimes: - #test "trecursive_actor" - #test "threadring" - #test "tthreadanalysis" - #test "tthreadsort" - test "tthreadanalysis2" - #test "tthreadanalysis3" - test "tthreadheapviolation1" - -# ------------------------- IO tests ------------------------------------------ - -proc ioTests(r: var TResults, cat: Category, options: string) = - # We need readall_echo to be compiled for this test to run. - # dummy compile result: - var c = initResults() - testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat) - testSpec r, makeTest("tests/system/io", options, cat) - -# ------------------------- debugger tests ------------------------------------ - -proc debuggerTests(r: var TResults, cat: Category, options: string) = - testNoSpec r, makeTest("tools/nimgrep", options & " --debugger:on", cat) - -# ------------------------- JS tests ------------------------------------------ - -proc jsTests(r: var TResults, cat: Category, options: string) = - template test(filename: expr): stmt = - testSpec r, makeTest(filename, options & " -d:nodejs", cat, - actionRun, targetJS) - testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat, - actionRun, targetJS) - - for t in os.walkFiles("tests/js/t*.nim"): - test(t) - for testfile in ["exception/texceptions", "exception/texcpt1", - "exception/texcsub", "exception/tfinally", - "exception/tfinally2", "exception/tfinally3", - "actiontable/tactiontable", "method/tmultim1", - "method/tmultim3", "method/tmultim4", - "varres/tvarres0", "varres/tvarres3", "varres/tvarres4", - "varres/tvartup", "misc/tints", "misc/tunsignedinc"]: - test "tests/" & testfile & ".nim" - - for testfile in ["pure/strutils", "pure/json", "pure/random", "pure/times"]: - test "lib/" & testfile & ".nim" - -# ------------------------- manyloc ------------------------------------------- -#proc runSpecialTests(r: var TResults, options: string) = -# for t in ["lib/packages/docutils/highlite"]: -# testSpec(r, t, options) - -proc findMainFile(dir: string): string = - # finds the file belonging to ".nim.cfg"; if there is no such file - # it returns the some ".nim" file if there is only one: - const cfgExt = ".nim.cfg" - result = "" - var nimFiles = 0 - for kind, file in os.walkDir(dir): - if kind == pcFile: - if file.endsWith(cfgExt): return file[.. ^(cfgExt.len+1)] & ".nim" - elif file.endsWith(".nim"): - if result.len == 0: result = file - inc nimFiles - if nimFiles != 1: result.setlen(0) - -proc manyLoc(r: var TResults, cat: Category, options: string) = - for kind, dir in os.walkDir("tests/manyloc"): - if kind == pcDir: - let mainfile = findMainFile(dir) - if mainfile != "": - testNoSpec r, makeTest(mainfile, options, cat) - -proc compileExample(r: var TResults, pattern, options: string, cat: Category) = - for test in os.walkFiles(pattern): - testNoSpec r, makeTest(test, options, cat) - -proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = - for test in os.walkFiles(pattern): - let contents = readFile(test).string - if contents.contains("when isMainModule"): - testSpec r, makeTest(test, options, cat, actionRunNoSpec) - else: - testNoSpec r, makeTest(test, options, cat, actionCompile) - -# ----------------------------- nimble ---------------------------------------- -type PackageFilter = enum - pfCoreOnly - pfExtraOnly - pfAll - -let - nimbleExe = findExe("nimble") - nimbleDir = getHomeDir() / ".nimble" - packageDir = nimbleDir / "pkgs" - packageIndex = nimbleDir / "packages.json" - -proc waitForExitEx(p: Process): int = - var outp = outputStream(p) - var line = newStringOfCap(120).TaintedString - while true: - if outp.readLine(line): - discard - else: - result = peekExitCode(p) - if result != -1: break - close(p) - -proc getPackageDir(package: string): string = - ## TODO - Replace this with dom's version comparison magic. - var commandOutput = execCmdEx("nimble path $#" % package) - if commandOutput.exitCode != QuitSuccess: - return "" - else: - result = commandOutput[0].string - -iterator listPackages(filter: PackageFilter): tuple[name, url: string] = - let packageList = parseFile(packageIndex) - - for package in packageList.items(): - let - name = package["name"].str - url = package["url"].str - isCorePackage = "nim-lang" in normalize(url) - case filter: - of pfCoreOnly: - if isCorePackage: - yield (name, url) - of pfExtraOnly: - if not isCorePackage: - yield (name, url) - of pfAll: - yield (name, url) - -proc testNimblePackages(r: var TResults, cat: Category, filter: PackageFilter) = - if nimbleExe == "": - echo("[Warning] - Cannot run nimble tests: Nimble binary not found.") - return - - if execCmd("$# update" % nimbleExe) == QuitFailure: - echo("[Warning] - Cannot run nimble tests: Nimble update failed.") - return - - let packageFileTest = makeTest("PackageFileParsed", "", cat) - try: - for name, url in listPackages(filter): - var test = makeTest(name, "", cat) - echo(url) - let - installProcess = startProcess(nimbleExe, "", ["install", "-y", name]) - installStatus = waitForExitEx(installProcess) - installProcess.close - if installStatus != QuitSuccess: - r.addResult(test, "", "", reInstallFailed) - continue - - let - buildPath = getPackageDir(name).strip - buildProcess = startProcess(nimbleExe, buildPath, ["build"]) - buildStatus = waitForExitEx(buildProcess) - buildProcess.close - if buildStatus != QuitSuccess: - r.addResult(test, "", "", reBuildFailed) - r.addResult(test, "", "", reSuccess) - r.addResult(packageFileTest, "", "", reSuccess) - except JsonParsingError: - echo("[Warning] - Cannot run nimble tests: Invalid package file.") - r.addResult(packageFileTest, "", "", reBuildFailed) - - -# ---------------------------------------------------------------------------- - -const AdditionalCategories = ["debugger", "examples", "lib"] - -proc `&.?`(a, b: string): string = - # candidate for the stdlib? - result = if b.startswith(a): b else: a & b - -proc `&?.`(a, b: string): string = - # candidate for the stdlib? - result = if a.endswith(b): a else: a & b - -proc processCategory(r: var TResults, cat: Category, options: string, fileGlob: string = "t*.nim") = - case cat.string.normalize - of "rodfiles": - discard # Disabled for now - #compileRodFiles(r, cat, options) - #runRodFiles(r, cat, options) - of "js": - # XXX JS doesn't need to be special anymore - jsTests(r, cat, options) - of "dll": - dllTests(r, cat, options) - of "gc": - gcTests(r, cat, options) - of "longgc": - longGCTests(r, cat, options) - of "debugger": - debuggerTests(r, cat, options) - of "manyloc": - manyLoc r, cat, options - of "threads": - threadTests r, cat, options & " --threads:on" - of "io": - ioTests r, cat, options - of "lib": - testStdlib(r, "lib/pure/*.nim", options, cat) - testStdlib(r, "lib/packages/docutils/highlite", options, cat) - of "examples": - compileExample(r, "examples/*.nim", options, cat) - compileExample(r, "examples/gtk/*.nim", options, cat) - compileExample(r, "examples/talk/*.nim", options, cat) - of "nimble-core": - testNimblePackages(r, cat, pfCoreOnly) - of "nimble-extra": - testNimblePackages(r, cat, pfExtraOnly) - of "nimble-all": - testNimblePackages(r, cat, pfAll) - of "untestable": - # We can't test it because it depends on a third party. - discard # TODO: Move untestable tests to someplace else, i.e. nimble repo. - else: - for name in os.walkFiles("tests" & DirSep &.? cat.string / fileGlob): - testSpec r, makeTest(name, options, cat) diff --git a/tests/testament/css/boilerplate.css b/tests/testament/css/boilerplate.css deleted file mode 100644 index b209b5aa1..000000000 --- a/tests/testament/css/boilerplate.css +++ /dev/null @@ -1,138 +0,0 @@ -/* ==== Scroll down to find where to put your styles :) ==== */ - -/* HTML5 ✰ Boilerplate */ - -html, body, div, span, object, iframe, -h1, h2, h3, h4, h5, h6, p, blockquote, pre, -abbr, address, cite, code, del, dfn, em, img, ins, kbd, q, samp, -small, strong, sub, sup, var, b, i, dl, dt, dd, ol, ul, li, -fieldset, form, label, legend, -table, caption, tbody, tfoot, thead, tr, th, td, -article, aside, canvas, details, figcaption, figure, -footer, header, hgroup, menu, nav, section, summary, -time, mark, audio, video { - margin: 0; - padding: 0; - border: 0; - font-size: 100%; - font: inherit; - vertical-align: baseline; -} - -article, aside, details, figcaption, figure, -footer, header, hgroup, menu, nav, section { - display: block; -} - -blockquote, q { quotes: none; } -blockquote:before, blockquote:after, -q:before, q:after { content: ''; content: none; } -ins { background-color: #ff9; color: #000; text-decoration: none; } -mark { background-color: #ff9; color: #000; font-style: italic; font-weight: bold; } -del { text-decoration: line-through; } -abbr[title], dfn[title] { border-bottom: 1px dotted; cursor: help; } -table { border-collapse: collapse; border-spacing: 0; } -hr { display: block; height: 1px; border: 0; border-top: 1px solid #ccc; margin: 1em 0; padding: 0; } -input, select { vertical-align: middle; } - -body { font:13px/1.231 sans-serif; *font-size:small; } -select, input, textarea, button { font:99% sans-serif; } -pre, code, kbd, samp { font-family: monospace, sans-serif; } - -html { overflow-y: scroll; } -a:hover, a:active { outline: none; } -ul, ol { margin-left: 2em; } -ol { list-style-type: decimal; } -nav ul, nav li { margin: 0; list-style:none; list-style-image: none; } -small { font-size: 85%; } -strong, th { font-weight: bold; } -td { vertical-align: top; } - -sub, sup { font-size: 75%; line-height: 0; position: relative; } -sup { top: -0.5em; } -sub { bottom: -0.25em; } - -pre { white-space: pre; white-space: pre-wrap; word-wrap: break-word; padding: 15px; } -textarea { overflow: auto; } -.ie6 legend, .ie7 legend { margin-left: -7px; } -input[type="radio"] { vertical-align: text-bottom; } -input[type="checkbox"] { vertical-align: bottom; } -.ie7 input[type="checkbox"] { vertical-align: baseline; } -.ie6 input { vertical-align: text-bottom; } -label, input[type="button"], input[type="submit"], input[type="image"], button { cursor: pointer; } -button, input, select, textarea { margin: 0; } -input:valid, textarea:valid { } -input:invalid, textarea:invalid { border-radius: 1px; -moz-box-shadow: 0px 0px 5px red; -webkit-box-shadow: 0px 0px 5px red; box-shadow: 0px 0px 5px red; } -.no-boxshadow input:invalid, .no-boxshadow textarea:invalid { background-color: #f0dddd; } - -a:link { -webkit-tap-highlight-color: #FF5E99; } - -button { width: auto; overflow: visible; } -.ie7 img { -ms-interpolation-mode: bicubic; } - -body, select, input, textarea { color: #444; } -h1, h2, h3, h4, h5, h6 { font-weight: bold; } -a, a:active, a:visited { color: #607890; } -a:hover { color: #036; } - -/* - // ========================================== \\ - || || - || Your styles ! || - || || - \\ ========================================== // -*/ - - - - - - - - - - - - - - - -.ir { display: block; text-indent: -999em; overflow: hidden; background-repeat: no-repeat; text-align: left; direction: ltr; } -.hidden { display: none; visibility: hidden; } -.visuallyhidden { border: 0; clip: rect(0 0 0 0); height: 1px; margin: -1px; overflow: hidden; padding: 0; position: absolute; width: 1px; } -.visuallyhidden.focusable:active, -.visuallyhidden.focusable:focus { clip: auto; height: auto; margin: 0; overflow: visible; position: static; width: auto; } -.invisible { visibility: hidden; } -.clearfix:before, .clearfix:after { content: "\0020"; display: block; height: 0; overflow: hidden; } -.clearfix:after { clear: both; } -.clearfix { zoom: 1; } - - -@media all and (orientation:portrait) { - -} - -@media all and (orientation:landscape) { - -} - -@media screen and (max-device-width: 480px) { - - /* html { -webkit-text-size-adjust:none; -ms-text-size-adjust:none; } */ -} - - -@media print { - * { background: transparent !important; color: black !important; text-shadow: none !important; filter:none !important; - -ms-filter: none !important; } - a, a:visited { color: #444 !important; text-decoration: underline; } - a[href]:after { content: " (" attr(href) ")"; } - abbr[title]:after { content: " (" attr(title) ")"; } - .ir a:after, a[href^="javascript:"]:after, a[href^="#"]:after { content: ""; } - pre, blockquote { border: 1px solid #999; page-break-inside: avoid; } - thead { display: table-header-group; } - tr, img { page-break-inside: avoid; } - @page { margin: 0.5cm; } - p, h2, h3 { orphans: 3; widows: 3; } - h2, h3{ page-break-after: avoid; } -} diff --git a/tests/testament/css/style.css b/tests/testament/css/style.css deleted file mode 100644 index 43a8add68..000000000 --- a/tests/testament/css/style.css +++ /dev/null @@ -1,114 +0,0 @@ -body { - font-size: medium; -} - -div#header { - font-size: 2em; - background-color: #3d3d3d; - border-bottom: solid 2px #000000; - padding: 0.25em; - color: #ffffff; -} - -div#content { - margin: 0.5em; -} - -table { - text-align: left; - margin-bottom: 0.5em; -} - -table td, table th { - padding: 0.15em 0.5em; -} - -tr:nth-child(even) { - background-color: #eee; -} - -/* Awesome buttons :P */ - -a.button { - border-radius: 2px 2px 2px 2px; - background: -moz-linear-gradient(top, #f7f7f7, #ebebeb); - background: -webkit-linear-gradient(top, #f7f7f7, #ebebeb); - background: -o-linear-gradient(top, #f7f7f7, #ebebeb); - text-decoration: none; - color: #3d3d3d; - padding: 5px; - border: solid 1px #9d9d9d; - display: inline-block; - position: relative; - text-align: center; - font-size: small; -} - -a.button.active { - background: -moz-linear-gradient(top, #00B40C, #03A90E); - background: -webkit-linear-gradient(top, #00B40C, #03A90E); - background: -o-linear-gradient(top, #00B40C, #03A90E); - border: solid 1px #148420; - color: #ffffff; -} - -a.button.left { - border-top-right-radius: 0; - border-bottom-right-radius: 0; -} - -a.button.middle { - border-radius: 0; - border-left: 0; -} - -a.button.right { - border-top-left-radius: 0; - border-bottom-left-radius: 0; - border-left: 0; -} - -a.button:hover { - background: -moz-linear-gradient(top, #0099c7, #0294C1); - background: -webkit-linear-gradient(top, #0099c7, #0294C1); - background: -o-linear-gradient(top, #0099c7, #0294C1); - border: solid 1px #077A9C; - color: #ffffff; -} - -a.button.middle:hover, a.button.right:hover { - border-left: 0; -} - -a.button span.download { - background-image: url("../images/icons.png"); - background-repeat: no-repeat; - display: inline-block; - margin: auto 3px auto auto; - height: 15px; - width: 14px; - position: relative; - background-position: 0 -30px; - top: 3px; -} - -a.button span.book { - background-image: url("../images/icons.png"); - background-repeat: no-repeat; - display: inline-block; - margin: auto 3px auto auto; - height: 15px; - width: 14px; - position: relative; - background-position: 0 0; - top: 3px; -} - -a.button.active span.download, a.button:hover span.download { - background-position: 0 -45px; -} - -a.button.active span.book, a.button:hover span.book { - background-position: 0 -15px; -} - diff --git a/tests/testament/htmlgen.nim b/tests/testament/htmlgen.nim deleted file mode 100644 index 15960f09a..000000000 --- a/tests/testament/htmlgen.nim +++ /dev/null @@ -1,228 +0,0 @@ -# -# -# Nim Tester -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## HTML generator for the tester. - -import db_sqlite, cgi, backend, strutils, json - -const - TableHeader = """<table border="1"> - <tr><td>Test</td><td>Category</td><td>Target</td> - <td>Action</td> - <td>Expected</td> - <td>Given</td> - <td>Success</td></tr>""" - TableFooter = "</table>" - HtmlBegin = """<html> - <head> - <title>Test results</title> - <style type="text/css"> - <!--""" & slurp("css/boilerplate.css") & "\n" & - slurp("css/style.css") & - """ -ul#tabs { list-style-type: none; margin: 30px 0 0 0; padding: 0 0 0.3em 0; } -ul#tabs li { display: inline; } -ul#tabs li a { color: #42454a; background-color: #dedbde; - border: 1px solid #c9c3ba; border-bottom: none; - padding: 0.3em; text-decoration: none; } -ul#tabs li a:hover { background-color: #f1f0ee; } -ul#tabs li a.selected { color: #000; background-color: #f1f0ee; - font-weight: bold; padding: 0.7em 0.3em 0.38em 0.3em; } -div.tabContent { border: 1px solid #c9c3ba; - padding: 0.5em; background-color: #f1f0ee; } -div.tabContent.hide { display: none; } - --> - </style> - <script> - - var tabLinks = new Array(); - var contentDivs = new Array(); - - function init() { - // Grab the tab links and content divs from the page - var tabListItems = document.getElementById('tabs').childNodes; - for (var i = 0; i < tabListItems.length; i++) { - if (tabListItems[i].nodeName == "LI") { - var tabLink = getFirstChildWithTagName(tabListItems[i], 'A'); - var id = getHash(tabLink.getAttribute('href')); - tabLinks[id] = tabLink; - contentDivs[id] = document.getElementById(id); - } - } - // Assign onclick events to the tab links, and - // highlight the first tab - var i = 0; - for (var id in tabLinks) { - tabLinks[id].onclick = showTab; - tabLinks[id].onfocus = function() { this.blur() }; - if (i == 0) tabLinks[id].className = 'selected'; - i++; - } - // Hide all content divs except the first - var i = 0; - for (var id in contentDivs) { - if (i != 0) contentDivs[id].className = 'tabContent hide'; - i++; - } - } - - function showTab() { - var selectedId = getHash(this.getAttribute('href')); - - // Highlight the selected tab, and dim all others. - // Also show the selected content div, and hide all others. - for (var id in contentDivs) { - if (id == selectedId) { - tabLinks[id].className = 'selected'; - contentDivs[id].className = 'tabContent'; - } else { - tabLinks[id].className = ''; - contentDivs[id].className = 'tabContent hide'; - } - } - // Stop the browser following the link - return false; - } - - function getFirstChildWithTagName(element, tagName) { - for (var i = 0; i < element.childNodes.length; i++) { - if (element.childNodes[i].nodeName == tagName) return element.childNodes[i]; - } - } - function getHash(url) { - var hashPos = url.lastIndexOf('#'); - return url.substring(hashPos + 1); - } - </script> - - </head> - <body onload="init()">""" - - HtmlEnd = "</body></html>" - -proc td(s: string): string = - result = "<td>" & s.substr(0, 200).xmlEncode & "</td>" - -proc getCommit(db: DbConn, c: int): string = - var commit = c - for thisCommit in db.rows(sql"select id from [Commit] order by id desc"): - if commit == 0: result = thisCommit[0] - inc commit - -proc generateHtml*(filename: string, commit: int; onlyFailing: bool) = - const selRow = """select name, category, target, action, - expected, given, result - from TestResult - where [commit] = ? and machine = ? - order by category""" - var db = open(connection="testament.db", user="testament", password="", - database="testament") - # search for proper commit: - let lastCommit = db.getCommit(commit) - - var outfile = open(filename, fmWrite) - outfile.write(HtmlBegin) - - let commit = db.getValue(sql"select hash from [Commit] where id = ?", - lastCommit) - let branch = db.getValue(sql"select branch from [Commit] where id = ?", - lastCommit) - outfile.write("<p><b>$# $#</b></p>" % [branch, commit]) - - # generate navigation: - outfile.write("""<ul id="tabs">""") - for m in db.rows(sql"select id, name, os, cpu from Machine order by id"): - outfile.writeLine """<li><a href="#$#">$#: $#, $#</a></li>""" % m - outfile.write("</ul>") - - for currentMachine in db.rows(sql"select id from Machine order by id"): - let m = currentMachine[0] - outfile.write("""<div class="tabContent" id="$#">""" % m) - - outfile.write(TableHeader) - for row in db.rows(sql(selRow), lastCommit, m): - if onlyFailing and row.len > 0 and row[row.high] == "reSuccess": - discard - else: - outfile.write("<tr>") - for x in row: - outfile.write(x.td) - outfile.write("</tr>") - - outfile.write(TableFooter) - outfile.write("</div>") - outfile.write(HtmlEnd) - close(db) - close(outfile) - -proc generateJson*(filename: string, commit: int) = - const - selRow = """select count(*), - sum(result = 'reSuccess'), - sum(result = 'reIgnored') - from TestResult - where [commit] = ? and machine = ? - order by category""" - selDiff = """select A.category || '/' || A.target || '/' || A.name, - A.result, - B.result - from TestResult A - inner join TestResult B - on A.name = B.name and A.category = B.category - where A.[commit] = ? and B.[commit] = ? and A.machine = ? - and A.result != B.result""" - selResults = """select - category || '/' || target || '/' || name, - category, target, action, result, expected, given - from TestResult - where [commit] = ?""" - var db = open(connection="testament.db", user="testament", password="", - database="testament") - let lastCommit = db.getCommit(commit) - if lastCommit.isNil: - quit "cannot determine commit " & $commit - - let previousCommit = db.getCommit(commit-1) - - var outfile = open(filename, fmWrite) - - let machine = $backend.getMachine(db) - let data = db.getRow(sql(selRow), lastCommit, machine) - - outfile.writeLine("""{"total": $#, "passed": $#, "skipped": $#""" % data) - - let results = newJArray() - for row in db.rows(sql(selResults), lastCommit): - var obj = newJObject() - obj["name"] = %row[0] - obj["category"] = %row[1] - obj["target"] = %row[2] - obj["action"] = %row[3] - obj["result"] = %row[4] - obj["expected"] = %row[5] - obj["given"] = %row[6] - results.add(obj) - outfile.writeLine(""", "results": """) - outfile.write(results.pretty) - - if not previousCommit.isNil: - let diff = newJArray() - - for row in db.rows(sql(selDiff), previousCommit, lastCommit, machine): - var obj = newJObject() - obj["name"] = %row[0] - obj["old"] = %row[1] - obj["new"] = %row[2] - diff.add obj - outfile.writeLine(""", "diff": """) - outfile.writeLine(diff.pretty) - - outfile.writeLine "}" - close(db) - close(outfile) diff --git a/tests/testament/specs.nim b/tests/testament/specs.nim deleted file mode 100644 index b9519c70f..000000000 --- a/tests/testament/specs.nim +++ /dev/null @@ -1,172 +0,0 @@ -# -# -# Nim Tester -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import parseutils, strutils, os, osproc, streams, parsecfg - -const - cmdTemplate* = r"compiler" / "nim $target --lib:lib --hints:on -d:testing $options $file" - -type - TTestAction* = enum - actionCompile = "compile" - actionRun = "run" - actionReject = "reject" - actionRunNoSpec = "runNoSpec" - TResultEnum* = enum - reNimcCrash, # nim compiler seems to have crashed - reMsgsDiffer, # error messages differ - reFilesDiffer, # expected and given filenames differ - reLinesDiffer, # expected and given line numbers differ - reOutputsDiffer, - reExitcodesDiffer, - reInvalidPeg, - reCodegenFailure, - reCodeNotFound, - reExeNotFound, - reInstallFailed # package installation failed - reBuildFailed # package building failed - reIgnored, # test is ignored - reSuccess # test was successful - TTarget* = enum - targetC = "C" - targetCpp = "C++" - targetObjC = "ObjC" - targetJS = "JS" - - TSpec* = object - action*: TTestAction - file*, cmd*: string - outp*: string - line*, column*: int - tfile*: string - tline*, tcolumn*: int - exitCode*: int - msg*: string - ccodeCheck*: string - err*: TResultEnum - substr*, sortoutput*: bool - targets*: set[TTarget] - nimout*: string - -const - targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"] - targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"] - -when not declared(parseCfgBool): - # candidate for the stdlib: - proc parseCfgBool(s: string): bool = - case normalize(s) - of "y", "yes", "true", "1", "on": result = true - of "n", "no", "false", "0", "off": result = false - else: raise newException(ValueError, "cannot interpret as a bool: " & s) - -proc extractSpec(filename: string): string = - const tripleQuote = "\"\"\"" - var x = readFile(filename).string - var a = x.find(tripleQuote) - var b = x.find(tripleQuote, a+3) - # look for """ only in the first section - if a >= 0 and b > a and a < 40: - result = x.substr(a+3, b-1).replace("'''", tripleQuote) - else: - #echo "warning: file does not contain spec: " & filename - result = "" - -when not defined(nimhygiene): - {.pragma: inject.} - -template parseSpecAux(fillResult: untyped) = - var ss = newStringStream(extractSpec(filename)) - var p {.inject.}: CfgParser - open(p, ss, filename, 1) - while true: - var e {.inject.} = next(p) - case e.kind - of cfgEof: break - of cfgSectionStart, cfgOption, cfgError: - echo ignoreMsg(p, e) - of cfgKeyValuePair: - fillResult - close(p) - -proc specDefaults*(result: var TSpec) = - result.msg = "" - result.outp = "" - result.nimout = "" - result.ccodeCheck = "" - result.cmd = cmdTemplate - result.line = 0 - result.column = 0 - result.tfile = "" - result.tline = 0 - result.tcolumn = 0 - -proc parseTargets*(value: string): set[TTarget] = - for v in value.normalize.split: - case v - of "c": result.incl(targetC) - of "cpp", "c++": result.incl(targetCpp) - of "objc": result.incl(targetObjC) - of "js": result.incl(targetJS) - else: echo "target ignored: " & v - -proc parseSpec*(filename: string): TSpec = - specDefaults(result) - result.file = filename - parseSpecAux: - case normalize(e.key) - of "action": - case e.value.normalize - of "compile": result.action = actionCompile - of "run": result.action = actionRun - of "reject": result.action = actionReject - else: echo ignoreMsg(p, e) - of "file": result.file = e.value - of "line": discard parseInt(e.value, result.line) - of "column": discard parseInt(e.value, result.column) - of "tfile": result.tfile = e.value - of "tline": discard parseInt(e.value, result.tline) - of "tcolumn": discard parseInt(e.value, result.tcolumn) - of "output": - result.action = actionRun - result.outp = e.value - of "outputsub": - result.action = actionRun - result.outp = e.value - result.substr = true - of "sortoutput": - result.sortoutput = parseCfgBool(e.value) - of "exitcode": - discard parseInt(e.value, result.exitCode) - of "msg": - result.msg = e.value - if result.action != actionRun: - result.action = actionCompile - of "errormsg": - result.msg = e.value - result.action = actionReject - of "nimout": - result.nimout = e.value - of "disabled": - if parseCfgBool(e.value): result.err = reIgnored - of "cmd": - if e.value.startsWith("nim "): - result.cmd = "compiler" / e.value - else: - result.cmd = e.value - of "ccodecheck": result.ccodeCheck = e.value - of "target", "targets": - for v in e.value.normalize.split: - case v - of "c": result.targets.incl(targetC) - of "cpp", "c++": result.targets.incl(targetCpp) - of "objc": result.targets.incl(targetObjC) - of "js": result.targets.incl(targetJS) - else: echo ignoreMsg(p, e) - else: echo ignoreMsg(p, e) diff --git a/tests/testament/t16576.nim b/tests/testament/t16576.nim new file mode 100644 index 000000000..8d0dd57e3 --- /dev/null +++ b/tests/testament/t16576.nim @@ -0,0 +1,7 @@ +discard """ + matrix:"-d:nimTest_t16576" +""" + +# bug #16576 +doAssert defined(nimTest_t16576) +doAssert not defined(nimMegatest) diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim deleted file mode 100644 index 9a253d0ff..000000000 --- a/tests/testament/tester.nim +++ /dev/null @@ -1,458 +0,0 @@ -# -# -# Nim Tester -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This program verifies Nim against the testcases. - -import - parseutils, strutils, pegs, os, osproc, streams, parsecfg, json, - marshal, backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, compiler/nodejs, re, times - -const - resultsFile = "testresults.html" - jsonFile = "testresults.json" - Usage = """Usage: - tester [options] command [arguments] - -Command: - all run all tests - c|category <category> run all the tests of a certain category - r|run <test> run single test file - html [commit] generate $1 from the database; uses the latest - commit or a specific one (use -1 for the commit - before latest etc) -Arguments: - arguments are passed to the compiler -Options: - --print also print results to the console - --failing only show failing/ignored tests - --pedantic return non-zero status code if there are failures - --targets:"c c++ js objc" run tests for specified targets (default: all) -""" % resultsFile - -type - Category = distinct string - TResults = object - total, passed, skipped: int - data: string - - TTest = object - name: string - cat: Category - options: string - target: TTarget - action: TTestAction - startTime: float - -# ---------------------------------------------------------------------------- - -let - pegLineError = - peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" - pegLineTemplate = - peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' 'template/generic instantiation from here'.*" - pegOtherError = peg"'Error:' \s* {.*}" - pegSuccess = peg"'Hint: operation successful'.*" - pegOfInterest = pegLineError / pegOtherError - -var targets = {low(TTarget)..high(TTarget)} - -proc callCompiler(cmdTemplate, filename, options: string, - target: TTarget): TSpec = - let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], - "options", options, "file", filename.quoteShell]) - var p = startProcess(command=c[0], args=c[1.. ^1], - options={poStdErrToStdOut, poUsePath}) - let outp = p.outputStream - var suc = "" - var err = "" - var tmpl = "" - var x = newStringOfCap(120) - result.nimout = "" - while outp.readLine(x.TaintedString) or running(p): - result.nimout.add(x & "\n") - if x =~ pegOfInterest: - # `err` should contain the last error/warning message - err = x - elif x =~ pegLineTemplate and err == "": - # `tmpl` contains the last template expansion before the error - tmpl = x - elif x =~ pegSuccess: - suc = x - close(p) - result.msg = "" - result.file = "" - result.outp = "" - result.line = 0 - result.column = 0 - result.tfile = "" - result.tline = 0 - result.tcolumn = 0 - if tmpl =~ pegLineTemplate: - result.tfile = extractFilename(matches[0]) - result.tline = parseInt(matches[1]) - result.tcolumn = parseInt(matches[2]) - if err =~ pegLineError: - result.file = extractFilename(matches[0]) - result.line = parseInt(matches[1]) - result.column = parseInt(matches[2]) - result.msg = matches[3] - elif err =~ pegOtherError: - result.msg = matches[0] - elif suc =~ pegSuccess: - result.err = reSuccess - - if result.err == reNimcCrash and - ("Your platform is not supported" in result.msg or - "cannot open 'sdl'" in result.msg or - "cannot open 'opengl'" in result.msg): - result.err = reIgnored - -proc callCCompiler(cmdTemplate, filename, options: string, - target: TTarget): TSpec = - let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], - "options", options, "file", filename.quoteShell]) - var p = startProcess(command="gcc", args=c[5.. ^1], - options={poStdErrToStdOut, poUsePath}) - let outp = p.outputStream - var x = newStringOfCap(120) - result.nimout = "" - result.msg = "" - result.file = "" - result.outp = "" - result.line = -1 - while outp.readLine(x.TaintedString) or running(p): - result.nimout.add(x & "\n") - close(p) - if p.peekExitCode == 0: - result.err = reSuccess - -proc initResults: TResults = - result.total = 0 - result.passed = 0 - result.skipped = 0 - result.data = "" - -proc readResults(filename: string): TResults = - result = marshal.to[TResults](readFile(filename).string) - -proc writeResults(filename: string, r: TResults) = - writeFile(filename, $$r) - -proc `$`(x: TResults): string = - result = ("Tests passed: $1 / $3 <br />\n" & - "Tests skipped: $2 / $3 <br />\n") % - [$x.passed, $x.skipped, $x.total] - -proc addResult(r: var TResults, test: TTest, - expected, given: string, success: TResultEnum) = - let name = test.name.extractFilename & test.options - let duration = epochTime() - test.startTime - backend.writeTestResult(name = name, - category = test.cat.string, - target = $test.target, - action = $test.action, - result = $success, - expected = expected, - given = given) - r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) - if success == reSuccess: - styledEcho fgGreen, "PASS: ", fgCyan, name - elif success == reIgnored: - styledEcho styleDim, fgYellow, "SKIP: ", styleBright, fgCyan, name - else: - styledEcho styleBright, fgRed, "FAIL: ", fgCyan, name - styledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" - styledEcho styleBright, fgRed, "Failure: ", $success - styledEcho fgYellow, "Expected:" - styledEcho styleBright, expected, "\n" - styledEcho fgYellow, "Gotten:" - styledEcho styleBright, given, "\n" - - if existsEnv("APPVEYOR"): - let (outcome, msg) = - if success == reSuccess: - ("Passed", "") - elif success == reIgnored: - ("Skipped", "") - else: - ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given) - var p = startProcess("appveyor", args=["AddTest", test.name.replace("\\", "/") & test.options, "-Framework", "nim-testament", "-FileName", test.cat.string, "-Outcome", outcome, "-ErrorMessage", msg, "-Duration", $(duration*1000).int], options={poStdErrToStdOut, poUsePath, poParentStreams}) - discard waitForExit(p) - close(p) - -proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) = - if strip(expected.msg) notin strip(given.msg): - r.addResult(test, expected.msg, given.msg, reMsgsDiffer) - elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and - "internal error:" notin expected.msg: - r.addResult(test, expected.file, given.file, reFilesDiffer) - elif expected.line != given.line and expected.line != 0 or - expected.column != given.column and expected.column != 0: - r.addResult(test, $expected.line & ':' & $expected.column, - $given.line & ':' & $given.column, - reLinesDiffer) - elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and - "internal error:" notin expected.msg: - r.addResult(test, expected.tfile, given.tfile, reFilesDiffer) - elif expected.tline != given.tline and expected.tline != 0 or - expected.tcolumn != given.tcolumn and expected.tcolumn != 0: - r.addResult(test, $expected.tline & ':' & $expected.tcolumn, - $given.tline & ':' & $given.tcolumn, - reLinesDiffer) - else: - r.addResult(test, expected.msg, given.msg, reSuccess) - inc(r.passed) - -proc generatedFile(path, name: string, target: TTarget): string = - let ext = targetToExt[target] - result = path / "nimcache" / - (if target == targetJS: path.splitPath.tail & "_" else: "compiler_") & - name.changeFileExt(ext) - -proc codegenCheck(test: TTest, check: string, given: var TSpec) = - try: - let (path, name, _) = test.name.splitFile - let genFile = generatedFile(path, name, test.target) - let contents = readFile(genFile).string - if check[0] == '\\': - # little hack to get 'match' support: - if not contents.match(check.peg): - given.err = reCodegenFailure - elif contents.find(check.peg) < 0: - given.err = reCodegenFailure - except ValueError: - given.err = reInvalidPeg - echo getCurrentExceptionMsg() - except IOError: - given.err = reCodeNotFound - -proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) = - let exp = expectedNimout.strip.replace("\C\L", "\L") - let giv = given.nimout.strip.replace("\C\L", "\L") - if exp notin giv: - given.err = reMsgsDiffer - -proc makeDeterministic(s: string): string = - var x = splitLines(s) - sort(x, system.cmp) - result = join(x, "\n") - -proc compilerOutputTests(test: TTest, given: var TSpec, expected: TSpec; - r: var TResults) = - var expectedmsg: string = "" - var givenmsg: string = "" - if given.err == reSuccess: - if expected.ccodeCheck.len > 0: - codegenCheck(test, expected.ccodeCheck, given) - expectedmsg = expected.ccodeCheck - givenmsg = given.msg - if expected.nimout.len > 0: - expectedmsg = expected.nimout - givenmsg = given.nimout.strip - nimoutCheck(test, expectedmsg, given) - else: - givenmsg = given.nimout.strip - if given.err == reSuccess: inc(r.passed) - r.addResult(test, expectedmsg, givenmsg, given.err) - -proc analyzeAndConsolidateOutput(s: string): string = - result = "" - let rows = s.splitLines - for i in 0 ..< rows.len: - if (let pos = find(rows[i], "Traceback (most recent call last)"); pos != -1): - result = substr(rows[i], pos) & "\n" - for i in i+1 ..< rows.len: - result.add rows[i] & "\n" - if not (rows[i] =~ re"^[^(]+\(\d+\)\s+"): - return - elif (let pos = find(rows[i], "SIGSEGV: Illegal storage access."); pos != -1): - result = substr(rows[i], pos) - return - -proc testSpec(r: var TResults, test: TTest) = - # major entry point for a single test - if test.target notin targets: - r.addResult(test, "", "", reIgnored) - inc(r.skipped) - return - - let tname = test.name.addFileExt(".nim") - #echo "TESTING ", tname - inc(r.total) - var expected: TSpec - if test.action != actionRunNoSpec: - expected = parseSpec(tname) - else: - specDefaults expected - expected.action = actionRunNoSpec - - if expected.err == reIgnored: - r.addResult(test, "", "", reIgnored) - inc(r.skipped) - return - - case expected.action - of actionCompile: - var given = callCompiler(expected.cmd, test.name, - test.options & " --stdout --hint[Path]:off --hint[Processing]:off", - test.target) - compilerOutputTests(test, given, expected, r) - of actionRun, actionRunNoSpec: - # In this branch of code "early return" pattern is clearer than deep - # nested conditionals - the empty rows in between to clarify the "danger" - var given = callCompiler(expected.cmd, test.name, test.options, - test.target) - - if given.err != reSuccess: - r.addResult(test, "", given.msg, given.err) - return - - let isJsTarget = test.target == targetJS - var exeFile: string - if isJsTarget: - let (dir, file, _) = splitFile(tname) - exeFile = dir / "nimcache" / file & ".js" # *TODO* hardcoded "nimcache" - else: - exeFile = changeFileExt(tname, ExeExt) - - if not existsFile(exeFile): - r.addResult(test, expected.outp, "executable not found", reExeNotFound) - return - - let nodejs = if isJsTarget: findNodeJs() else: "" - if isJsTarget and nodejs == "": - r.addResult(test, expected.outp, "nodejs binary not in PATH", - reExeNotFound) - return - - let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile - var (buf, exitCode) = execCmdEx(exeCmd, options = {poStdErrToStdOut}) - let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string)) - else: strip(buf.string) - let expectedOut = strip(expected.outp) - - if exitCode != expected.exitCode: - r.addResult(test, "exitcode: " & $expected.exitCode, - "exitcode: " & $exitCode & "\n\nOutput:\n" & - analyzeAndConsolidateOutput(bufB), - reExitCodesDiffer) - return - - if bufB != expectedOut and expected.action != actionRunNoSpec: - if not (expected.substr and expectedOut in bufB): - given.err = reOutputsDiffer - r.addResult(test, expected.outp, bufB, reOutputsDiffer) - return - - compilerOutputTests(test, given, expected, r) - return - - of actionReject: - var given = callCompiler(expected.cmd, test.name, test.options, - test.target) - cmpMsgs(r, expected, given, test) - return - -proc testNoSpec(r: var TResults, test: TTest) = - # does not extract the spec because the file is not supposed to have any - #let tname = test.name.addFileExt(".nim") - inc(r.total) - let given = callCompiler(cmdTemplate, test.name, test.options, test.target) - r.addResult(test, "", given.msg, given.err) - if given.err == reSuccess: inc(r.passed) - -proc testC(r: var TResults, test: TTest) = - # runs C code. Doesn't support any specs, just goes by exit code. - let tname = test.name.addFileExt(".c") - inc(r.total) - styledEcho "Processing ", fgCyan, extractFilename(tname) - var given = callCCompiler(cmdTemplate, test.name & ".c", test.options, test.target) - if given.err != reSuccess: - r.addResult(test, "", given.msg, given.err) - elif test.action == actionRun: - let exeFile = changeFileExt(test.name, ExeExt) - var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath}) - if exitCode != 0: given.err = reExitCodesDiffer - if given.err == reSuccess: inc(r.passed) - -proc makeTest(test, options: string, cat: Category, action = actionCompile, - target = targetC, env: string = ""): TTest = - # start with 'actionCompile', will be overwritten in the spec: - result = TTest(cat: cat, name: test, options: options, - target: target, action: action, startTime: epochTime()) - -include categories - -# proc runCaasTests(r: var TResults) = -# for test, output, status, mode in caasTestsRunner(): -# r.addResult(test, "", output & "-> " & $mode, -# if status: reSuccess else: reOutputsDiffer) - -proc main() = - os.putenv "NIMTEST_NO_COLOR", "1" - os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES" - - backend.open() - var optPrintResults = false - var optFailing = false - var optPedantic = false - - var p = initOptParser() - p.next() - while p.kind == cmdLongoption: - case p.key.string.normalize - of "print", "verbose": optPrintResults = true - of "failing": optFailing = true - of "pedantic": optPedantic = true - of "targets": targets = parseTargets(p.val.string) - else: quit Usage - p.next() - if p.kind != cmdArgument: quit Usage - var action = p.key.string.normalize - p.next() - var r = initResults() - case action - of "all": - let testsDir = "tests" & DirSep - for kind, dir in walkDir(testsDir): - assert testsDir.startsWith(testsDir) - let cat = dir[testsDir.len .. ^1] - if kind == pcDir and cat notin ["testament", "testdata", "nimcache"]: - processCategory(r, Category(cat), p.cmdLineRest.string) - for a in AdditionalCategories: - processCategory(r, Category(a), p.cmdLineRest.string) - of "c", "cat", "category": - var cat = Category(p.key) - p.next - processCategory(r, cat, p.cmdLineRest.string) - of "r", "run": - let (dir, file) = splitPath(p.key.string) - let (_, subdir) = splitPath(dir) - var cat = Category(subdir) - processCategory(r, cat, p.cmdLineRest.string, file) - of "html": - var commit = 0 - discard parseInt(p.cmdLineRest.string, commit) - generateHtml(resultsFile, commit, optFailing) - generateJson(jsonFile, commit) - else: - quit Usage - - if optPrintResults: - if action == "html": openDefaultBrowser(resultsFile) - else: echo r, r.data - backend.close() - if optPedantic: - var failed = r.total - r.passed - r.skipped - if failed > 0 : quit(QuitFailure) - -if paramCount() == 0: - quit Usage -main() diff --git a/tests/testament/tester.nim.cfg b/tests/testament/tester.nim.cfg deleted file mode 100644 index 27fd67075..000000000 --- a/tests/testament/tester.nim.cfg +++ /dev/null @@ -1 +0,0 @@ -path = "$nim" # For compiler/nodejs diff --git a/tests/testament/tinlinemsg.nim b/tests/testament/tinlinemsg.nim new file mode 100644 index 000000000..199c263e9 --- /dev/null +++ b/tests/testament/tinlinemsg.nim @@ -0,0 +1,8 @@ +discard """ + matrix: "--errorMax:0 --styleCheck:error" +""" + +proc generic_proc*[T](a_a: int) = #[tt.Error + ^ 'generic_proc' should be: 'genericProc'; tt.Error + ^ 'a_a' should be: 'aA' ]# + discard diff --git a/tests/testament/tjoinable.nim b/tests/testament/tjoinable.nim new file mode 100644 index 000000000..c651780de --- /dev/null +++ b/tests/testament/tjoinable.nim @@ -0,0 +1,9 @@ +discard """ + output: "ok" +""" + +# checks that this is joinable +# checks that megatest allows duplicate names, see also `tests/misc/tjoinable.nim` +doAssert defined(testing) +doAssert defined(nimMegatest) +echo "ok" # intentional to make sure this doesn't prevent `isJoinableSpec` diff --git a/tests/testament/treject.nim b/tests/testament/treject.nim new file mode 100644 index 000000000..be2db86a9 --- /dev/null +++ b/tests/testament/treject.nim @@ -0,0 +1,6 @@ +discard """ +action: "reject" +""" + +# this line does not compile, so the test should pass, since action="reject" +let x: int = "type mismatch" diff --git a/tests/testament/tshould_not_work.nim b/tests/testament/tshould_not_work.nim new file mode 100644 index 000000000..8c99510a0 --- /dev/null +++ b/tests/testament/tshould_not_work.nim @@ -0,0 +1,53 @@ +discard """ + joinable: false +""" + +const expected = """ +FAIL: tests/shouldfail/tccodecheck.nim +Failure: reCodegenFailure +Expected: +baz +FAIL: tests/shouldfail/tcolumn.nim +Failure: reLinesDiffer +FAIL: tests/shouldfail/terrormsg.nim +Failure: reMsgsDiffer +FAIL: tests/shouldfail/texitcode1.nim +Failure: reExitcodesDiffer +FAIL: tests/shouldfail/tfile.nim +Failure: reFilesDiffer +FAIL: tests/shouldfail/tline.nim +Failure: reLinesDiffer +FAIL: tests/shouldfail/tmaxcodesize.nim +Failure: reCodegenFailure +max allowed size: 1 +FAIL: tests/shouldfail/tnimout.nim +Failure: reMsgsDiffer +FAIL: tests/shouldfail/tnimoutfull.nim +Failure: reMsgsDiffer +FAIL: tests/shouldfail/toutput.nim +Failure: reOutputsDiffer +FAIL: tests/shouldfail/toutputsub.nim +Failure: reOutputsDiffer +FAIL: tests/shouldfail/treject.nim +Failure: reFilesDiffer +FAIL: tests/shouldfail/tsortoutput.nim +Failure: reOutputsDiffer +FAIL: tests/shouldfail/ttimeout.nim +Failure: reTimeout +FAIL: tests/shouldfail/tvalgrind.nim +Failure: reExitcodesDiffer +""" + +import std/[os,strformat,osproc] +import stdtest/testutils + +proc main = + const nim = getCurrentCompilerExe() + let testamentExe = "bin/testament" + let cmd = fmt"{testamentExe} --directory:testament --colors:off --backendLogging:off --nim:{nim} category shouldfail" + let (outp, status) = execCmdEx(cmd) + doAssert status == 1, $status + + let ok = greedyOrderedSubsetLines(expected, outp, allowPrefixMatch = true) + doAssert ok, &"\nexpected:\n{expected}\noutp:\n{outp}" +main() diff --git a/tests/testament/tspecialpaths.nim b/tests/testament/tspecialpaths.nim new file mode 100644 index 000000000..3c97dc88a --- /dev/null +++ b/tests/testament/tspecialpaths.nim @@ -0,0 +1,9 @@ +import stdtest/specialpaths +import std/os +block: # splitTestFile + doAssert splitTestFile("tests/fakedir/tfakename.nim") == ("fakedir", "tests/fakedir/tfakename.nim".unixToNativePath) + doAssert splitTestFile("/pathto/tests/fakedir/tfakename.nim") == ("fakedir", "/pathto/tests/fakedir/tfakename.nim".unixToNativePath) + doAssert splitTestFile(getCurrentDir() / "tests/fakedir/tfakename.nim") == ("fakedir", "tests/fakedir/tfakename.nim".unixToNativePath) + doAssert splitTestFile(getCurrentDir() / "sub/tests/fakedir/tfakename.nim") == ("fakedir", "sub/tests/fakedir/tfakename.nim".unixToNativePath) + doAssertRaises(AssertionDefect): discard splitTestFile("testsbad/fakedir/tfakename.nim") + doAssertRaises(AssertionDefect): discard splitTestFile("tests/tfakename.nim") |