diff options
Diffstat (limited to 'testament')
30 files changed, 3514 insertions, 0 deletions
diff --git a/testament/azure.nim b/testament/azure.nim new file mode 100644 index 000000000..af65d6a1c --- /dev/null +++ b/testament/azure.nim @@ -0,0 +1,147 @@ +# +# +# The Nim Tester +# (c) Copyright 2019 Leorize +# +# Look at license.txt for more info. +# All rights reserved. + +import base64, json, httpclient, os, strutils, uri +import specs + +const + RunIdEnv = "TESTAMENT_AZURE_RUN_ID" + CacheSize = 8 # How many results should be cached before uploading to + # Azure Pipelines. This prevents throttling that might arise. + +proc getAzureEnv(env: string): string = + # Conversion rule at: + # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#set-variables-in-pipeline + env.toUpperAscii().replace('.', '_').getEnv + +template getRun(): string = + ## Get the test run attached to this instance + getEnv(RunIdEnv) + +template setRun(id: string) = + ## Attach a test run to this instance and its future children + putEnv(RunIdEnv, id) + +template delRun() = + ## Unattach the test run associtated with this instance and its future children + delEnv(RunIdEnv) + +template warning(args: varargs[untyped]) = + ## Add a warning to the current task + stderr.writeLine "##vso[task.logissue type=warning;]", args + +let + ownRun = not existsEnv RunIdEnv + ## Whether the test run is owned by this instance + accessToken = getAzureEnv("System.AccessToken") + ## Access token to Azure Pipelines + +var + active = false ## Whether the backend should be activated + requestBase: Uri ## Base URI for all API requests + requestHeaders: HttpHeaders ## Headers required for all API requests + results: JsonNode ## A cache for test results before uploading + +proc request(api: string, httpMethod: HttpMethod, body = ""): Response {.inline.} = + let client = newHttpClient(timeout = 3000) + defer: close client + result = client.request($(requestBase / api), httpMethod, body, requestHeaders) + if result.code != Http200: + raise newException(CatchableError, "Request failed") + +proc init*() = + ## Initialize the Azure Pipelines backend. + ## + ## If an access token is provided and no test run is associated with the + ## current instance, this proc will create a test run named after the current + ## Azure Pipelines' job name, then associate it to the current testament + ## instance and its future children. Should this fail, the backend will be + ## disabled. + if isAzure and accessToken.len > 0: + active = true + requestBase = parseUri(getAzureEnv("System.TeamFoundationCollectionUri")) / + getAzureEnv("System.TeamProjectId") / "_apis" ? {"api-version": "5.0"} + requestHeaders = newHttpHeaders { + "Accept": "application/json", + "Authorization": "Basic " & encode(':' & accessToken), + "Content-Type": "application/json" + } + results = newJArray() + if ownRun: + try: + let resp = request( + "test/runs", + HttpPost, + $ %* { + "automated": true, + "build": { "id": getAzureEnv("Build.BuildId") }, + "buildPlatform": hostCPU, + "controller": "nim-testament", + "name": getAzureEnv("Agent.JobName") + } + ) + setRun $resp.body.parseJson["id"].getInt + except: + warning "Couldn't create test run for Azure Pipelines integration" + # Set run id to empty to prevent child processes from trying to request + # for yet another test run id, which wouldn't be shared with other + # instances. + setRun "" + active = false + elif getRun().len == 0: + # Disable integration if there aren't any valid test run id + active = false + +proc uploadAndClear() = + ## Upload test results from cache to Azure Pipelines. Then clear the cache + ## after. + if results.len > 0: + try: + discard request("test/runs/" & getRun() & "/results", HttpPost, $results) + except: + for i in results: + warning "Couldn't log test result to Azure Pipelines: ", + i["automatedTestName"], ", outcome: ", i["outcome"] + results = newJArray() + +proc finalize*() {.noconv.} = + ## Finalize the Azure Pipelines backend. + ## + ## If a test run has been associated and is owned by this instance, it will + ## be marked as complete. + if active: + if ownRun: + uploadAndClear() + try: + discard request("test/runs/" & getRun(), HttpPatch, + $ %* {"state": "Completed"}) + except: + warning "Couldn't update test run ", getRun(), " on Azure Pipelines" + delRun() + +proc addTestResult*(name, category: string; durationInMs: int; errorMsg: string; + outcome: TResultEnum) = + if not active: + return + + let outcome = case outcome + of reSuccess: "Passed" + of reDisabled, reJoined: "NotExecuted" + else: "Failed" + + results.add(%* { + "automatedTestName": name, + "automatedTestStorage": category, + "durationInMs": durationInMs, + "errorMessage": errorMsg, + "outcome": outcome, + "testCaseTitle": name + }) + + if results.len > CacheSize: + uploadAndClear() diff --git a/testament/backend.nim b/testament/backend.nim new file mode 100644 index 000000000..1770c6657 --- /dev/null +++ b/testament/backend.nim @@ -0,0 +1,70 @@ +# +# +# The Nim Tester +# (c) Copyright 2017 Andreas Rumpf +# +# Look at license.txt for more info. +# All rights reserved. + +import strutils, os, osproc, json + +type + MachineId* = distinct string + CommitId = distinct string + +proc `$`*(id: MachineId): string {.borrow.} + +var + thisMachine: MachineId + thisCommit: CommitId + thisBranch: string + +proc getMachine*(): MachineId = + var name = execProcess("hostname").strip + if name.len == 0: + name = when defined(posix): getEnv("HOSTNAME") + else: getEnv("COMPUTERNAME") + if name.len == 0: + quit "cannot determine the machine name" + + result = MachineId(name) + +proc getCommit(): CommitId = + const commLen = "commit ".len + let hash = execProcess("git log -n 1").strip[commLen..commLen+10] + thisBranch = execProcess("git symbolic-ref --short HEAD").strip + if hash.len == 0 or thisBranch.len == 0: quit "cannot determine git HEAD" + result = CommitId(hash) + +var + results: File + currentCategory: string + entries: int + +proc writeTestResult*(name, category, target, action, result, expected, given: string) = + createDir("testresults") + if currentCategory != category: + if currentCategory.len > 0: + results.writeLine("]") + close(results) + currentCategory = category + results = open("testresults" / category.addFileExt"json", fmWrite) + results.writeLine("[") + entries = 0 + + let jentry = %*{"name": name, "category": category, "target": target, + "action": action, "result": result, "expected": expected, "given": given, + "machine": thisMachine.string, "commit": thisCommit.string, "branch": thisBranch} + if entries > 0: + results.writeLine(",") + results.write($jentry) + inc entries + +proc open*() = + thisMachine = getMachine() + thisCommit = getCommit() + +proc close*() = + if currentCategory.len > 0: + results.writeLine("]") + close(results) diff --git a/testament/caasdriver.nim b/testament/caasdriver.nim new file mode 100644 index 000000000..01e402e07 --- /dev/null +++ b/testament/caasdriver.nim @@ -0,0 +1,195 @@ +import osproc, streams, os, strutils, re +{.experimental.} + +## Compiler as a service tester. +## +## Please read docs/idetools.txt for information about this. + + +type + TRunMode = enum + ProcRun, CaasRun, SymbolProcRun + + NimSession* = object + nim: Process # Holds the open process for CaasRun sessions, nil otherwise. + mode: TRunMode # Stores the type of run mode the session was started with. + lastOutput: string # Preserves the last output, needed for ProcRun mode. + filename: string # Appended to each command starting with '>'. Also a var. + modname: string # Like filename but without extension. + nimcache: string # Input script based name for the nimcache dir. + +const + modes = [CaasRun, ProcRun, SymbolProcRun] + filenameReplaceVar = "$TESTNIM" + moduleReplaceVar = "$MODULE" + silentReplaceVar = "$SILENT" + silentReplaceText = "--verbosity:0 --hints:off" + +var + TesterDir = getAppDir() / ".." + NimBin = TesterDir / "../bin/nim" + +proc replaceVars(session: var NimSession, text: string): string = + result = text.replace(filenameReplaceVar, session.filename) + result = result.replace(moduleReplaceVar, session.modname) + result = result.replace(silentReplaceVar, silentReplaceText) + +proc startNimSession(project, script: string, mode: TRunMode): + NimSession = + let (dir, name, ext) = project.splitFile + result.mode = mode + result.lastOutput = "" + result.filename = name & ext + result.modname = name + + let (nimcacheDir, nimcacheName, nimcacheExt) = script.splitFile + result.nimcache = "SymbolProcRun." & nimcacheName + + if mode == SymbolProcRun: + removeDir(nimcacheDir / result.nimcache) + else: + removeDir(nimcacheDir / "nimcache") + + if mode == CaasRun: + result.nim = startProcess(NimBin, workingDir = dir, + args = ["serve", "--server.type:stdin", name]) + +proc doCaasCommand(session: var NimSession, command: string): string = + assert session.mode == CaasRun + session.nim.inputStream.write(session.replaceVars(command) & "\n") + session.nim.inputStream.flush + + result = "" + + while true: + var line = "" + if session.nim.outputStream.readLine(line): + if line.string == "": break + result.add(line.string & "\n") + else: + result = "FAILED TO EXECUTE: " & command & "\n" & result + break + +proc doProcCommand(session: var NimSession, command: string): string = + try: + assert session.mode == ProcRun or session.mode == SymbolProcRun + except: + result = "FAILED TO EXECUTE: " & command & "\n" & result + var + process = startProcess(NimBin, args = session.replaceVars(command).split) + stream = outputStream(process) + line = "" + + result = "" + while stream.readLine(line): + if result.len > 0: result &= "\n" + result &= line.string + + process.close() + +proc doCommand(session: var NimSession, command: string) = + if session.mode == CaasRun: + if not session.nim.running: + session.lastOutput = "FAILED TO EXECUTE: " & command & "\n" & + "Exit code " & $session.nim.peekExitCode + return + session.lastOutput = doCaasCommand(session, + command & " " & session.filename) + else: + var command = command + # For symbol runs we prepend the necessary parameters to avoid clobbering + # the normal nimcache. + if session.mode == SymbolProcRun: + command = "--symbolFiles:on --nimcache:" & session.nimcache & + " " & command + session.lastOutput = doProcCommand(session, + command & " " & session.filename) + +proc destroy(session: var NimSession) {.destructor.} = + if session.mode == CaasRun: + session.nim.close + +proc doScenario(script: string, output: Stream, mode: TRunMode, verbose: bool): bool = + result = true + + var f = open(script) + var project = "" + + if f.readLine(project): + var + s = startNimSession(script.parentDir / project.string, script, mode) + tline = "" + ln = 1 + + while f.readLine(tline): + var line = tline.string + inc ln + + # Filter lines by run mode, removing the prefix if the mode is current. + for testMode in modes: + if line.startsWith($testMode): + if testMode != mode: + line = "" + else: + line = line[len($testMode)..len(line) - 1].strip + break + + if line.strip.len == 0: continue + + if line.startsWith("#"): + output.writeLine line + continue + elif line.startsWith(">"): + s.doCommand(line.substr(1).strip) + output.writeLine line, "\n", if verbose: s.lastOutput else: "" + else: + var expectMatch = true + var pattern = s.replaceVars(line) + if line.startsWith("!"): + pattern = pattern.substr(1).strip + expectMatch = false + + let actualMatch = + s.lastOutput.find(re(pattern, flags = {reStudy})) != -1 + + if expectMatch == actualMatch: + output.writeLine "SUCCESS ", line + else: + output.writeLine "FAILURE ", line + result = false + +iterator caasTestsRunner*(filter = "", verbose = false): tuple[test, + output: string, status: bool, + mode: TRunMode] = + for scenario in os.walkFiles(TesterDir / "caas/*.txt"): + if filter.len > 0 and find(scenario, filter) == -1: continue + for mode in modes: + var outStream = newStringStream() + let r = doScenario(scenario, outStream, mode, verbose) + yield (scenario, outStream.data, r, mode) + +when isMainModule: + var + filter = "" + failures = 0 + verbose = false + + for i in 0..paramCount() - 1: + let param = paramStr(i + 1) + case param + of "verbose": verbose = true + else: filter = param + + if verbose and len(filter) > 0: + echo "Running only test cases matching filter '$1'" % [filter] + + for test, output, result, mode in caasTestsRunner(filter, verbose): + if not result or verbose: + echo "Mode ", $mode, " (", if result: "succeeded)" else: "failed)" + echo test + echo output + echo "---------\n" + if not result: + failures += 1 + + quit(failures) diff --git a/testament/categories.nim b/testament/categories.nim new file mode 100644 index 000000000..843bef3f9 --- /dev/null +++ b/testament/categories.nim @@ -0,0 +1,774 @@ +# +# +# Nim Tester +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Include for the tester that contains test suites that test special features +## of the compiler. + +# included from testament.nim + +import important_packages +import std/[strformat, strutils] +from std/sequtils import filterIt + +const + specialCategories = [ + "assert", + "async", + "debugger", + "dll", + "examples", + "gc", + "io", + "js", + "ic", + "lib", + "manyloc", + "nimble-packages", + "niminaction", + "threads", + "untestable", # see trunner_special + "testdata", + "nimcache", + "coroutines", + "osproc", + "shouldfail", + "destructor" + ] + +proc isTestFile*(file: string): bool = + let (_, name, ext) = splitFile(file) + result = ext == ".nim" and name.startsWith("t") + +# --------------------- DLL generation tests ---------------------------------- + +proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = false) = + const rpath = when defined(macosx): + " --passL:-rpath --passL:@loader_path" + else: + "" + + var test1 = makeTest("lib/nimrtl.nim", options & " --outdir:tests/dll", cat) + test1.spec.action = actionCompile + testSpec c, test1 + var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) + test2.spec.action = actionCompile + testSpec c, test2 + + var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) + test3.spec.action = actionCompile + testSpec c, test3 + var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat) + test4.spec.action = actionCompile + testSpec c, test4 + + # windows looks in the dir of the exe (yay!): + when not defined(windows): + # posix relies on crappy LD_LIBRARY_PATH (ugh!): + const libpathenv = when defined(haiku): "LIBRARY_PATH" + else: "LD_LIBRARY_PATH" + var libpath = getEnv(libpathenv) + # Temporarily add the lib directory to LD_LIBRARY_PATH: + putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: "")) + defer: putEnv(libpathenv, libpath) + + testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) + testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) + testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) + + if "boehm" notin options: + # hcr tests + + var basicHcrTest = makeTest("tests/dll/nimhcr_basic.nim", options & " --threads:off --forceBuild --hotCodeReloading:on " & rpath, cat) + # test segfaults for now but compiles: + if isOrc: basicHcrTest.spec.action = actionCompile + testSpec r, basicHcrTest + + # force build required - see the comments in the .nim file for more details + var hcri = makeTest("tests/dll/nimhcr_integration.nim", + options & " --threads:off --forceBuild --hotCodeReloading:on" & rpath, cat) + let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget()) + let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name, + hcri.options, nimcache, getTestSpecTarget()) + hcri.testArgs = cmd.parseCmdLine + testSpec r, hcri + +proc dllTests(r: var TResults, cat: Category, options: string) = + # dummy compile result: + var c = initResults() + + runBasicDLLTest c, r, cat, options & " --mm:refc" + runBasicDLLTest c, r, cat, options & " -d:release --mm:refc" + runBasicDLLTest c, r, cat, options, isOrc = true + runBasicDLLTest c, r, cat, options & " -d:release", isOrc = true + when not defined(windows): + # still cannot find a recent Windows version of boehm.dll: + runBasicDLLTest c, r, cat, options & " --gc:boehm" + runBasicDLLTest c, r, cat, options & " -d:release --gc:boehm" + +# ------------------------------ GC tests ------------------------------------- + +proc gcTests(r: var TResults, cat: Category, options: string) = + template testWithoutMs(filename: untyped) = + testSpec r, makeTest("tests/gc" / filename, options & "--mm:refc", cat) + testSpec r, makeTest("tests/gc" / filename, options & + " -d:release -d:useRealtimeGC --mm:refc", cat) + when filename != "gctest": + testSpec r, makeTest("tests/gc" / filename, options & + " --gc:orc", cat) + testSpec r, makeTest("tests/gc" / filename, options & + " --gc:orc -d:release", cat) + + template testWithoutBoehm(filename: untyped) = + testWithoutMs filename + testSpec r, makeTest("tests/gc" / filename, options & + " --gc:markAndSweep", cat) + testSpec r, makeTest("tests/gc" / filename, options & + " -d:release --gc:markAndSweep", cat) + + template test(filename: untyped) = + testWithoutBoehm filename + when not defined(windows) and not defined(android): + # AR: cannot find any boehm.dll on the net, right now, so disabled + # for windows: + testSpec r, makeTest("tests/gc" / filename, options & + " --gc:boehm", cat) + testSpec r, makeTest("tests/gc" / filename, options & + " -d:release --gc:boehm", cat) + + testWithoutBoehm "foreign_thr" + test "gcemscripten" + test "growobjcrash" + test "gcbench" + test "gcleak" + test "gcleak2" + testWithoutBoehm "gctest" + test "gcleak3" + test "gcleak4" + # Disabled because it works and takes too long to run: + #test "gcleak5" + testWithoutBoehm "weakrefs" + test "cycleleak" + testWithoutBoehm "closureleak" + testWithoutMs "refarrayleak" + + testWithoutBoehm "tlists" + testWithoutBoehm "thavlak" + + test "stackrefleak" + test "cyclecollector" + testWithoutBoehm "trace_globals" + +# ------------------------- threading tests ----------------------------------- + +proc threadTests(r: var TResults, cat: Category, options: string) = + template test(filename: untyped) = + testSpec r, makeTest(filename, options, cat) + testSpec r, makeTest(filename, options & " -d:release", cat) + testSpec r, makeTest(filename, options & " --tlsEmulation:on", cat) + for t in os.walkFiles("tests/threads/t*.nim"): + test(t) + +# ------------------------- IO tests ------------------------------------------ + +proc ioTests(r: var TResults, cat: Category, options: string) = + # We need readall_echo to be compiled for this test to run. + # dummy compile result: + var c = initResults() + testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat) + # ^- why is this not appended to r? Should this be discarded? + # EDIT: this should be replaced by something like in D20210524T180826, + # likewise in similar instances where `testSpec c` is used, or more generally + # when a test depends on another test, as it makes tests non-independent, + # creating complications for batching and megatest logic. + testSpec r, makeTest("tests/system/tio", options, cat) + +# ------------------------- async tests --------------------------------------- +proc asyncTests(r: var TResults, cat: Category, options: string) = + template test(filename: untyped) = + testSpec r, makeTest(filename, options, cat) + for t in os.walkFiles("tests/async/t*.nim"): + test(t) + +# ------------------------- debugger tests ------------------------------------ + +proc debuggerTests(r: var TResults, cat: Category, options: string) = + if fileExists("tools/nimgrep.nim"): + var t = makeTest("tools/nimgrep", options & " --debugger:on", cat) + t.spec.action = actionCompile + # force target to C because of MacOS 10.15 SDK headers bug + # https://github.com/nim-lang/Nim/pull/15612#issuecomment-712471879 + t.spec.targets = {targetC} + testSpec r, t + +# ------------------------- JS tests ------------------------------------------ + +proc jsTests(r: var TResults, cat: Category, options: string) = + template test(filename: untyped) = + testSpec r, makeTest(filename, options, cat), {targetJS} + testSpec r, makeTest(filename, options & " -d:release", cat), {targetJS} + + for t in os.walkFiles("tests/js/t*.nim"): + test(t) + for testfile in ["exception/texceptions", "exception/texcpt1", + "exception/texcsub", "exception/tfinally", + "exception/tfinally2", "exception/tfinally3", + "collections/tactiontable", "method/tmultimjs", + "varres/tvarres0", "varres/tvarres3", "varres/tvarres4", + "varres/tvartup", "int/tints", "int/tunsignedinc", + "async/tjsandnativeasync"]: + test "tests/" & testfile & ".nim" + + for testfile in ["strutils", "json", "random", "times", "logging"]: + test "lib/pure/" & testfile & ".nim" + +# ------------------------- nim in action ----------- + +proc testNimInAction(r: var TResults, cat: Category, options: string) = + template test(filename: untyped) = + testSpec r, makeTest(filename, options, cat) + + template testJS(filename: untyped) = + testSpec r, makeTest(filename, options, cat), {targetJS} + + template testCPP(filename: untyped) = + testSpec r, makeTest(filename, options, cat), {targetCpp} + + let tests = [ + "niminaction/Chapter1/various1", + "niminaction/Chapter2/various2", + "niminaction/Chapter2/resultaccept", + "niminaction/Chapter2/resultreject", + "niminaction/Chapter2/explicit_discard", + "niminaction/Chapter2/no_def_eq", + "niminaction/Chapter2/no_iterator", + "niminaction/Chapter2/no_seq_type", + "niminaction/Chapter3/ChatApp/src/server", + "niminaction/Chapter3/ChatApp/src/client", + "niminaction/Chapter3/various3", + "niminaction/Chapter6/WikipediaStats/concurrency_regex", + "niminaction/Chapter6/WikipediaStats/concurrency", + "niminaction/Chapter6/WikipediaStats/naive", + "niminaction/Chapter6/WikipediaStats/parallel_counts", + "niminaction/Chapter6/WikipediaStats/race_condition", + "niminaction/Chapter6/WikipediaStats/sequential_counts", + "niminaction/Chapter6/WikipediaStats/unguarded_access", + "niminaction/Chapter7/Tweeter/src/tweeter", + "niminaction/Chapter7/Tweeter/src/createDatabase", + "niminaction/Chapter7/Tweeter/tests/database_test", + "niminaction/Chapter8/sdl/sdl_test" + ] + + when false: + # Verify that the files have not been modified. Death shall fall upon + # whoever edits these hashes without dom96's permission, j/k. But please only + # edit when making a conscious breaking change, also please try to make your + # commit message clear and notify me so I can easily compile an errata later. + # --------------------------------------------------------- + # Hash-checks are disabled for Nim 1.1 and beyond + # since we needed to fix the deprecated unary '<' operator. + const refHashes = @[ + "51afdfa84b3ca3d810809d6c4e5037ba", + "30f07e4cd5eaec981f67868d4e91cfcf", + "d14e7c032de36d219c9548066a97e846", + "b335635562ff26ec0301bdd86356ac0c", + "6c4add749fbf50860e2f523f548e6b0e", + "76de5833a7cc46f96b006ce51179aeb1", + "705eff79844e219b47366bd431658961", + "a1e87b881c5eb161553d119be8b52f64", + "2d706a6ec68d2973ec7e733e6d5dce50", + "c11a013db35e798f44077bc0763cc86d", + "3e32e2c5e9a24bd13375e1cd0467079c", + "a5452722b2841f0c1db030cf17708955", + "dc6c45eb59f8814aaaf7aabdb8962294", + "69d208d281a2e7bffd3eaf4bab2309b1", + "ec05666cfb60211bedc5e81d4c1caf3d", + "da520038c153f4054cb8cc5faa617714", + "59906c8cd819cae67476baa90a36b8c1", + "9a8fe78c588d08018843b64b57409a02", + "8b5d28e985c0542163927d253a3e4fc9", + "783299b98179cc725f9c46b5e3b5381f", + "1a2b3fba1187c68d6a9bfa66854f3318", + "391ff57b38d9ea6f3eeb3fe69ab539d3" + ] + for i, test in tests: + let filename = testsDir / test.addFileExt("nim") + let testHash = getMD5(readFile(filename).string) + doAssert testHash == refHashes[i], "Nim in Action test " & filename & + " was changed: " & $(i: i, testHash: testHash, refHash: refHashes[i]) + + # Run the tests. + for testfile in tests: + test "tests/" & testfile & ".nim" + let jsFile = "tests/niminaction/Chapter8/canvas/canvas_test.nim" + testJS jsFile + let cppFile = "tests/niminaction/Chapter8/sfml/sfml_test.nim" + testCPP cppFile + +# ------------------------- manyloc ------------------------------------------- + +proc findMainFile(dir: string): string = + # finds the file belonging to ".nim.cfg"; if there is no such file + # it returns the some ".nim" file if there is only one: + const cfgExt = ".nim.cfg" + result = "" + var nimFiles = 0 + for kind, file in os.walkDir(dir): + if kind == pcFile: + if file.endsWith(cfgExt): return file[0..^(cfgExt.len+1)] & ".nim" + elif file.endsWith(".nim"): + if result.len == 0: result = file + inc nimFiles + if nimFiles != 1: result.setLen(0) + +proc manyLoc(r: var TResults, cat: Category, options: string) = + for kind, dir in os.walkDir("tests/manyloc"): + if kind == pcDir: + when defined(windows): + if dir.endsWith"nake": continue + if dir.endsWith"named_argument_bug": continue + let mainfile = findMainFile(dir) + if mainfile != "": + var test = makeTest(mainfile, options, cat) + test.spec.action = actionCompile + testSpec r, test + +proc compileExample(r: var TResults, pattern, options: string, cat: Category) = + for test in os.walkFiles(pattern): + var test = makeTest(test, options, cat) + test.spec.action = actionCompile + testSpec r, test + +proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = + var files: seq[string] + + proc isValid(file: string): bool = + for dir in parentDirs(file, inclusive = false): + if dir.lastPathPart in ["includes", "nimcache"]: + # e.g.: lib/pure/includes/osenv.nim gives: Error: This is an include file for os.nim! + return false + let name = extractFilename(file) + if name.splitFile.ext != ".nim": return false + for namei in disabledFiles: + # because of `LockFreeHash.nim` which has case + if namei.cmpPaths(name) == 0: return false + return true + + for testFile in os.walkDirRec(pattern): + if isValid(testFile): + files.add testFile + + files.sort # reproducible order + for testFile in files: + let contents = readFile(testFile) + var testObj = makeTest(testFile, options, cat) + #[ + todo: + this logic is fragile: + false positives (if appears in a comment), or false negatives, e.g. + `when defined(osx) and isMainModule`. + Instead of fixing this, see https://github.com/nim-lang/Nim/issues/10045 + for a much better way. + ]# + if "when isMainModule" notin contents: + testObj.spec.action = actionCompile + testSpec r, testObj + +# ----------------------------- nimble ---------------------------------------- +proc listPackagesAll(): seq[NimblePackage] = + var nimbleDir = getEnv("NIMBLE_DIR") + if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" + let packageIndex = nimbleDir / "packages_official.json" + let packageList = parseFile(packageIndex) + proc findPackage(name: string): JsonNode = + for a in packageList: + if a["name"].str == name: return a + for pkg in important_packages.packages.items: + var pkg = pkg + if pkg.url.len == 0: + let pkg2 = findPackage(pkg.name) + if pkg2 == nil: + raise newException(ValueError, "Cannot find package '$#'." % pkg.name) + pkg.url = pkg2["url"].str + result.add pkg + +proc listPackages(packageFilter: string): seq[NimblePackage] = + let pkgs = listPackagesAll() + if packageFilter.len != 0: + # xxx document `packageFilter`, seems like a bad API, + # at least should be a regex; a substring match makes no sense. + result = pkgs.filterIt(packageFilter in it.name) + else: + if testamentData0.batchArg == "allowed_failures": + result = pkgs.filterIt(it.allowFailure) + elif testamentData0.testamentNumBatch == 0: + result = pkgs + else: + let pkgs2 = pkgs.filterIt(not it.allowFailure) + for i in 0..<pkgs2.len: + if i mod testamentData0.testamentNumBatch == testamentData0.testamentBatch: + result.add pkgs2[i] + +proc makeSupTest(test, options: string, cat: Category, debugInfo = ""): TTest = + result.cat = cat + result.name = test + result.options = options + result.debugInfo = debugInfo + result.startTime = epochTime() + +import std/private/gitutils + +proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = + let nimbleExe = findExe("nimble") + doAssert nimbleExe != "", "Cannot run nimble tests: Nimble binary not found." + doAssert execCmd("$# update" % nimbleExe) == 0, "Cannot run nimble tests: Nimble update failed." + let packageFileTest = makeSupTest("PackageFileParsed", "", cat) + let packagesDir = "pkgstemp" + createDir(packagesDir) + var errors = 0 + try: + let pkgs = listPackages(packageFilter) + for i, pkg in pkgs: + inc r.total + var test = makeSupTest(pkg.name, "", cat, "[$#/$#] " % [$i, $pkgs.len]) + let buildPath = packagesDir / pkg.name + template tryCommand(cmd: string, workingDir2 = buildPath, reFailed = reInstallFailed, maxRetries = 1): string = + var outp: string + let ok = retryCall(maxRetry = maxRetries, backoffDuration = 10.0): + var status: int + (outp, status) = execCmdEx(cmd, workingDir = workingDir2) + status == QuitSuccess + if not ok: + if pkg.allowFailure: + inc r.passed + inc r.failedButAllowed + addResult(r, test, targetC, "", "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure) + continue + outp + + if not dirExists(buildPath): + discard tryCommand("git clone $# $#" % [pkg.url.quoteShell, buildPath.quoteShell], workingDir2 = ".", maxRetries = 3) + if not pkg.useHead: + discard tryCommand("git fetch --tags", maxRetries = 3) + let describeOutput = tryCommand("git describe --tags --abbrev=0") + discard tryCommand("git checkout $#" % [describeOutput.strip.quoteShell]) + discard tryCommand("nimble install --depsOnly -y", maxRetries = 3) + let cmds = pkg.cmd.split(';') + for i in 0 ..< cmds.len - 1: + discard tryCommand(cmds[i], maxRetries = 3) + discard tryCommand(cmds[^1], reFailed = reBuildFailed) + inc r.passed + r.addResult(test, targetC, "", "", "", reSuccess, allowFailure = pkg.allowFailure) + + errors = r.total - r.passed + if errors == 0: + r.addResult(packageFileTest, targetC, "", "", "", reSuccess) + else: + r.addResult(packageFileTest, targetC, "", "", "", reBuildFailed) + + except JsonParsingError: + errors = 1 + r.addResult(packageFileTest, targetC, "", "", "Invalid package file", reBuildFailed) + raise + except ValueError: + errors = 1 + r.addResult(packageFileTest, targetC, "", "", "Unknown package", reBuildFailed) + raise # bug #18805 + finally: + if errors == 0: removeDir(packagesDir) + +# ---------------- IC tests --------------------------------------------- + +proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; + isNavigatorTest: bool) = + const + tooltests = ["compiler/nim.nim"] + writeOnly = " --incremental:writeonly " + readOnly = " --incremental:readonly " + incrementalOn = " --incremental:on -d:nimIcIntegrityChecks " + navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint:Conf:off --warnings:off " + + template test(x: untyped) = + testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) + + template editedTest(x: untyped) = + var test = makeTest(file, x & options, cat) + if isNavigatorTest: + test.spec.action = actionCompile + test.spec.targets = {getTestSpecTarget()} + testSpecWithNimcache(r, test, nimcache) + + template checkTest() = + var test = makeRawTest(file, options, cat) + test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:on $options " & file + testSpecWithNimcache(r, test, nimcache) + + if not isNavigatorTest: + for file in tooltests: + let nimcache = nimcacheDir(file, options, getTestSpecTarget()) + removeDir(nimcache) + + let oldPassed = r.passed + checkTest() + + if r.passed == oldPassed+1: + checkTest() + if r.passed == oldPassed+2: + checkTest() + + const tempExt = "_temp.nim" + for it in walkDirRec(testsDir): + # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test + if isTestFile(it) and not it.endsWith(tempExt): + let nimcache = nimcacheDir(it, options, getTestSpecTarget()) + removeDir(nimcache) + + let content = readFile(it) + for fragment in content.split("#!EDIT!#"): + let file = it.replace(".nim", tempExt) + writeFile(file, fragment) + let oldPassed = r.passed + editedTest(if isNavigatorTest: navTestConfig else: incrementalOn) + if r.passed != oldPassed+1: break + +# ---------------------------------------------------------------------------- + +const AdditionalCategories = ["debugger", "examples", "lib", "ic", "navigator"] +const MegaTestCat = "megatest" + +proc `&.?`(a, b: string): string = + # candidate for the stdlib? + result = if b.startsWith(a): b else: a & b + +proc processSingleTest(r: var TResults, cat: Category, options, test: string, targets: set[TTarget], targetsSet: bool) = + var targets = targets + if not targetsSet: + let target = if cat.string.normalize == "js": targetJS else: targetC + targets = {target} + doAssert fileExists(test), test & " test does not exist" + testSpec r, makeTest(test, options, cat), targets + +proc isJoinableSpec(spec: TSpec): bool = + # xxx simplify implementation using a whitelist of fields that are allowed to be + # set to non-default values (use `fieldPairs`), to avoid issues like bug #16576. + result = useMegatest and not spec.sortoutput and + spec.action == actionRun and + not fileExists(spec.file.changeFileExt("cfg")) and + not fileExists(spec.file.changeFileExt("nims")) and + not fileExists(parentDir(spec.file) / "nim.cfg") and + not fileExists(parentDir(spec.file) / "config.nims") and + spec.cmd.len == 0 and + spec.err != reDisabled and + not spec.unjoinable and + spec.exitCode == 0 and + spec.input.len == 0 and + spec.nimout.len == 0 and + spec.nimoutFull == false and + # so that tests can have `nimoutFull: true` with `nimout.len == 0` with + # the meaning that they expect empty output. + spec.matrix.len == 0 and + spec.outputCheck != ocSubstr and + spec.ccodeCheck.len == 0 and + (spec.targets == {} or spec.targets == {targetC}) + if result: + if spec.file.readFile.contains "when isMainModule": + result = false + +proc quoted(a: string): string = + # todo: consider moving to system.nim + result.addQuoted(a) + +proc runJoinedTest(r: var TResults, cat: Category, testsDir: string, options: string) = + ## returns a list of tests that have problems + #[ + xxx create a reusable megatest API after abstracting out testament specific code, + refs https://github.com/timotheecour/Nim/issues/655 + and https://github.com/nim-lang/gtk2/pull/28; it's useful in other contexts. + ]# + var specs: seq[TSpec] = @[] + for kind, dir in walkDir(testsDir): + assert dir.startsWith(testsDir) + let cat = dir[testsDir.len .. ^1] + if kind == pcDir and cat notin specialCategories: + for file in walkDirRec(testsDir / cat): + if isTestFile(file): + var spec: TSpec + try: + spec = parseSpec(file) + except ValueError: + # e.g. for `tests/navigator/tincludefile.nim` which have multiple + # specs; this will be handled elsewhere + echo "parseSpec raised ValueError for: '$1', assuming this will be handled outside of megatest" % file + continue + if isJoinableSpec(spec): + specs.add spec + + proc cmp(a: TSpec, b: TSpec): auto = cmp(a.file, b.file) + sort(specs, cmp = cmp) # reproducible order + echo "joinable specs: ", specs.len + + if simulate: + var s = "runJoinedTest: " + for a in specs: s.add a.file & " " + echo s + return + + var megatest: string + # xxx (minor) put outputExceptedFile, outputGottenFile, megatestFile under here or `buildDir` + var outDir = nimcacheDir(testsDir / "megatest", "", targetC) + template toMarker(file, i): string = + "megatest:processing: [$1] $2" % [$i, file] + for i, runSpec in specs: + let file = runSpec.file + let file2 = outDir / ("megatest_a_$1.nim" % $i) + # `include` didn't work with `trecmod2.nim`, so using `import` + let code = "echo $1\nstatic: echo \"CT:\", $1\n" % [toMarker(file, i).quoted] + createDir(file2.parentDir) + writeFile(file2, code) + megatest.add "import $1\nimport $2 as megatest_b_$3\n" % [file2.quoted, file.quoted, $i] + + let megatestFile = testsDir / "megatest.nim" # so it uses testsDir / "config.nims" + writeFile(megatestFile, megatest) + + let root = getCurrentDir() + + var args = @["c", "--nimCache:" & outDir, "-d:testing", "-d:nimMegatest", "--listCmd", + "--path:" & root] + args.add options.parseCmdLine + args.add megatestFile + var (cmdLine, buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, input = "") + if exitCode != 0: + echo "$ " & cmdLine & "\n" & buf + quit(failString & "megatest compilation failed") + + (buf, exitCode) = execCmdEx(megatestFile.changeFileExt(ExeExt).dup normalizeExe) + if exitCode != 0: + echo buf + quit(failString & "megatest execution failed") + + const outputExceptedFile = "outputExpected.txt" + const outputGottenFile = "outputGotten.txt" + writeFile(outputGottenFile, buf) + var outputExpected = "" + for i, runSpec in specs: + outputExpected.add toMarker(runSpec.file, i) & "\n" + if runSpec.output.len > 0: + outputExpected.add runSpec.output + if not runSpec.output.endsWith "\n": + outputExpected.add '\n' + + if buf != outputExpected: + writeFile(outputExceptedFile, outputExpected) + echo diffFiles(outputGottenFile, outputExceptedFile).output + echo failString & "megatest output different, see $1 vs $2" % [outputGottenFile, outputExceptedFile] + # outputGottenFile, outputExceptedFile not removed on purpose for debugging. + quit 1 + else: + echo "megatest output OK" + + +# --------------------------------------------------------------------------- + +proc processCategory(r: var TResults, cat: Category, + options, testsDir: string, + runJoinableTests: bool) = + let cat2 = cat.string.normalize + var handled = false + if isNimRepoTests(): + handled = true + case cat2 + of "js": + # only run the JS tests on Windows or Linux because Travis is bad + # and other OSes like Haiku might lack nodejs: + if not defined(linux) and isTravis: + discard + else: + jsTests(r, cat, options) + of "dll": + dllTests(r, cat, options & " -d:nimDebugDlOpen") + of "gc": + gcTests(r, cat, options) + of "debugger": + debuggerTests(r, cat, options) + of "manyloc": + manyLoc r, cat, options + of "threads": + threadTests r, cat, options & " --threads:on" + of "io": + ioTests r, cat, options + of "async": + asyncTests r, cat, options + of "lib": + testStdlib(r, "lib/pure/", options, cat) + testStdlib(r, "lib/packages/docutils/", options, cat) + of "examples": + compileExample(r, "examples/*.nim", options, cat) + compileExample(r, "examples/gtk/*.nim", options, cat) + compileExample(r, "examples/talk/*.nim", options, cat) + of "nimble-packages": + testNimblePackages(r, cat, options) + of "niminaction": + testNimInAction(r, cat, options) + of "ic": + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=false) + of "navigator": + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=true) + of "untestable": + # These require special treatment e.g. because they depend on a third party + # dependency; see `trunner_special` which runs some of those. + discard + else: + handled = false + if not handled: + case cat2 + of "megatest": + runJoinedTest(r, cat, testsDir, options) + if isNimRepoTests(): + runJoinedTest(r, cat, testsDir, options & " --mm:refc") + else: + var testsRun = 0 + var files: seq[string] + for file in walkDirRec(testsDir &.? cat.string): + if isTestFile(file): files.add file + files.sort # give reproducible order + for i, name in files: + var test = makeTest(name, options, cat) + if runJoinableTests or not isJoinableSpec(test.spec) or cat.string in specialCategories: + discard "run the test" + else: + test.spec.err = reJoined + testSpec r, test + inc testsRun + if testsRun == 0: + const whiteListedDirs = ["deps", "htmldocs", "pkgs"] + # `pkgs` because bug #16556 creates `pkgs` dirs and this can affect some users + # that try an old version of choosenim. + doAssert cat.string in whiteListedDirs, + "Invalid category specified: '$#' not in whilelist: $#" % [cat.string, $whiteListedDirs] + +proc processPattern(r: var TResults, pattern, options: string; simulate: bool) = + var testsRun = 0 + if dirExists(pattern): + for k, name in walkDir(pattern): + if k in {pcFile, pcLinkToFile} and name.endsWith(".nim"): + if simulate: + echo "Detected test: ", name + else: + var test = makeTest(name, options, Category"pattern") + testSpec r, test + inc testsRun + else: + for name in walkPattern(pattern): + if simulate: + echo "Detected test: ", name + else: + var test = makeTest(name, options, Category"pattern") + testSpec r, test + inc testsRun + if testsRun == 0: + echo "no tests were found for pattern: ", pattern diff --git a/testament/htmlgen.nim b/testament/htmlgen.nim new file mode 100644 index 000000000..174f36d0b --- /dev/null +++ b/testament/htmlgen.nim @@ -0,0 +1,149 @@ +# +# +# Nim Tester +# (c) Copyright 2017 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## HTML generator for the tester. + +import strutils, json, os, times + +import "testamenthtml.nimf" + +proc generateTestResultPanelPartial(outfile: File, testResultRow: JsonNode) = + let + trId = htmlQuote(testResultRow["category"].str & "_" & testResultRow["name"].str). + multiReplace({".": "_", " ": "_", ":": "_", "/": "_"}) + name = testResultRow["name"].str.htmlQuote() + category = testResultRow["category"].str.htmlQuote() + target = testResultRow["target"].str.htmlQuote() + action = testResultRow["action"].str.htmlQuote() + result = htmlQuote testResultRow["result"].str + expected = testResultRow["expected"].getStr + gotten = testResultRow["given"].getStr + timestamp = "unknown" + var + panelCtxClass, textCtxClass, bgCtxClass: string + resultSign, resultDescription: string + case result + of "reSuccess": + panelCtxClass = "success" + textCtxClass = "success" + bgCtxClass = "success" + resultSign = "ok" + resultDescription = "PASS" + of "reDisabled", "reJoined": + panelCtxClass = "info" + textCtxClass = "info" + bgCtxClass = "info" + resultSign = "question" + resultDescription = if result != "reJoined": "SKIP" else: "JOINED" + else: + panelCtxClass = "danger" + textCtxClass = "danger" + bgCtxClass = "danger" + resultSign = "exclamation" + resultDescription = "FAIL" + + outfile.generateHtmlTestresultPanelBegin( + trId, name, target, category, action, resultDescription, + timestamp, result, resultSign, panelCtxClass, textCtxClass, bgCtxClass + ) + if expected.isEmptyOrWhitespace() and gotten.isEmptyOrWhitespace(): + outfile.generateHtmlTestresultOutputNone() + else: + outfile.generateHtmlTestresultOutputDetails( + expected.strip().htmlQuote, + gotten.strip().htmlQuote + ) + outfile.generateHtmlTestresultPanelEnd() + +type + AllTests = object + data: JsonNode + totalCount, successCount, ignoredCount, failedCount: int + successPercentage, ignoredPercentage, failedPercentage: BiggestFloat + +proc allTestResults(onlyFailing = false): AllTests = + result.data = newJArray() + for file in os.walkFiles("testresults/*.json"): + let data = parseFile(file) + if data.kind != JArray: + echo "[ERROR] ignoring json file that is not an array: ", file + else: + for elem in data: + let state = elem["result"].str + inc result.totalCount + if state.contains("reSuccess"): inc result.successCount + elif state.contains("reDisabled") or state.contains("reJoined"): + inc result.ignoredCount + if not onlyFailing or not(state.contains("reSuccess")): + result.data.add elem + result.successPercentage = 100 * + (result.successCount.toBiggestFloat / result.totalCount.toBiggestFloat) + result.ignoredPercentage = 100 * + (result.ignoredCount.toBiggestFloat / result.totalCount.toBiggestFloat) + result.failedCount = result.totalCount - + result.successCount - result.ignoredCount + result.failedPercentage = 100 * + (result.failedCount.toBiggestFloat / result.totalCount.toBiggestFloat) + +proc generateTestResultsPanelGroupPartial(outfile: File, allResults: JsonNode) = + for testresultRow in allResults: + generateTestResultPanelPartial(outfile, testresultRow) + +proc generateAllTestsContent(outfile: File, allResults: AllTests, + onlyFailing = false) = + if allResults.data.len < 1: return # Nothing to do if there is no data. + # Only results from one test run means that test run environment info is the + # same for all tests + let + firstRow = allResults.data[0] + commit = htmlQuote firstRow["commit"].str + branch = htmlQuote firstRow["branch"].str + machine = htmlQuote firstRow["machine"].str + + outfile.generateHtmlAllTestsBegin( + machine, commit, branch, + allResults.totalCount, + allResults.successCount, + formatBiggestFloat(allResults.successPercentage, ffDecimal, 2) & "%", + allResults.ignoredCount, + formatBiggestFloat(allResults.ignoredPercentage, ffDecimal, 2) & "%", + allResults.failedCount, + formatBiggestFloat(allResults.failedPercentage, ffDecimal, 2) & "%", + onlyFailing + ) + generateTestResultsPanelGroupPartial(outfile, allResults.data) + outfile.generateHtmlAllTestsEnd() + +proc generateHtml*(filename: string, onlyFailing: bool) = + let + currentTime = getTime().local() + timestring = htmlQuote format(currentTime, "yyyy-MM-dd HH:mm:ss 'UTC'zzz") + var outfile = open(filename, fmWrite) + + outfile.generateHtmlBegin() + + generateAllTestsContent(outfile, allTestResults(onlyFailing), onlyFailing) + + outfile.generateHtmlEnd(timestring) + + outfile.flushFile() + close(outfile) + +proc dumpJsonTestResults*(prettyPrint, onlyFailing: bool) = + var + outfile = stdout + jsonString: string + + let results = allTestResults(onlyFailing) + if prettyPrint: + jsonString = results.data.pretty() + else: + jsonString = $ results.data + + outfile.writeLine(jsonString) diff --git a/testament/important_packages.nim b/testament/important_packages.nim new file mode 100644 index 000000000..efec04b3c --- /dev/null +++ b/testament/important_packages.nim @@ -0,0 +1,193 @@ +##[ +## note 1 +`useHead` should ideally be used as the default but lots of packages (e.g. `chronos`) +don't have release tags (or have really old ones compared to HEAD), making it +impossible to test them reliably here. + +packages listed here should ideally have regularly updated release tags, so that: +* we're testing recent versions of the package +* the version that's tested is stable enough even if HEAD may occasionally break + +## note 2: D20210308T165435:here +nimble packages should be testable as follows: +git clone $url $dir && cd $dir +NIMBLE_DIR=$TMP_NIMBLE_DIR XDG_CONFIG_HOME= nimble install --depsOnly -y +NIMBLE_DIR=$TMP_NIMBLE_DIR XDG_CONFIG_HOME= nimble test + +if this fails (e.g. nimcrypto), it could be because a package lacks a `tests/nim.cfg` with `--path:..`, +so the above commands would've worked by accident with `nimble install` but not with `nimble install --depsOnly`. +When this is the case, a workaround is to test this package here by adding `--path:$srcDir` on the test `cmd`. +]## + +type NimblePackage* = object + name*, cmd*, url*: string + useHead*: bool + allowFailure*: bool + ## When true, we still run the test but the test is allowed to fail. + ## This is useful for packages that currently fail but that we still want to + ## run in CI, e.g. so that we can monitor when they start working again and + ## are reminded about those failures without making CI fail for unrelated PRs. + +var packages*: seq[NimblePackage] + +proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true, allowFailure = false) = + packages.add NimblePackage(name: name, cmd: cmd, url: url, useHead: useHead, allowFailure: allowFailure) + +pkg "alea" +pkg "argparse" +pkg "arraymancer", "nim c tests/tests_cpu.nim" +pkg "ast_pattern_matching", "nim c -r tests/test1.nim" +pkg "asyncftpclient", "nimble compileExample" +pkg "asyncthreadpool", "nimble test --mm:refc" +pkg "awk" +pkg "bigints" +pkg "binaryheap", "nim c -r binaryheap.nim" +pkg "BipBuffer" +pkg "blscurve", allowFailure = true +pkg "bncurve" +pkg "brainfuck", "nim c -d:release -r tests/compile.nim" +pkg "bump", "nim c --mm:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump", allowFailure = true +pkg "c2nim", "nim c testsuite/tester.nim" +pkg "cascade" +pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true +pkg "checksums" +pkg "chroma" +pkg "chronicles", "nim c -o:chr -r chronicles.nim" +pkg "chronos", "nim c -r -d:release tests/testall" +pkg "cligen", "nim c --path:. -r cligen.nim" +pkg "combparser", "nimble test --mm:orc" +pkg "compactdict" +pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehension" +pkg "constantine", "nimble make_lib" +pkg "cowstrings" +pkg "criterion", allowFailure = true # needs testing binary +pkg "datamancer" +pkg "dashing", "nim c tests/functional.nim" +pkg "delaunay" +pkg "dnsclient", allowFailure = true # super fragile +pkg "docopt" +pkg "dotenv" +# when defined(linux): pkg "drchaos" +pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" +pkg "elvis" +pkg "faststreams" +pkg "fidget" +pkg "fragments", "nim c -r fragments/dsl.nim", allowFailure = true # pending https://github.com/nim-lang/packages/issues/2115 +pkg "fusion" +pkg "gara" +pkg "glob" +pkg "ggplotnim", "nim c -d:noCairo -r tests/tests.nim" +pkg "gittyup", "nimble test", "https://github.com/disruptek/gittyup", allowFailure = true +pkg "gnuplot", "nim c gnuplot.nim" +# pkg "gram", "nim c -r --mm:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram" + # pending https://github.com/nim-lang/Nim/issues/16509 +pkg "hts", "nim c -o:htss src/hts.nim" +pkg "httpauth" +pkg "httputils" +pkg "illwill", "nimble examples" +pkg "inim" +pkg "itertools", "nim doc src/itertools.nim" +pkg "iterutils" +pkg "json_rpc" +pkg "json_serialization" +pkg "jstin" +pkg "karax", "nim c -r tests/tester.nim" +pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree" +pkg "loopfusion" +pkg "lockfreequeues" +pkg "macroutils" +pkg "manu" +pkg "markdown" +pkg "measuremancer", "nimble testDeps; nimble -y test" +pkg "memo" +pkg "msgpack4nim", "nim c -r tests/test_spec.nim" +pkg "nake", "nim c nakefile.nim" +pkg "neo", "nim c -d:blas=openblas --mm:refc tests/all.nim" +pkg "nesm", "nimble tests", "https://github.com/nim-lang/NESM", useHead = true, allowFailure = true + # inactive, tests not adapted to #23096 +pkg "netty" +pkg "nico", allowFailure = true +pkg "nicy", "nim c -r src/nicy.nim" +pkg "nigui", "nim c -o:niguii -r src/nigui.nim" +pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround needed, see D20210308T165435 +pkg "NimData", "nim c -o:nimdataa src/nimdata.nim" +pkg "nimes", "nim c src/nimes.nim" +pkg "nimfp", "nim c -o:nfp -r src/fp.nim" +pkg "nimgame2", "nim c --mm:refc nimgame2/nimgame.nim" +pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" +pkg "nimib" +pkg "nimlsp" +pkg "nimly", "nim c -r tests/test_readme_example.nim" +pkg "nimongo", "nimble test_ci", allowFailure = true +pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true +pkg "nimPNG", useHead = true +pkg "nimpy", "nim c -r tests/nimfrompy.nim" +pkg "nimquery" +pkg "nimsl" +pkg "nimsvg" +pkg "nimterop", "nimble minitest", url = "https://github.com/nim-lang/nimterop" +pkg "nimwc", "nim c nimwc.nim" +pkg "nimx", "nim c test/main.nim", allowFailure = true +pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" +pkg "norm", "testament r tests/common/tmodel.nim" +pkg "normalize" +pkg "npeg", "nimble testarc" +pkg "numericalnim", "nimble nimCI" +pkg "optionsutils" +pkg "ormin", "nim c -o:orminn ormin.nim" +pkg "parsetoml" +pkg "patty" +pkg "pixie" +pkg "plotly", "nim c examples/all.nim" +pkg "pnm" +pkg "polypbren" +pkg "presto" +pkg "prologue", "nimble tcompile" +# remove fork after https://github.com/PMunch/combparser/pull/7 is merged: +pkg "protobuf", "nimble install -y https://github.com/metagn/combparser@#HEAD; nim c -o:protobuff -r src/protobuf.nim" +pkg "rbtree" +pkg "react", "nimble example" +pkg "regex", "nim c src/regex" +pkg "results", "nim c -r results.nim" +pkg "RollingHash", "nim c -r tests/test_cyclichash.nim" +pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim" +pkg "sdl1", "nim c -r src/sdl.nim" +pkg "sdl2_nim", "nim c -r sdl2/sdl.nim" +pkg "serialization" +pkg "sigv4", "nim c --mm:arc -r sigv4.nim", "https://github.com/disruptek/sigv4" +pkg "sim" +pkg "smtp", "nimble compileExample" +pkg "snip", "nimble test", "https://github.com/genotrance/snip" +pkg "ssostrings" +pkg "stew" +pkg "stint", "nim c stint.nim" +pkg "strslice" +pkg "strunicode", "nim c -r --mm:refc src/strunicode.nim" +pkg "supersnappy" +pkg "synthesis" +pkg "taskpools" +pkg "telebot", "nim c -o:tbot -r src/telebot.nim" +pkg "tempdir" +pkg "templates" +pkg "tensordsl", "nim c -r --mm:refc tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" +pkg "terminaltables", "nim c src/terminaltables.nim" +pkg "termstyle", "nim c -r termstyle.nim" +pkg "testutils" +pkg "timeit" +pkg "timezones" +pkg "tiny_sqlite" +pkg "unicodedb", "nim c -d:release -r tests/tests.nim" +pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" +pkg "union", "nim c -r tests/treadme.nim", url = "https://github.com/alaviss/union" +pkg "unittest2" +pkg "unpack" +pkg "weave", "nimble install -y cligen@#HEAD; nimble test_gc_arc", useHead = true +pkg "websock" +pkg "websocket", "nim c websocket.nim" +# pkg "winim", allowFailure = true +pkg "with" +pkg "ws", allowFailure = true +pkg "yaml" +pkg "zero_functional", "nim c -r test.nim" +pkg "zippy" +pkg "zxcvbn" diff --git a/testament/lib/readme.md b/testament/lib/readme.md new file mode 100644 index 000000000..20e866338 --- /dev/null +++ b/testament/lib/readme.md @@ -0,0 +1,4 @@ +This directory contains helper files used by several tests, to avoid +code duplication, akin to a std extension tailored for testament. + +Some of these could later migrate to stdlib. diff --git a/testament/lib/stdtest/netutils.nim b/testament/lib/stdtest/netutils.nim new file mode 100644 index 000000000..5115390e0 --- /dev/null +++ b/testament/lib/stdtest/netutils.nim @@ -0,0 +1,13 @@ +import std/[nativesockets, asyncdispatch, os] + +proc bindAvailablePort*(handle: SocketHandle, port = Port(0)): Port = + ## See also `asynchttpserver.getPort`. + block: + var name: Sockaddr_in + name.sin_family = typeof(name.sin_family)(toInt(AF_INET)) + name.sin_port = htons(uint16(port)) + name.sin_addr.s_addr = htonl(INADDR_ANY) + if bindAddr(handle, cast[ptr SockAddr](addr(name)), + sizeof(name).Socklen) < 0'i32: + raiseOSError(osLastError(), $port) + result = getLocalAddr(handle, AF_INET)[1] diff --git a/testament/lib/stdtest/specialpaths.nim b/testament/lib/stdtest/specialpaths.nim new file mode 100644 index 000000000..e214d113d --- /dev/null +++ b/testament/lib/stdtest/specialpaths.nim @@ -0,0 +1,55 @@ +#[ +todo: move findNimStdLibCompileTime, findNimStdLib here +xxx: factor pending https://github.com/timotheecour/Nim/issues/616 + +## note: $lib vs $nim +note: these can resolve to 3 different paths if running via `nim c --lib:lib foo`, +eg if compiler was installed via nimble (or is in nim path), and nim is external +(ie not in `$lib/../bin/` dir) + +import "$lib/../compiler/nimpaths" # <- most robust if you want to favor --lib:lib +import "$nim/compiler/nimpaths" +import compiler/nimpaths +]# + +import os +when defined(nimPreviewSlimSystem): + import std/assertions + +# Note: all the const paths defined here are known at compile time and valid +# so long Nim repo isn't relocated after compilation. +# This means the binaries they produce will embed hardcoded paths, which +# isn't appropriate for some applications that need to be relocatable. + +const + sourcePath = currentSourcePath() + # robust way to derive other paths here + # We don't depend on PATH so this is robust to having multiple nim binaries + nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir ## root of Nim repo + testsFname* = "tests" + stdlibDir* = nimRootDir / "lib" + systemPath* = stdlibDir / "system.nim" + testsDir* = nimRootDir / testsFname + buildDir* = nimRootDir / "build" + ## refs #10268: all testament generated files should go here to avoid + ## polluting .gitignore + +proc splitTestFile*(file: string): tuple[cat: string, path: string] = + ## At least one directory is required in the path, to use as a category name + runnableExamples: + doAssert splitTestFile("tests/fakedir/tfakename.nim") == ("fakedir", "tests/fakedir/tfakename.nim".unixToNativePath) + for p in file.parentDirs(inclusive = false): + let parent = p.parentDir + if parent.lastPathPart == testsFname: + result.cat = p.lastPathPart + let dir = getCurrentDir() + if file.isRelativeTo(dir): + result.path = file.relativePath(dir) + else: + result.path = file + return result + raiseAssert "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'" + +static: + # sanity check + doAssert fileExists(systemPath) diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim new file mode 100644 index 000000000..a490b17c8 --- /dev/null +++ b/testament/lib/stdtest/testutils.nim @@ -0,0 +1,126 @@ +import std/private/miscdollars +when defined(nimscript): + import std/os # xxx investigate why needed +else: + from std/os import getEnv +import std/[macros, genasts] + +template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = + ## API to deal with flaky or failing tests. This avoids disabling entire tests + ## altogether so that at least the parts that are working are kept being + ## tested. This also avoids making CI fail periodically for tests known to + ## be flaky. Finally, for known failures, passing `notifySuccess = true` will + ## log that the test succeeded, which may indicate that a bug was fixed + ## "by accident" and should be looked into. + const info = instantiationInfo(-1, true) + const expr = astToStr(cond) + if cond and not notifySuccess: + discard # silent success + else: + var msg2 = "" + toLocation(msg2, info.filename, info.line, info.column) + if cond: + # a flaky test is failing, we still report it but we don't fail CI + msg2.add " FLAKY_SUCCESS " + else: + # a previously failing test is now passing, a pre-existing bug might've been + # fixed by accidend + msg2.add " FLAKY_FAILURE " + msg2.add $expr & " " & msg + echo msg2 + +when not defined(js) and not defined(nimscript): + import std/strutils + + proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool = + ## Returns true if each stripped line in `lhs` appears in rhs, using a greedy matching. + # xxx improve error reporting by showing the last matched pair + iterator splitLinesClosure(): string {.closure.} = + for line in splitLines(rhs.strip): + yield line + template isMatch(lhsi, rhsi): bool = + if allowPrefixMatch: + startsWith(rhsi, lhsi) + else: + lhsi == rhsi + + var rhsIter = splitLinesClosure + var currentLine = strip(rhsIter()) + + for line in lhs.strip.splitLines: + let line = line.strip + if line.len != 0: + while not isMatch(line, currentLine): + currentLine = strip(rhsIter()) + if rhsIter.finished: + return false + + if rhsIter.finished: + return false + return true + +template enableRemoteNetworking*: bool = + ## Allows contolling whether to run some test at a statement-level granularity. + ## Using environment variables simplifies propagating this all the way across + ## process calls, e.g. `testament all` calls itself, which in turns invokes + ## a `nim` invocation (possibly via additional intermediate processes). + getEnv("NIM_TESTAMENT_REMOTE_NETWORKING") == "1" + +template disableSSLTesting*: bool = + ## TODO: workaround for GitHub Action gcc 14 matrix; remove this + ## matrix and the flag after Azure agent supports ubuntu 24.04 + getEnv("NIM_TESTAMENT_DISABLE_SSL") == "1" + +template whenRuntimeJs*(bodyIf, bodyElse) = + ##[ + Behaves as `when defined(js) and not nimvm` (which isn't legal yet). + pending improvements to `nimvm`, this sugar helps; use as follows: + + whenRuntimeJs: + doAssert defined(js) + when nimvm: doAssert false + else: discard + do: + discard + ]## + when nimvm: bodyElse + else: + when defined(js): bodyIf + else: bodyElse + +template whenVMorJs*(bodyIf, bodyElse) = + ## Behaves as: `when defined(js) or nimvm` + when nimvm: bodyIf + else: + when defined(js): bodyIf + else: bodyElse + +template accept*(a) = + doAssert compiles(a) + +template reject*(a) = + doAssert not compiles(a) + +template disableVm*(body) = + when nimvm: discard + else: body + +macro assertAll*(body) = + ## works in VM, unlike `check`, `require` + runnableExamples: + assertAll: + 1+1 == 2 + var a = @[1, 2] # statements work + a.len == 2 + # remove this once these support VM, pending #10129 (closed but not yet fixed) + result = newStmtList() + for a in body: + result.add genAst(a, a2 = a.repr, info = lineInfo(a)) do: + # D20210421T014713:here + # xxx pending https://github.com/nim-lang/Nim/issues/12030, + # `typeof` should introduce its own scope, so that this + # is sufficient: `typeof(a)` instead of `typeof(block: a)` + when typeof(block: a) is void: a + else: + if not a: + raise newException(AssertionDefect, info & " " & a2) diff --git a/testament/lib/stdtest/unittest_light.nim b/testament/lib/stdtest/unittest_light.nim new file mode 100644 index 000000000..4ab1d7543 --- /dev/null +++ b/testament/lib/stdtest/unittest_light.nim @@ -0,0 +1,37 @@ +import std/assertions + + +proc mismatch*[T](lhs: T, rhs: T): string = + ## Simplified version of `unittest.require` that satisfies a common use case, + ## while avoiding pulling too many dependencies. On failure, diagnostic + ## information is provided that in particular makes it easy to spot + ## whitespace mismatches and where the mismatch is. + proc replaceInvisible(s: string): string = + for a in s: + case a + of '\n': result.add "\\n\n" + else: result.add a + + proc quoted(s: string): string = result.addQuoted s + + result.add '\n' + result.add "lhs:{" & replaceInvisible( + $lhs) & "}\nrhs:{" & replaceInvisible($rhs) & "}\n" + when compiles(lhs.len): + if lhs.len != rhs.len: + result.add "lhs.len: " & $lhs.len & " rhs.len: " & $rhs.len & '\n' + when compiles(lhs[0]): + var i = 0 + while i < lhs.len and i < rhs.len: + if lhs[i] != rhs[i]: break + i.inc + result.add "first mismatch index: " & $i & '\n' + if i < lhs.len and i < rhs.len: + result.add "lhs[i]: {" & quoted($lhs[i]) & "}\nrhs[i]: {" & quoted( + $rhs[i]) & "}\n" + result.add "lhs[0..<i]:{" & replaceInvisible($lhs[ + 0..<i]) & '}' + +proc assertEquals*[T](lhs: T, rhs: T, msg = "") = + if lhs != rhs: + doAssert false, mismatch(lhs, rhs) & '\n' & msg diff --git a/testament/specs.nim b/testament/specs.nim new file mode 100644 index 000000000..c3040c1d8 --- /dev/null +++ b/testament/specs.nim @@ -0,0 +1,521 @@ +# +# +# Nim Tester +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import sequtils, parseutils, strutils, os, streams, parsecfg, + tables, hashes, sets +import compiler/platform + +type TestamentData* = ref object + # better to group globals under 1 object; could group the other ones here too + batchArg*: string + testamentNumBatch*: int + testamentBatch*: int + +let testamentData0* = TestamentData() + +var compilerPrefix* = findExe("nim") + +let isTravis* = existsEnv("TRAVIS") +let isAppVeyor* = existsEnv("APPVEYOR") +let isAzure* = existsEnv("TF_BUILD") + +var skips*: seq[string] + +type + TTestAction* = enum + actionRun = "run" + actionCompile = "compile" + actionReject = "reject" + + TOutputCheck* = enum + ocIgnore = "ignore" + ocEqual = "equal" + ocSubstr = "substr" + + TResultEnum* = enum + reNimcCrash, # nim compiler seems to have crashed + reMsgsDiffer, # error messages differ + reFilesDiffer, # expected and given filenames differ + reLinesDiffer, # expected and given line numbers differ + reOutputsDiffer, + reExitcodesDiffer, # exit codes of program or of valgrind differ + reTimeout, + reInvalidPeg, + reCodegenFailure, + reCodeNotFound, + reExeNotFound, + reInstallFailed # package installation failed + reBuildFailed # package building failed + reDisabled, # test is disabled + reJoined, # test is disabled because it was joined into the megatest + reSuccess # test was successful + reInvalidSpec # test had problems to parse the spec + + TTarget* = enum + targetC = "c" + targetCpp = "cpp" + targetObjC = "objc" + targetJS = "js" + + InlineError* = object + kind*: string + msg*: string + line*, col*: int + + ValgrindSpec* = enum + disabled, enabled, leaking + + TSpec* = object + # xxx make sure `isJoinableSpec` takes into account each field here. + action*: TTestAction + file*, cmd*: string + filename*: string ## Test filename (without path). + input*: string + outputCheck*: TOutputCheck + sortoutput*: bool + output*: string + line*, column*: int + exitCode*: int + msg*: string + ccodeCheck*: seq[string] + maxCodeSize*: int + err*: TResultEnum + inCurrentBatch*: bool + targets*: set[TTarget] + matrix*: seq[string] + nimout*: string + nimoutFull*: bool # whether nimout is all compiler output or a subset + parseErrors*: string # when the spec definition is invalid, this is not empty. + unjoinable*: bool + unbatchable*: bool + # whether this test can be batchable via `NIM_TESTAMENT_BATCH`; only very + # few tests are not batchable; the ones that are not could be turned batchable + # by making the dependencies explicit + useValgrind*: ValgrindSpec + timeout*: float # in seconds, fractions possible, + # but don't rely on much precision + inlineErrors*: seq[InlineError] # line information to error message + debugInfo*: string # debug info to give more context + +proc getCmd*(s: TSpec): string = + if s.cmd.len == 0: + result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:build/deps/pkgs2 $options $file" + else: + result = s.cmd + +const + targetToExt*: array[TTarget, string] = ["nim.c", "nim.cpp", "nim.m", "js"] + targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"] + +proc defaultOptions*(a: TTarget): string = + case a + of targetJS: "-d:nodejs" + # once we start testing for `nim js -d:nimbrowser` (eg selenium or similar), + # we can adapt this logic; or a given js test can override with `-u:nodejs`. + else: "" + +when not declared(parseCfgBool): + # candidate for the stdlib: + proc parseCfgBool(s: string): bool = + case normalize(s) + of "y", "yes", "true", "1", "on": result = true + of "n", "no", "false", "0", "off": result = false + else: raise newException(ValueError, "cannot interpret as a bool: " & s) + +proc addLine*(self: var string; pieces: varargs[string]) = + for piece in pieces: + self.add piece + self.add "\n" + + +const + inlineErrorKindMarker = "tt." + inlineErrorMarker = "#[" & inlineErrorKindMarker + +proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int = + ## Extract inline error messages. + ## + ## Can parse a single message for a line: + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name] ]# + ## ``` + ## + ## Can parse multiple messages for a line when they are separated by ';': + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error + ## ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error + ## ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` + ## + ## ```nim + ## proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error + ## ^ 'generic_proc' should be: 'genericProc' [Name]; + ## tt.Error ^ 'no_destroy' should be: 'nodestroy' [Name]; + ## tt.Error ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]# + ## ``` + result = i + len(inlineErrorMarker) + inc col, len(inlineErrorMarker) + let msgLine = line + var msgCol = -1 + var msg = "" + var kind = "" + + template parseKind = + while result < s.len and s[result] in IdentChars: + kind.add s[result] + inc result + inc col + if kind notin ["Hint", "Warning", "Error"]: + spec.parseErrors.addLine "expected inline message kind: Hint, Warning, Error" + + template skipWhitespace = + while result < s.len and s[result] in Whitespace: + if s[result] == '\n': + col = 1 + inc line + else: + inc col + inc result + + template parseCaret = + if result < s.len and s[result] == '^': + msgCol = col + inc result + inc col + skipWhitespace() + else: + spec.parseErrors.addLine "expected column marker ('^') for inline message" + + template isMsgDelimiter: bool = + s[result] == ';' and + (block: + let nextTokenIdx = result + 1 + parseutils.skipWhitespace(s, result + 1) + if s.len > nextTokenIdx + len(inlineErrorKindMarker) and + s[nextTokenIdx..(nextTokenIdx + len(inlineErrorKindMarker) - 1)] == inlineErrorKindMarker: + true + else: + false) + + template trimTrailingMsgWhitespace = + while msg.len > 0 and msg[^1] in Whitespace: + setLen msg, msg.len - 1 + + template addInlineError = + doAssert msg[^1] notin Whitespace + if kind == "Error": spec.action = actionReject + spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: msgLine, col: msgCol) + + parseKind() + skipWhitespace() + parseCaret() + + while result < s.len-1: + if s[result] == '\n': + if result > 0 and s[result - 1] == '\r': + msg[^1] = '\n' + else: + msg.add '\n' + inc result + inc line + col = 1 + elif isMsgDelimiter(): + trimTrailingMsgWhitespace() + inc result + skipWhitespace() + addInlineError() + inc result, len(inlineErrorKindMarker) + inc col, 1 + len(inlineErrorKindMarker) + kind.setLen 0 + msg.setLen 0 + parseKind() + skipWhitespace() + parseCaret() + elif s[result] == ']' and s[result+1] == '#': + trimTrailingMsgWhitespace() + inc result, 2 + inc col, 2 + addInlineError() + break + else: + msg.add s[result] + inc result + inc col + + if spec.inlineErrors.len > 0: + spec.unjoinable = true + +proc extractSpec(filename: string; spec: var TSpec): string = + const + tripleQuote = "\"\"\"" + specStart = "discard " & tripleQuote + var s = readFile(filename) + + var i = 0 + var a = -1 + var b = -1 + var line = 1 + var col = 1 + while i < s.len: + if (i == 0 or s[i-1] != ' ') and s.continuesWith(specStart, i): + # `s[i-1] == '\n'` would not work because of `tests/stdlib/tbase64.nim` which contains BOM (https://en.wikipedia.org/wiki/Byte_order_mark) + const lineMax = 10 + if a != -1: + raise newException(ValueError, "testament spec violation: duplicate `specStart` found: " & $(filename, a, b, line)) + elif line > lineMax: + # not overly restrictive, but prevents mistaking some `specStart` as spec if deeep inside a test file + raise newException(ValueError, "testament spec violation: `specStart` should be before line $1, or be indented; info: $2" % [$lineMax, $(filename, a, b, line)]) + i += specStart.len + a = i + elif a > -1 and b == -1 and s.continuesWith(tripleQuote, i): + b = i + i += tripleQuote.len + elif s[i] == '\n': + inc line + inc i + col = 1 + elif s.continuesWith(inlineErrorMarker, i): + i = extractErrorMsg(s, i, line, col, spec) + else: + inc col + inc i + + if a >= 0 and b > a: + result = s.substr(a, b-1).multiReplace({"'''": tripleQuote, "\\31": "\31"}) + elif a >= 0: + raise newException(ValueError, "testament spec violation: `specStart` found but not trailing `tripleQuote`: $1" % $(filename, a, b, line)) + else: + result = "" + +proc parseTargets*(value: string): set[TTarget] = + for v in value.normalize.splitWhitespace: + case v + of "c": result.incl(targetC) + of "cpp", "c++": result.incl(targetCpp) + of "objc": result.incl(targetObjC) + of "js": result.incl(targetJS) + else: raise newException(ValueError, "invalid target: '$#'" % v) + +proc initSpec*(filename: string): TSpec = + result.file = filename + +proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool = + if testamentData.testamentNumBatch != 0: + hash(filename) mod testamentData.testamentNumBatch == testamentData.testamentBatch + else: + true + +proc parseSpec*(filename: string): TSpec = + result.file = filename + result.filename = extractFilename(filename) + let specStr = extractSpec(filename, result) + var ss = newStringStream(specStr) + var p: CfgParser + open(p, ss, filename, 1) + var flags: HashSet[string] + var nimoutFound = false + while true: + var e = next(p) + case e.kind + of cfgKeyValuePair: + let key = e.key.normalize + const whiteListMulti = ["disabled", "ccodecheck"] + ## list of flags that are correctly handled when passed multiple times + ## (instead of being overwritten) + if key notin whiteListMulti: + doAssert key notin flags, $(key, filename) + flags.incl key + case key + of "action": + case e.value.normalize + of "compile": + result.action = actionCompile + of "run": + result.action = actionRun + of "reject": + result.action = actionReject + else: + result.parseErrors.addLine "cannot interpret as action: ", e.value + of "file": + if result.msg.len == 0 and result.nimout.len == 0: + result.parseErrors.addLine "errormsg or msg needs to be specified before file" + result.file = e.value + of "line": + if result.msg.len == 0 and result.nimout.len == 0: + result.parseErrors.addLine "errormsg, msg or nimout needs to be specified before line" + discard parseInt(e.value, result.line) + of "column": + if result.msg.len == 0 and result.nimout.len == 0: + result.parseErrors.addLine "errormsg or msg needs to be specified before column" + discard parseInt(e.value, result.column) + of "output": + if result.outputCheck != ocSubstr: + result.outputCheck = ocEqual + result.output = e.value + of "input": + result.input = e.value + of "outputsub": + result.outputCheck = ocSubstr + result.output = strip(e.value) + of "sortoutput": + try: + result.sortoutput = parseCfgBool(e.value) + except: + result.parseErrors.addLine getCurrentExceptionMsg() + of "exitcode": + discard parseInt(e.value, result.exitCode) + result.action = actionRun + of "errormsg": + result.msg = e.value + result.action = actionReject + of "nimout": + result.nimout = e.value + nimoutFound = true + of "nimoutfull": + result.nimoutFull = parseCfgBool(e.value) + of "batchable": + result.unbatchable = not parseCfgBool(e.value) + of "joinable": + result.unjoinable = not parseCfgBool(e.value) + of "valgrind": + when defined(linux) and sizeof(int) == 8: + result.useValgrind = if e.value.normalize == "leaks": leaking + else: ValgrindSpec(parseCfgBool(e.value)) + result.unjoinable = true + if result.useValgrind != disabled: + result.outputCheck = ocSubstr + else: + # Windows lacks valgrind. Silly OS. + # Valgrind only supports OSX <= 17.x + result.useValgrind = disabled + of "disabled": + let value = e.value.normalize + case value + of "y", "yes", "true", "1", "on": result.err = reDisabled + of "n", "no", "false", "0", "off": discard + # These values are defined in `compiler/options.isDefined` + of "win": + when defined(windows): result.err = reDisabled + of "linux": + when defined(linux): result.err = reDisabled + of "bsd": + when defined(bsd): result.err = reDisabled + of "osx": + when defined(osx): result.err = reDisabled + of "unix", "posix": + when defined(posix): result.err = reDisabled + of "freebsd": + when defined(freebsd): result.err = reDisabled + of "littleendian": + when defined(littleendian): result.err = reDisabled + of "bigendian": + when defined(bigendian): result.err = reDisabled + of "cpu8", "8bit": + when defined(cpu8): result.err = reDisabled + of "cpu16", "16bit": + when defined(cpu16): result.err = reDisabled + of "cpu32", "32bit": + when defined(cpu32): result.err = reDisabled + of "cpu64", "64bit": + when defined(cpu64): result.err = reDisabled + # These values are for CI environments + of "travis": # deprecated + if isTravis: result.err = reDisabled + of "appveyor": # deprecated + if isAppVeyor: result.err = reDisabled + of "azure": + if isAzure: result.err = reDisabled + else: + # Check whether the value exists as an OS or CPU that is + # defined in `compiler/platform`. + block checkHost: + for os in platform.OS: + # Check if the value exists as OS. + if value == os.name.normalize: + # The value exists; is it the same as the current host? + if value == hostOS.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + for cpu in platform.CPU: + # Check if the value exists as CPU. + if value == cpu.name.normalize: + # The value exists; is it the same as the current host? + if value == hostCPU.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + # The value doesn't exist as an OS, CPU, or any previous value + # defined in this case statement, so raise an error. + result.parseErrors.addLine "cannot interpret as a bool: ", e.value + of "cmd": + if e.value.startsWith("nim "): + result.cmd = compilerPrefix & e.value[3..^1] + else: + result.cmd = e.value + of "ccodecheck": + result.ccodeCheck.add e.value + of "maxcodesize": + discard parseInt(e.value, result.maxCodeSize) + of "timeout": + try: + result.timeout = parseFloat(e.value) + except ValueError: + result.parseErrors.addLine "cannot interpret as a float: ", e.value + of "targets", "target": + try: + result.targets.incl parseTargets(e.value) + except ValueError as e: + result.parseErrors.addLine e.msg + of "matrix": + for v in e.value.split(';'): + result.matrix.add(v.strip) + else: + result.parseErrors.addLine "invalid key for test spec: ", e.key + + of cfgSectionStart: + result.parseErrors.addLine "section ignored: ", e.section + of cfgOption: + result.parseErrors.addLine "command ignored: ", e.key & ": " & e.value + of cfgError: + result.parseErrors.addLine e.msg + of cfgEof: + break + close(p) + + if skips.anyIt(it in result.file): + result.err = reDisabled + + if nimoutFound and result.nimout.len == 0 and not result.nimoutFull: + result.parseErrors.addLine "empty `nimout` is vacuously true, use `nimoutFull:true` if intentional" + + result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable + if not result.inCurrentBatch: + result.err = reDisabled + + # Interpolate variables in msgs: + template varSub(msg: string): string = + try: + msg % ["/", $DirSep, "file", result.filename] + except ValueError: + result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unit-tests-output-message-variable-interpolation')" + msg + result.nimout = result.nimout.varSub + result.msg = result.msg.varSub + for inlineError in result.inlineErrors.mitems: + inlineError.msg = inlineError.msg.varSub diff --git a/testament/testament.nim b/testament/testament.nim new file mode 100644 index 000000000..1e892e636 --- /dev/null +++ b/testament/testament.nim @@ -0,0 +1,806 @@ +# +# +# Nim Testament +# (c) Copyright 2017 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This program verifies Nim against the testcases. + +import + std/[strutils, pegs, os, osproc, streams, json, + parseopt, browsers, terminal, exitprocs, + algorithm, times, intsets, macros] + +import backend, specs, azure, htmlgen + +from std/sugar import dup +import compiler/nodejs +import lib/stdtest/testutils +from lib/stdtest/specialpaths import splitTestFile +from std/private/gitutils import diffStrings + +import ../dist/checksums/src/checksums/md5 + +proc trimUnitSep(x: var string) = + let L = x.len + if L > 0 and x[^1] == '\31': + setLen x, L-1 + +var useColors = true +var backendLogging = true +var simulate = false +var optVerbose = false +var useMegatest = true +var valgrindEnabled = true + +proc verboseCmd(cmd: string) = + if optVerbose: + echo "executing: ", cmd + +const + failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs + testsDir = "tests" & DirSep + resultsFile = "testresults.html" + Usage = """Usage: + testament [options] command [arguments] + +Command: + p|pat|pattern <glob> run all the tests matching the given pattern + all run all tests in category folders + c|cat|category <category> run all the tests of a certain category + r|run <test> run single test file + html generate $1 from the database +Arguments: + arguments are passed to the compiler +Options: + --print print results to the console + --verbose print commands (compiling and running tests) + --simulate see what tests would be run but don't run them (for debugging) + --failing only show failing/ignored tests + --targets:"c cpp js objc" run tests for specified targets (default: c) + --nim:path use a particular nim executable (default: $$PATH/nim) + --directory:dir Change to directory dir before reading the tests or doing anything else. + --colors:on|off Turn messages coloring on|off. + --backendLogging:on|off Disable or enable backend logging. By default turned on. + --megatest:on|off Enable or disable megatest. Default is on. + --valgrind:on|off Enable or disable valgrind support. Default is on. + --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored + +On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API +provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN. + +Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables +tests with remote networking (as in CI). +""" % resultsFile + +proc isNimRepoTests(): bool = + # this logic could either be specific to cwd, or to some file derived from + # the input file, eg testament r /pathto/tests/foo/tmain.nim; we choose + # the former since it's simpler and also works with `testament all`. + let file = "testament"/"testament.nim.cfg" + result = file.fileExists + +type + Category = distinct string + TResults = object + total, passed, failedButAllowed, skipped: int + ## xxx rename passed to passedOrAllowedFailure + data: string + TTest = object + name: string + cat: Category + options: string + testArgs: seq[string] + spec: TSpec + startTime: float + debugInfo: string + +# ---------------------------------------------------------------------------- + +let + pegLineError = + peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" + pegOtherError = peg"'Error:' \s* {.*}" + pegOfInterest = pegLineError / pegOtherError + +var gTargets = {low(TTarget)..high(TTarget)} +var targetsSet = false + +proc isSuccess(input: string): bool = + # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs + # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we + # could handle colors, there would be other issues such as handling other flags + # that may appear in user config (eg: `--filenames`). + # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config + # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory + input.startsWith("Hint: ") and input.endsWith("[SuccessX]") + +proc getFileDir(filename: string): string = + result = filename.splitFile().dir + if not result.isAbsolute(): + result = getCurrentDir() / result + +proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: string = ""): tuple[ + cmdLine: string, + output: string, + exitCode: int] {.tags: + [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = + + result.cmdLine.add quoteShell(command) + for arg in args: + result.cmdLine.add ' ' + result.cmdLine.add quoteShell(arg) + verboseCmd(result.cmdLine) + var p = startProcess(command, workingDir = workingDir, args = args, + options = {poStdErrToStdOut, poUsePath}) + var outp = outputStream(p) + + # There is no way to provide input for the child process + # anymore. Closing it will create EOF on stdin instead of eternal + # blocking. + let instream = inputStream(p) + instream.write(input) + close instream + + result.exitCode = -1 + var line = newStringOfCap(120) + while true: + if outp.readLine(line): + result.output.add line + result.output.add '\n' + else: + result.exitCode = peekExitCode(p) + if result.exitCode != -1: break + close(p) + +proc nimcacheDir(filename, options: string, target: TTarget): string = + ## Give each test a private nimcache dir so they don't clobber each other's. + let hashInput = options & $target + result = "nimcache" / (filename & '_' & hashInput.getMD5) + +proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): string = + var options = target.defaultOptions & ' ' & options + if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell) + options.add ' ' & extraOptions + # we avoid using `parseCmdLine` which is buggy, refs bug #14343 + result = cmdTemplate % ["target", targetToCmd[target], + "options", options, "file", filename.quoteShell, + "filedir", filename.getFileDir(), "nim", compilerPrefix] + +proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): TSpec = + result.cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache, target, + extraOptions) + verboseCmd(result.cmd) + var p = startProcess(command = result.cmd, + options = {poStdErrToStdOut, poUsePath, poEvalCommand}) + let outp = p.outputStream + var foundSuccessMsg = false + var foundErrorMsg = false + var err = "" + var x = newStringOfCap(120) + result.nimout = "" + while true: + if outp.readLine(x): + trimUnitSep x + result.nimout.add(x & '\n') + if x =~ pegOfInterest: + # `err` should contain the last error message + err = x + foundErrorMsg = true + elif x.isSuccess: + foundSuccessMsg = true + elif not running(p): + break + result.msg = "" + result.file = "" + result.output = "" + result.line = 0 + result.column = 0 + + result.err = reNimcCrash + result.exitCode = p.peekExitCode + close p + case result.exitCode + of 0: + if foundErrorMsg: + result.debugInfo.add " compiler exit code was 0 but some Error's were found." + else: + result.err = reSuccess + of 1: + if not foundErrorMsg: + result.debugInfo.add " compiler exit code was 1 but no Error's were found." + if foundSuccessMsg: + result.debugInfo.add " compiler exit code was 1 but no `isSuccess` was true." + else: + result.debugInfo.add " expected compiler exit code 0 or 1, got $1." % $result.exitCode + + if err =~ pegLineError: + result.file = extractFilename(matches[0]) + result.line = parseInt(matches[1]) + result.column = parseInt(matches[2]) + result.msg = matches[3] + elif err =~ pegOtherError: + result.msg = matches[0] + trimUnitSep result.msg + +proc initResults: TResults = + result.total = 0 + result.passed = 0 + result.failedButAllowed = 0 + result.skipped = 0 + result.data = "" + +macro ignoreStyleEcho(args: varargs[typed]): untyped = + let typForegroundColor = bindSym"ForegroundColor".getType + let typBackgroundColor = bindSym"BackgroundColor".getType + let typStyle = bindSym"Style".getType + let typTerminalCmd = bindSym"TerminalCmd".getType + result = newCall(bindSym"echo") + for arg in children(args): + if arg.kind == nnkNilLit: continue + let typ = arg.getType + if typ.kind != nnkEnumTy or + typ != typForegroundColor and + typ != typBackgroundColor and + typ != typStyle and + typ != typTerminalCmd: + result.add(arg) + +template maybeStyledEcho(args: varargs[untyped]): untyped = + if useColors: + styledEcho(args) + else: + ignoreStyleEcho(args) + + +proc `$`(x: TResults): string = + result = """ +Tests passed or allowed to fail: $2 / $1 <br /> +Tests failed and allowed to fail: $3 / $1 <br /> +Tests skipped: $4 / $1 <br /> +""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] + +proc testName(test: TTest, target: TTarget, extraOptions: string, allowFailure: bool): string = + var name = test.name.replace(DirSep, '/') + name.add ' ' & $target + if allowFailure: + name.add " (allowed to fail) " + if test.options.len > 0: name.add ' ' & test.options + if extraOptions.len > 0: name.add ' ' & extraOptions + name.strip() + +proc addResult(r: var TResults, test: TTest, target: TTarget, + extraOptions, expected, given: string, successOrig: TResultEnum, + allowFailure = false, givenSpec: ptr TSpec = nil) = + # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need + # instead of having to pass individual fields, or abusing existing ones like expected vs given. + # test.name is easier to find than test.name.extractFilename + # A bit hacky but simple and works with tests/testament/tshould_not_work.nim + let name = testName(test, target, extraOptions, allowFailure) + let duration = epochTime() - test.startTime + let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout + else: successOrig + + let durationStr = duration.formatFloat(ffDecimal, precision = 2).align(5) + if backendLogging: + backend.writeTestResult(name = name, + category = test.cat.string, + target = $target, + action = $test.spec.action, + result = $success, + expected = expected, + given = given) + r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) + template dispNonSkipped(color, outcome) = + maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)" + template disp(msg) = + maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name + if success == reSuccess: + dispNonSkipped(fgGreen, "PASS: ") + elif success == reDisabled: + if test.spec.inCurrentBatch: disp("SKIP:") + else: disp("NOTINBATCH:") + elif success == reJoined: disp("JOINED:") + else: + dispNonSkipped(fgRed, failString) + maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" + maybeStyledEcho styleBright, fgRed, "Failure: ", $success + if givenSpec != nil and givenSpec.debugInfo.len > 0: + echo "debugInfo: " & givenSpec.debugInfo + if success in {reBuildFailed, reNimcCrash, reInstallFailed}: + # expected is empty, no reason to print it. + echo given + else: + maybeStyledEcho fgYellow, "Expected:" + maybeStyledEcho styleBright, expected, "\n" + maybeStyledEcho fgYellow, "Gotten:" + maybeStyledEcho styleBright, given, "\n" + echo diffStrings(expected, given).output + + if backendLogging and (isAppVeyor or isAzure): + let (outcome, msg) = + case success + of reSuccess: + ("Passed", "") + of reDisabled, reJoined: + ("Skipped", "") + of reBuildFailed, reNimcCrash, reInstallFailed: + ("Failed", "Failure: " & $success & '\n' & given) + else: + ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given) + if isAzure: + azure.addTestResult(name, test.cat.string, int(duration * 1000), msg, success) + else: + var p = startProcess("appveyor", args = ["AddTest", test.name.replace("\\", "/") & test.options, + "-Framework", "nim-testament", "-FileName", + test.cat.string, + "-Outcome", outcome, "-ErrorMessage", msg, + "-Duration", $(duration * 1000).int], + options = {poStdErrToStdOut, poUsePath, poParentStreams}) + discard waitForExit(p) + close(p) + +proc toString(inlineError: InlineError, filename: string): string = + result.add "$file($line, $col) $kind: $msg" % [ + "file", filename, + "line", $inlineError.line, + "col", $inlineError.col, + "kind", $inlineError.kind, + "msg", $inlineError.msg + ] + +proc inlineErrorsMsgs(expected: TSpec): string = + for inlineError in expected.inlineErrors.items: + result.addLine inlineError.toString(expected.filename) + +proc checkForInlineErrors(expected, given: TSpec): bool = + for inlineError in expected.inlineErrors: + if inlineError.toString(expected.filename) notin given.nimout: + return false + true + +proc nimoutCheck(expected, given: TSpec): bool = + result = true + if expected.nimoutFull: + if expected.nimout != given.nimout: + result = false + elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): + result = false + +proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, + target: TTarget, extraOptions: string) = + if not checkForInlineErrors(expected, given) or + (not expected.nimoutFull and not nimoutCheck(expected, given)): + r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer) + elif strip(expected.msg) notin strip(given.msg): + r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer) + elif not nimoutCheck(expected, given): + r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer) + elif extractFilename(expected.file) != extractFilename(given.file) and + "internal error:" notin expected.msg: + r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer) + elif expected.line != given.line and expected.line != 0 or + expected.column != given.column and expected.column != 0: + r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column, + $given.line & ':' & $given.column, reLinesDiffer) + else: + r.addResult(test, target, extraOptions, expected.msg, given.msg, reSuccess) + inc(r.passed) + +proc generatedFile(test: TTest, target: TTarget): string = + if target == targetJS: + result = test.name.changeFileExt("js") + else: + let (_, name, _) = test.name.splitFile + let ext = targetToExt[target] + result = nimcacheDir(test.name, test.options, target) / "@m" & name.changeFileExt(ext) + +proc needsCodegenCheck(spec: TSpec): bool = + result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0 + +proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var string, + given: var TSpec) = + try: + let genFile = generatedFile(test, target) + let contents = readFile(genFile) + for check in spec.ccodeCheck: + if check.len > 0 and check[0] == '\\': + # little hack to get 'match' support: + if not contents.match(check.peg): + given.err = reCodegenFailure + elif contents.find(check.peg) < 0: + given.err = reCodegenFailure + expectedMsg = check + if spec.maxCodeSize > 0 and contents.len > spec.maxCodeSize: + given.err = reCodegenFailure + given.msg = "generated code size: " & $contents.len + expectedMsg = "max allowed size: " & $spec.maxCodeSize + except ValueError: + given.err = reInvalidPeg + echo getCurrentExceptionMsg() + except IOError: + given.err = reCodeNotFound + echo getCurrentExceptionMsg() + +proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string, + given: var TSpec, expected: TSpec; r: var TResults) = + var expectedmsg: string = "" + var givenmsg: string = "" + if given.err == reSuccess: + if expected.needsCodegenCheck: + codegenCheck(test, target, expected, expectedmsg, given) + givenmsg = given.msg + if not nimoutCheck(expected, given) or + not checkForInlineErrors(expected, given): + given.err = reMsgsDiffer + expectedmsg = expected.nimout & inlineErrorsMsgs(expected) + givenmsg = given.nimout.strip + else: + givenmsg = "$ " & given.cmd & '\n' & given.nimout + if given.err == reSuccess: inc(r.passed) + r.addResult(test, target, extraOptions, expectedmsg, givenmsg, given.err) + +proc getTestSpecTarget(): TTarget = + if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": + result = targetCpp + else: + result = targetC + +var count = 0 + +proc equalModuloLastNewline(a, b: string): bool = + # allow lazy output spec that omits last newline, but really those should be fixed instead + result = a == b or b.endsWith("\n") and a == b[0 ..< ^1] + +proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, + target: TTarget, extraOptions: string, nimcache: string) = + test.startTime = epochTime() + if testName(test, target, extraOptions, false) in skips: + test.spec.err = reDisabled + + if test.spec.err in {reDisabled, reJoined}: + r.addResult(test, target, extraOptions, "", "", test.spec.err) + inc(r.skipped) + return + var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions) + case expected.action + of actionCompile: + compilerOutputTests(test, target, extraOptions, given, expected, r) + of actionRun: + if given.err != reSuccess: + r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr) + else: + let isJsTarget = target == targetJS + var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt) + if not fileExists(exeFile): + r.addResult(test, target, extraOptions, expected.output, + "executable not found: " & exeFile, reExeNotFound) + else: + let nodejs = if isJsTarget: findNodeJs() else: "" + if isJsTarget and nodejs == "": + r.addResult(test, target, extraOptions, expected.output, "nodejs binary not in PATH", + reExeNotFound) + else: + var exeCmd: string + var args = test.testArgs + if isJsTarget: + exeCmd = nodejs + # see D20210217T215950 + args = @["--unhandled-rejections=strict", exeFile] & args + else: + exeCmd = exeFile.dup(normalizeExe) + if valgrindEnabled and expected.useValgrind != disabled: + var valgrindOptions = @["--error-exitcode=1"] + if expected.useValgrind != leaking: + valgrindOptions.add "--leak-check=yes" + args = valgrindOptions & exeCmd & args + exeCmd = "valgrind" + var (_, buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input) + # Treat all failure codes from nodejs as 1. Older versions of nodejs used + # to return other codes, but for us it is sufficient to know that it's not 0. + if exitCode != 0: exitCode = 1 + let bufB = + if expected.sortoutput: + var buf2 = buf + buf2.stripLineEnd + var x = splitLines(buf2) + sort(x, system.cmp) + join(x, "\n") & '\n' + else: + buf + if exitCode != expected.exitCode: + given.err = reExitcodesDiffer + r.addResult(test, target, extraOptions, "exitcode: " & $expected.exitCode, + "exitcode: " & $exitCode & "\n\nOutput:\n" & + bufB, reExitcodesDiffer) + elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or + (expected.outputCheck == ocSubstr and expected.output notin bufB): + given.err = reOutputsDiffer + r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer) + compilerOutputTests(test, target, extraOptions, given, expected, r) + of actionReject: + # Make sure its the compiler rejecting and not the system (e.g. segfault) + cmpMsgs(r, expected, given, test, target, extraOptions) + if given.exitCode != QuitFailure: + r.addResult(test, target, extraOptions, "exitcode: " & $QuitFailure, + "exitcode: " & $given.exitCode & "\n\nOutput:\n" & + given.nimout, reExitcodesDiffer) + + + +proc changeTarget(extraOptions: string; defaultTarget: TTarget): TTarget = + result = defaultTarget + var p = parseopt.initOptParser(extraOptions) + + while true: + parseopt.next(p) + case p.kind + of cmdEnd: break + of cmdLongOption, cmdShortOption: + if p.key == "b" or p.key == "backend": + result = parseEnum[TTarget](p.val.normalize) + # chooses the last one + else: + discard + +proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) = + for target in expected.targets: + inc(r.total) + if target notin gTargets: + r.addResult(test, target, extraOptions, "", "", reDisabled) + inc(r.skipped) + elif simulate: + inc count + echo "testSpec count: ", count, " expected: ", expected + else: + let nimcache = nimcacheDir(test.name, test.options, target) + var testClone = test + let target = changeTarget(extraOptions, target) + testSpecHelper(r, testClone, expected, target, extraOptions, nimcache) + +proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = + var expected = test.spec + if expected.parseErrors.len > 0: + # targetC is a lie, but a parameter is required + r.addResult(test, targetC, "", "", expected.parseErrors, reInvalidSpec) + inc(r.total) + return + + expected.targets.incl targets + # still no target specified at all + if expected.targets == {}: + expected.targets = {getTestSpecTarget()} + if test.spec.matrix.len > 0: + for m in test.spec.matrix: + targetHelper(r, test, expected, m) + else: + targetHelper(r, test, expected, "") + +proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} = + for target in test.spec.targets: + inc(r.total) + var testClone = test + testSpecHelper(r, testClone, test.spec, target, "", nimcache) + +proc makeTest(test, options: string, cat: Category): TTest = + result.cat = cat + result.name = test + result.options = options + result.spec = parseSpec(addFileExt(test, ".nim")) + result.startTime = epochTime() + +proc makeRawTest(test, options: string, cat: Category): TTest {.used.} = + result.cat = cat + result.name = test + result.options = options + result.spec = initSpec(addFileExt(test, ".nim")) + result.spec.action = actionCompile + result.spec.targets = {getTestSpecTarget()} + result.startTime = epochTime() + +# TODO: fix these files +const disabledFilesDefault = @[ + "tableimpl.nim", + "setimpl.nim", + "hashcommon.nim", + + # Requires compiling with '--threads:on` + "sharedlist.nim", + "sharedtables.nim", + + # Error: undeclared identifier: 'hasThreadSupport' + "ioselectors_epoll.nim", + "ioselectors_kqueue.nim", + "ioselectors_poll.nim", + + # Error: undeclared identifier: 'Timeval' + "ioselectors_select.nim", +] + +when defined(windows): + const + # array of modules disabled from compilation test of stdlib. + disabledFiles = disabledFilesDefault & @["coro.nim"] +else: + const + # array of modules disabled from compilation test of stdlib. + disabledFiles = disabledFilesDefault + +include categories + +proc loadSkipFrom(name: string): seq[string] = + if name.len == 0: return + # One skip per line, comments start with # + # used by `nlvm` (at least) + for line in lines(name): + let sline = line.strip() + if sline.len > 0 and not sline.startsWith('#'): + result.add sline + +proc main() = + azure.init() + backend.open() + var optPrintResults = false + var optFailing = false + var targetsStr = "" + var isMainProcess = true + var skipFrom = "" + + var p = initOptParser() + p.next() + while p.kind in {cmdLongOption, cmdShortOption}: + case p.key.normalize + of "print": optPrintResults = true + of "verbose": optVerbose = true + of "failing": optFailing = true + of "pedantic": discard # deadcode refs https://github.com/nim-lang/Nim/issues/16731 + of "targets": + targetsStr = p.val + gTargets = parseTargets(targetsStr) + targetsSet = true + of "nim": + compilerPrefix = addFileExt(p.val.absolutePath, ExeExt) + of "directory": + setCurrentDir(p.val) + of "colors": + case p.val: + of "on": + useColors = true + of "off": + useColors = false + else: + quit Usage + of "batch": + testamentData0.batchArg = p.val + if p.val != "_" and p.val.len > 0 and p.val[0] in {'0'..'9'}: + let s = p.val.split("_") + doAssert s.len == 2, $(p.val, s) + testamentData0.testamentBatch = s[0].parseInt + testamentData0.testamentNumBatch = s[1].parseInt + doAssert testamentData0.testamentNumBatch > 0 + doAssert testamentData0.testamentBatch >= 0 and testamentData0.testamentBatch < testamentData0.testamentNumBatch + of "simulate": + simulate = true + of "megatest": + case p.val: + of "on": + useMegatest = true + of "off": + useMegatest = false + else: + quit Usage + of "valgrind": + case p.val: + of "on": + valgrindEnabled = true + of "off": + valgrindEnabled = false + else: + quit Usage + of "backendlogging": + case p.val: + of "on": + backendLogging = true + of "off": + backendLogging = false + else: + quit Usage + of "skipfrom": + skipFrom = p.val + else: + quit Usage + p.next() + if p.kind != cmdArgument: + quit Usage + var action = p.key.normalize + p.next() + var r = initResults() + case action + of "all": + #processCategory(r, Category"megatest", p.cmdLineRest, testsDir, runJoinableTests = false) + + var myself = quoteShell(getAppFilename()) + if targetsStr.len > 0: + myself &= " " & quoteShell("--targets:" & targetsStr) + + myself &= " " & quoteShell("--nim:" & compilerPrefix) + if testamentData0.batchArg.len > 0: + myself &= " --batch:" & testamentData0.batchArg + + if skipFrom.len > 0: + myself &= " " & quoteShell("--skipFrom:" & skipFrom) + + var cats: seq[string] + let rest = if p.cmdLineRest.len > 0: " " & p.cmdLineRest else: "" + for kind, dir in walkDir(testsDir): + assert testsDir.startsWith(testsDir) + let cat = dir[testsDir.len .. ^1] + if kind == pcDir and cat notin ["testdata", "nimcache"]: + cats.add cat + if isNimRepoTests(): + cats.add AdditionalCategories + if useMegatest: cats.add MegaTestCat + + var cmds: seq[string] + for cat in cats: + let runtype = if useMegatest: " pcat " else: " cat " + cmds.add(myself & runtype & quoteShell(cat) & rest) + + proc progressStatus(idx: int) = + echo "progress[all]: $1/$2 starting: cat: $3" % [$idx, $cats.len, cats[idx]] + + if simulate: + skips = loadSkipFrom(skipFrom) + for i, cati in cats: + progressStatus(i) + processCategory(r, Category(cati), p.cmdLineRest, testsDir, runJoinableTests = false) + else: + addExitProc azure.finalize + quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus) + of "c", "cat", "category": + skips = loadSkipFrom(skipFrom) + var cat = Category(p.key) + processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = true) + of "pcat": + skips = loadSkipFrom(skipFrom) + # 'pcat' is used for running a category in parallel. Currently the only + # difference is that we don't want to run joinable tests here as they + # are covered by the 'megatest' category. + isMainProcess = false + var cat = Category(p.key) + p.next + processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = false) + of "p", "pat", "pattern": + skips = loadSkipFrom(skipFrom) + let pattern = p.key + p.next + processPattern(r, pattern, p.cmdLineRest, simulate) + of "r", "run": + let (cat, path) = splitTestFile(p.key) + processSingleTest(r, cat.Category, p.cmdLineRest, path, gTargets, targetsSet) + of "html": + generateHtml(resultsFile, optFailing) + else: + quit Usage + + if optPrintResults: + if action == "html": openDefaultBrowser(resultsFile) + else: echo r, r.data + azure.finalize() + backend.close() + var failed = r.total - r.passed - r.skipped + if failed != 0: + echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ", + r.skipped, " failed: ", failed + quit(QuitFailure) + if isMainProcess: + echo "Used ", compilerPrefix, " to run the tests. Use --nim to override." + +if paramCount() == 0: + quit Usage +main() diff --git a/testament/testament.nim.cfg b/testament/testament.nim.cfg new file mode 100644 index 000000000..c97284129 --- /dev/null +++ b/testament/testament.nim.cfg @@ -0,0 +1,6 @@ +# don't move this file without updating the logic in `isNimRepoTests` + +path = "$nim" # For compiler/nodejs +-d:ssl # For azure +# my SSL doesn't have this feature and I don't care: +-d:nimDisableCertificateValidation diff --git a/testament/testamenthtml.nimf b/testament/testamenthtml.nimf new file mode 100644 index 000000000..9190f370e --- /dev/null +++ b/testament/testamenthtml.nimf @@ -0,0 +1,297 @@ +#? stdtmpl(subsChar = '%', metaChar = '#', emit = "outfile.write") +#import strutils +# +#proc htmlQuote*(raw: string): string = +# result = raw.multiReplace( +# ("&", "&"), +# ("\"", """), +# ("'", "'"), +# ("<", "<"), +# (">", ">") +# ) +# +#end proc +#proc generateHtmlBegin*(outfile: File) = +<!DOCTYPE html> +<html> +<head> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <title>Testament Test Results</title> + <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.0/jquery.min.js" integrity="sha256-ihAoc6M/JPfrIiIeayPE9xjin4UWjsx2mjW/rtmxLM4=" crossorigin="anonymous"></script> + <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous"></script> + <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha256-916EbMg70RQy9LHiGkXzG8hSg9EdNy97GazNG/aiY1w=" crossorigin="anonymous" /> + <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap-theme.min.css" integrity="sha256-ZT4HPpdCOt2lvDkXokHuhJfdOKSPFLzeAJik5U/Q+l4=" crossorigin="anonymous" /> + <script> + /** + * Callback function that is executed for each Element in an array. + * @callback executeForElement + * @param {Element} elem Element to operate on + */ + + /** + * + * @param {number} index + * @param {Element[]} elemArray + * @param {executeForElement} executeOnItem + */ + function executeAllAsync(elemArray, index, executeOnItem) { + for (var i = 0; index < elemArray.length && i < 100; i++ , index++) { + var item = elemArray[index]; + executeOnItem(item); + } + if (index < elemArray.length) { + setTimeout(executeAllAsync, 0, elemArray, index, executeOnItem); + } + } + + /** @param {Element} elem */ + function executeShowOnElement(elem) { + while (elem.classList.contains("hidden")) { + elem.classList.remove("hidden"); + } + } + + /** @param {Element} elem */ + function executeHideOnElement(elem) { + if (!elem.classList.contains("hidden")) { + elem.classList.add("hidden"); + } + } + + /** @param {Element} elem */ + function executeExpandOnElement(elem) { + $(elem).collapse("show"); + } + + /** @param {Element} elem */ + function executeCollapseOnElement(elem) { + $(elem).collapse("hide"); + } + + /** + * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) + * @param {executeForElement} executeOnEachPanel + */ + function wholePanelAll(category, executeOnEachPanel) { + var selector = "div.panel"; + if (typeof category === "string" && category) { + selector += "-" + category; + } + + var jqPanels = $(selector); + /** @type {Element[]} */ + var elemArray = jqPanels.toArray(); + + setTimeout(executeAllAsync, 0, elemArray, 0, executeOnEachPanel); + } + + /** + * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) + * @param {executeForElement} executeOnEachPanel + */ + function panelBodyAll(category, executeOnEachPanelBody) { + var selector = "div.panel"; + if (typeof category === "string" && category) { + selector += "-" + category; + } + + var jqPanels = $(selector); + + var jqPanelBodies = $("div.panel-body", jqPanels); + /** @type {Element[]} */ + var elemArray = jqPanelBodies.toArray(); + + setTimeout(executeAllAsync, 0, elemArray, 0, executeOnEachPanelBody); + } + + /** + * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) + */ + function showAll(category) { + wholePanelAll(category, executeShowOnElement); + } + + /** + * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) + */ + function hideAll(category) { + wholePanelAll(category, executeHideOnElement); + } + + /** + * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) + */ + function expandAll(category) { + panelBodyAll(category, executeExpandOnElement); + } + + /** + * @param {string} [category] Optional bootstrap panel context class (danger, warning, info, success) + */ + function collapseAll(category) { + panelBodyAll(category, executeCollapseOnElement); + } + </script> +</head> +<body> + <div class="container"> + <h1>Testament Test Results <small>Nim Tester</small></h1> +#end proc +#proc generateHtmlAllTestsBegin*(outfile: File, machine, commit, branch: string, +# totalCount: BiggestInt, +# successCount: BiggestInt, successPercentage: string, +# ignoredCount: BiggestInt, ignoredPercentage: string, +# failedCount: BiggestInt, failedPercentage: string, onlyFailing = false) = + <dl class="dl-horizontal"> + <dt>Hostname</dt> + <dd>%machine</dd> + <dt>Git Commit</dt> + <dd><code>%commit</code></dd> + <dt title="Git Branch reference">Branch ref.</dt> + <dd>%branch</dd> + </dl> + <dl class="dl-horizontal"> + <dt>All Tests</dt> + <dd> + <span class="glyphicon glyphicon-th-list"></span> + %totalCount + </dd> + <dt>Successful Tests</dt> + <dd> + <span class="glyphicon glyphicon-ok-sign"></span> + %successCount (%successPercentage) + </dd> + <dt>Skipped Tests</dt> + <dd> + <span class="glyphicon glyphicon-question-sign"></span> + %ignoredCount (%ignoredPercentage) + </dd> + <dt>Failed Tests</dt> + <dd> + <span class="glyphicon glyphicon-exclamation-sign"></span> + %failedCount (%failedPercentage) + </dd> + </dl> + <div class="table-responsive"> + <table class="table table-condensed"> +# if not onlyFailing: + <tr> + <th class="text-right" style="vertical-align:middle">All Tests</th> + <td> + <div class="btn-group"> + <button class="btn btn-default" type="button" onclick="showAll();">Show All</button> + <button class="btn btn-default" type="button" onclick="hideAll();">Hide All</button> + <button class="btn btn-default" type="button" onclick="expandAll();">Expand All</button> + <button class="btn btn-default" type="button" onclick="collapseAll();">Collapse All</button> + </div> + </td> + </tr> + <tr> + <th class="text-right" style="vertical-align:middle">Successful Tests</th> + <td> + <div class="btn-group"> + <button class="btn btn-default" type="button" onclick="showAll('success');">Show All</button> + <button class="btn btn-default" type="button" onclick="hideAll('success');">Hide All</button> + <button class="btn btn-default" type="button" onclick="expandAll('success');">Expand All</button> + <button class="btn btn-default" type="button" onclick="collapseAll('success');">Collapse All</button> + </div> + </td> + </tr> +# end if + <tr> + <th class="text-right" style="vertical-align:middle">Skipped Tests</th> + <td> + <div class="btn-group"> + <button class="btn btn-default" type="button" onclick="showAll('info');">Show All</button> + <button class="btn btn-default" type="button" onclick="hideAll('info');">Hide All</button> + <button class="btn btn-default" type="button" onclick="expandAll('info');">Expand All</button> + <button class="btn btn-default" type="button" onclick="collapseAll('info');">Collapse All</button> + </div> + </td> + </tr> + <tr> + <th class="text-right" style="vertical-align:middle">Failed Tests</th> + <td> + <div class="btn-group"> + <button class="btn btn-default" type="button" onclick="showAll('danger');">Show All</button> + <button class="btn btn-default" type="button" onclick="hideAll('danger');">Hide All</button> + <button class="btn btn-default" type="button" onclick="expandAll('danger');">Expand All</button> + <button class="btn btn-default" type="button" onclick="collapseAll('danger');">Collapse All</button> + </div> + </td> + </tr> + </table> + </div> + <div class="panel-group"> +#end proc +#proc generateHtmlTestresultPanelBegin*(outfile: File, trId, name, target, category, +# action, resultDescription, timestamp, result, resultSign, +# panelCtxClass, textCtxClass, bgCtxClass: string) = + <div id="panel-testResult-%trId" class="panel panel-%panelCtxClass"> + <div class="panel-heading" style="cursor:pointer" data-toggle="collapse" data-target="#panel-body-testResult-%trId" aria-controls="panel-body-testResult-%trId" aria-expanded="false"> + <div class="row"> + <h4 class="col-xs-3 col-sm-1 panel-title"> + <span class="glyphicon glyphicon-%resultSign-sign"></span> + <strong>%resultDescription</strong> + </h4> + <h4 class="col-xs-1 panel-title"><span class="badge">%target</span></h4> + <h4 class="col-xs-5 col-sm-7 panel-title" title="%name"><code class="text-%textCtxClass">%name</code></h4> + <h4 class="col-xs-3 col-sm-3 panel-title text-right"><span class="badge">%category</span></h4> + </div> + </div> + <div id="panel-body-testResult-%trId" class="panel-body collapse bg-%bgCtxClass"> + <dl class="dl-horizontal"> + <dt>Name</dt> + <dd><code class="text-%textCtxClass">%name</code></dd> + <dt>Category</dt> + <dd><span class="badge">%category</span></dd> + <dt>Timestamp</dt> + <dd>%timestamp</dd> + <dt>Nim Action</dt> + <dd><code class="text-%textCtxClass">%action</code></dd> + <dt>Nim Backend Target</dt> + <dd><span class="badge">%target</span></dd> + <dt>Code</dt> + <dd><code class="text-%textCtxClass">%result</code></dd> + </dl> +#end proc +#proc generateHtmlTestresultOutputDetails*(outfile: File, expected, gotten: string) = + <div class="table-responsive"> + <table class="table table-condensed"> + <thead> + <tr> + <th>Expected</th> + <th>Actual</th> + </tr> + </thead> + <tbody> + <tr> + <td><pre>%expected</pre></td> + <td><pre>%gotten</pre></td> + </tr> + </tbody> + </table> + </div> +#end proc +#proc generateHtmlTestresultOutputNone*(outfile: File) = + <p class="sr-only">No output details</p> +#end proc +#proc generateHtmlTestresultPanelEnd*(outfile: File) = + </div> + </div> +#end proc +#proc generateHtmlAllTestsEnd*(outfile: File) = + </div> +#end proc +#proc generateHtmlEnd*(outfile: File, timestamp: string) = + <hr /> + <footer> + <p> + Report generated by: <code>testament</code> – Nim Tester + <br /> + Made with Nim. Generated on: %timestamp + </p> + </footer> + </div> +</body> +</html> diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim new file mode 100644 index 000000000..477da1e23 --- /dev/null +++ b/testament/tests/shouldfail/tccodecheck.nim @@ -0,0 +1,8 @@ +discard """ + ccodecheck: "baz" +""" + +proc foo(): void {.exportc: "bar".}= + echo "Hello World" + +foo() diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim new file mode 100644 index 000000000..809ddec74 --- /dev/null +++ b/testament/tests/shouldfail/tcolumn.nim @@ -0,0 +1,8 @@ +discard """ + errormsg: "undeclared identifier: 'undeclared'" + line: 9 + column: 7 +""" + +# test should fail because the line directive is wrong +echo undeclared diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim new file mode 100644 index 000000000..6c130d107 --- /dev/null +++ b/testament/tests/shouldfail/terrormsg.nim @@ -0,0 +1,8 @@ +discard """ + errormsg: "wrong error message" + line: 9 + column: 6 +""" + +# test should fail because the line directive is wrong +echo undeclared diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim new file mode 100644 index 000000000..605f046db --- /dev/null +++ b/testament/tests/shouldfail/texitcode1.nim @@ -0,0 +1,3 @@ +discard """ + exitcode: 1 +""" diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim new file mode 100644 index 000000000..b40a4f44f --- /dev/null +++ b/testament/tests/shouldfail/tfile.nim @@ -0,0 +1,6 @@ +discard """ + errormsg: "undeclared identifier: 'undefined'" + file: "notthisfile.nim" +""" + +echo undefined diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim new file mode 100644 index 000000000..fe782eb03 --- /dev/null +++ b/testament/tests/shouldfail/tline.nim @@ -0,0 +1,8 @@ +discard """ + errormsg: "undeclared identifier: 'undeclared'" + line: 10 + column: 6 +""" + +# test should fail because the line directive is wrong +echo undeclared diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim new file mode 100644 index 000000000..92022ee97 --- /dev/null +++ b/testament/tests/shouldfail/tmaxcodesize.nim @@ -0,0 +1,5 @@ +discard """ + maxcodesize: 1 +""" + +echo "Hello World" diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim new file mode 100644 index 000000000..0a65bfb70 --- /dev/null +++ b/testament/tests/shouldfail/tnimout.nim @@ -0,0 +1,7 @@ +discard """ + nimout: "Hello World!" + action: compile +""" + +static: + echo "something else" diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim new file mode 100644 index 000000000..4fc93f6d2 --- /dev/null +++ b/testament/tests/shouldfail/tnimoutfull.nim @@ -0,0 +1,14 @@ +discard """ + nimout: ''' +msg1 +msg2 +''' + action: compile + nimoutFull: true +""" + +# should fail because `msg3` is not in nimout and `nimoutFill: true` was given +static: + echo "msg1" + echo "msg2" + echo "msg3" diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim new file mode 100644 index 000000000..eaf9e8652 --- /dev/null +++ b/testament/tests/shouldfail/toutput.nim @@ -0,0 +1,7 @@ +discard """ + output: ''' + done + ''' +""" + +echo "broken" diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim new file mode 100644 index 000000000..47324ecee --- /dev/null +++ b/testament/tests/shouldfail/toutputsub.nim @@ -0,0 +1,5 @@ +discard """ + outputsub: "something else" +""" + +echo "Hello World!" diff --git a/testament/tests/shouldfail/treject.nim b/testament/tests/shouldfail/treject.nim new file mode 100644 index 000000000..1e7258f70 --- /dev/null +++ b/testament/tests/shouldfail/treject.nim @@ -0,0 +1,7 @@ +discard """ + action: "reject" +""" + +# Because we set action="reject", we expect this line not to compile. But the +# line does compile, therefore the test fails. +assert true diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim new file mode 100644 index 000000000..69dfbc0a0 --- /dev/null +++ b/testament/tests/shouldfail/tsortoutput.nim @@ -0,0 +1,11 @@ +discard """ + sortoutput: true + output: ''' +2 +1 +''' +""" + +# this test should ensure that the output is actually sorted +echo "2" +echo "1" diff --git a/testament/tests/shouldfail/ttimeout.nim b/testament/tests/shouldfail/ttimeout.nim new file mode 100644 index 000000000..fd3e1a598 --- /dev/null +++ b/testament/tests/shouldfail/ttimeout.nim @@ -0,0 +1,7 @@ +discard """ + timeout: "0.1" +""" + +import os + +os.sleep(1000) diff --git a/testament/tests/shouldfail/tvalgrind.nim b/testament/tests/shouldfail/tvalgrind.nim new file mode 100644 index 000000000..d551ff12e --- /dev/null +++ b/testament/tests/shouldfail/tvalgrind.nim @@ -0,0 +1,17 @@ +discard """ + valgrind: true + cmd: "nim $target --gc:arc -d:useMalloc $options $file" +""" + +# this is the same check used by testament/specs.nim whether or not valgrind +# tests are supported +when defined(linux) and sizeof(int) == 8: + # discarding this allocation will cause valgrind to fail (which is what we + # want), but valgrind only runs on 64-bit Linux machines... + discard alloc(1) +else: + # ...so on all other OS/architectures, simulate any non-zero exit code to + # mimic how valgrind would have failed on this test. We cannot use things like + # `disabled: "freebsd"` in the Testament configs above or else the tests will + # be SKIP-ed rather than FAIL-ed + quit(1) # choose 1 to match valgrind's `--error-exit=1`, but could be anything |