unit separator (#17730)

* use the ASCII Unit Separator so that error messages can be handled precisely by the tooling
* updated testament

8 files changed, 55 insertions, 39 deletions

diff --git a/compiler/main.nim b/compiler/main.nim
index 6eb830164..d66a5f329 100644
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -335,8 +335,8 @@ proc mainCommand*(graph: ModuleGraph) =
       msgWriteln(conf, $dumpdata, {msgStdout, msgSkipHook})
     else:
       msgWriteln(conf, "-- list of currently defined symbols --",
-                 {msgStdout, msgSkipHook})
-      for s in definedSymbolNames(conf.symbols): msgWriteln(conf, s, {msgStdout, msgSkipHook})
+                 {msgStdout, msgSkipHook, msgNoUnitSep})
+      for s in definedSymbolNames(conf.symbols): msgWriteln(conf, s, {msgStdout, msgSkipHook, msgNoUnitSep})
       msgWriteln(conf, "-- end of list --", {msgStdout, msgSkipHook})
 
       for it in conf.searchPaths: msgWriteln(conf, it.string)
diff --git a/compiler/msgs.nim b/compiler/msgs.nim
index 39b10d7df..30421fe6a 100644
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -241,12 +241,13 @@ template toFullPath*(conf: ConfigRef; info: TLineInfo): string =
 template toFullPathConsiderDirty*(conf: ConfigRef; info: TLineInfo): string =
   string toFullPathConsiderDirty(conf, info.fileIndex)
 
-type FilenameOption* = enum
-  foAbs # absolute path, e.g.: /pathto/bar/foo.nim
-  foRelProject # relative to project path, e.g.: ../foo.nim
-  foMagicSauce # magic sauce, shortest of (foAbs, foRelProject)
-  foName # lastPathPart, e.g.: foo.nim
-  foStacktrace # if optExcessiveStackTrace: foAbs else: foName
+type
+  FilenameOption* = enum
+    foAbs # absolute path, e.g.: /pathto/bar/foo.nim
+    foRelProject # relative to project path, e.g.: ../foo.nim
+    foMagicSauce # magic sauce, shortest of (foAbs, foRelProject)
+    foName # lastPathPart, e.g.: foo.nim
+    foStacktrace # if optExcessiveStackTrace: foAbs else: foName
 
 proc toFilenameOption*(conf: ConfigRef, fileIdx: FileIndex, opt: FilenameOption): string =
   case opt
@@ -295,10 +296,14 @@ proc `??`* (conf: ConfigRef; info: TLineInfo, filename: string): bool =
   # only for debugging purposes
   result = filename in toFilename(conf, info)
 
+const
+  UnitSep = "\31"
+
 type
   MsgFlag* = enum  ## flags altering msgWriteln behavior
     msgStdout,     ## force writing to stdout, even stderr is default
     msgSkipHook    ## skip message hook even if it is present
+    msgNoUnitSep  ## the message is a complete "paragraph".
   MsgFlags* = set[MsgFlag]
 
 proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) =
@@ -310,17 +315,20 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) =
   ## This is used for 'nim dump' etc. where we don't have nimsuggest
   ## support.
   #if conf.cmd == cmdIdeTools and optCDebug notin gGlobalOptions: return
+  let sep = if msgNoUnitSep notin flags: UnitSep else: ""
   if not isNil(conf.writelnHook) and msgSkipHook notin flags:
-    conf.writelnHook(s)
+    conf.writelnHook(s & sep)
   elif optStdout in conf.globalOptions or msgStdout in flags:
     if eStdOut in conf.m.errorOutputs:
       flushDot(conf)
-      writeLine(stdout, s)
+      write stdout, s
+      writeLine(stdout, sep)
       flushFile(stdout)
   else:
     if eStdErr in conf.m.errorOutputs:
       flushDot(conf)
-      writeLine(stderr, s)
+      write stderr, s
+      writeLine(stderr, sep)
       # On Windows stderr is fully-buffered when piped, regardless of C std.
       when defined(windows):
         flushFile(stderr)
@@ -366,7 +374,7 @@ proc msgWrite(conf: ConfigRef; s: string) =
     flushFile(stdOrr)
     conf.lastMsgWasDot.incl stdOrr.toStdOrrKind() # subsequent writes need `flushDot`
 
-template styledMsgWriteln*(args: varargs[typed]) =
+template styledMsgWriteln(args: varargs[typed]) =
   if not isNil(conf.writelnHook):
     callIgnoringStyle(callWritelnHook, nil, args)
   elif optStdout in conf.globalOptions:
@@ -407,7 +415,7 @@ proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} =
         styledMsgWriteln(fgRed, """
 No stack traceback available
 To create a stacktrace, rerun compilation with './koch temp $1 <file>', see $2 for details""" %
-          [conf.command, "intern.html#debugging-the-compiler".createDocLink])
+          [conf.command, "intern.html#debugging-the-compiler".createDocLink], UnitSep)
   quit 1
 
 proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) =
@@ -444,10 +452,11 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
         conf.structuredErrorHook(conf, context.info, instantiationFrom,
                                  Severity.Hint)
       else:
-        let message = if context.detail == "":
-          instantiationFrom
-        else:
-          instantiationOfFrom.format(context.detail)
+        let message =
+          if context.detail == "":
+            instantiationFrom
+          else:
+            instantiationOfFrom.format(context.detail)
         styledMsgWriteln(styleBright, conf.toFileLineCol(context.info), " ", resetStyle, message)
     info = context.info
 
@@ -479,11 +488,12 @@ proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
 
   result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1]
 
-proc writeSurroundingSrc(conf: ConfigRef; info: TLineInfo) =
-  const indent = "  "
-  msgWriteln(conf, indent & $sourceLine(conf, info))
-  if info.col >= 0:
-    msgWriteln(conf, indent & spaces(info.col) & '^')
+proc getSurroundingSrc(conf: ConfigRef; info: TLineInfo): string =
+  if conf.hasHint(hintSource) and info != unknownLineInfo:
+    const indent = "  "
+    result = "\n" & indent & $sourceLine(conf, info)
+    if info.col >= 0:
+      result.add "\n" & indent & spaces(info.col) & '^'
 
 proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): string =
   let title = case msg
@@ -545,14 +555,13 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
       if msg == hintProcessing:
         msgWrite(conf, ".")
       else:
-        styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg)
-        if conf.hasHint(hintSource) and info != unknownLineInfo:
-          conf.writeSurroundingSrc(info)
+        styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg,
+                         resetStyle, conf.getSurroundingSrc(info), UnitSep)
         if hintMsgOrigin in conf.mainPackageNotes:
           styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle,
             " compiler msg initiated here", KindColor,
             KindFormat % $hintMsgOrigin,
-            resetStyle)
+            resetStyle, UnitSep)
   handleError(conf, msg, eh, s)
 
 template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
@@ -632,8 +641,8 @@ proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
     result = conf.m.fileInfos[i.fileIndex.int32].quotedName
 
 template listMsg(title, r) =
-  msgWriteln(conf, title)
-  for a in r: msgWriteln(conf, "  [$1] $2" % [if a in conf.notes: "x" else: " ", $a])
+  msgWriteln(conf, title, {msgNoUnitSep})
+  for a in r: msgWriteln(conf, "  [$1] $2" % [if a in conf.notes: "x" else: " ", $a], {msgNoUnitSep})
 
 proc listWarnings*(conf: ConfigRef) = listMsg("Warnings:", warnMin..warnMax)
 proc listHints*(conf: ConfigRef) = listMsg("Hints:", hintMin..hintMax)
diff --git a/compiler/vm.nim b/compiler/vm.nim
index d588067c7..7a231a482 100644
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -37,7 +37,7 @@ proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) =
       while x != nil:
         inc calls
         x = x.next
-      msgWriteln(c.config, $calls & " calls omitted\n")
+      msgWriteln(c.config, $calls & " calls omitted\n", {msgNoUnitSep})
       return
     stackTraceAux(c, x.next, x.comesFrom, recursionLimit-1)
     var info = c.debug[pc]
@@ -59,12 +59,12 @@ proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) =
     if x.prc != nil:
       for k in 1..max(1, 25-s.len): s.add(' ')
       s.add(x.prc.name.s)
-    msgWriteln(c.config, s)
+    msgWriteln(c.config, s, {msgNoUnitSep})
 
 proc stackTraceImpl(c: PCtx, tos: PStackFrame, pc: int,
   msg: string, lineInfo: TLineInfo, infoOrigin: InstantiationInfo) {.noinline.} =
   # noinline to avoid code bloat
-  msgWriteln(c.config, "stack trace: (most recent call last)")
+  msgWriteln(c.config, "stack trace: (most recent call last)", {msgNoUnitSep})
   stackTraceAux(c, tos, pc)
   let action = if c.mode == emRepl: doRaise else: doNothing
     # XXX test if we want 'globalError' for every mode
@@ -470,7 +470,7 @@ template handleJmpBack() {.dirty.} =
     if allowInfiniteLoops in c.features:
       c.loopIterations = c.config.maxLoopIterationsVM
     else:
-      msgWriteln(c.config, "stack trace: (most recent call last)")
+      msgWriteln(c.config, "stack trace: (most recent call last)", {msgNoUnitSep})
       stackTraceAux(c, tos, pc)
       globalError(c.config, c.debug[pc], errTooManyIterations % $c.config.maxLoopIterationsVM)
   dec(c.loopIterations)
@@ -1163,7 +1163,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         stackTrace(c, tos, pc, "node is not a proc symbol")
     of opcEcho:
       let rb = instr.regB
-      template fn(s) = msgWriteln(c.config, s, {msgStdout})
+      template fn(s) = msgWriteln(c.config, s, {msgStdout, msgNoUnitSep})
       if rb == 1: fn(regs[ra].node.strVal)
       else:
         var outp = ""
diff --git a/testament/important_packages.nim b/testament/important_packages.nim
index 28d7dc367..ecae42811 100644
--- a/testament/important_packages.nim
+++ b/testament/important_packages.nim
@@ -74,7 +74,7 @@ pkg "gnuplot", "nim c gnuplot.nim"
 pkg "hts", "nim c -o:htss src/hts.nim"
 pkg "httpauth"
 pkg "illwill", "nimble examples"
-pkg "inim"
+pkg "inim", allowFailure=true
 pkg "itertools", "nim doc src/itertools.nim"
 pkg "iterutils"
 pkg "karax", "nim c -r tests/tester.nim"
diff --git a/testament/specs.nim b/testament/specs.nim
index b5e6de7c2..043f14ed3 100644
--- a/testament/specs.nim
+++ b/testament/specs.nim
@@ -202,7 +202,7 @@ proc extractSpec(filename: string; spec: var TSpec): string =
 
   # look for """ only in the first section
   if a >= 0 and b > a and a < 40:
-    result = s.substr(a+3, b-1).replace("'''", tripleQuote)
+    result = s.substr(a+3, b-1).multiReplace({"'''": tripleQuote, "\\31": "\31"})
   else:
     #echo "warning: file does not contain spec: " & filename
     result = ""
diff --git a/testament/testament.nim b/testament/testament.nim
index f2a8e9055..b79b86e51 100644
--- a/testament/testament.nim
+++ b/testament/testament.nim
@@ -18,6 +18,11 @@ import compiler/nodejs
 import lib/stdtest/testutils
 from lib/stdtest/specialpaths import splitTestFile
 
+proc trimUnitSep(x: var string) =
+  let L = x.len
+  if L > 0 and x[^1] == '\31':
+    setLen x, L-1
+
 var useColors = true
 var backendLogging = true
 var simulate = false
@@ -172,6 +177,7 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string,
   result.nimout = ""
   while true:
     if outp.readLine(x):
+      trimUnitSep x
       result.nimout.add(x & '\n')
       if x =~ pegOfInterest:
         # `err` should contain the last error/warning message
@@ -196,6 +202,7 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string,
     result.msg = matches[0]
   elif suc.isSuccess:
     result.err = reSuccess
+  trimUnitSep result.msg
 
 proc callCCompiler(cmdTemplate, filename, options: string,
                   target: TTarget): TSpec =
diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim
index 1895eeb97..df9ee9a48 100644
--- a/tests/misc/trunner.nim
+++ b/tests/misc/trunner.nim
@@ -235,7 +235,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines
     var expected = ""
     for a in files:
       let b = dir / a
-      expected.add &"Hint: used config file '{b}' [Conf]\n"
+      expected.add &"Hint: used config file '{b}' [Conf]\31\n"
     doAssert outp.endsWith expected, outp & "\n" & expected
 
   block: # mfoo2.customext
@@ -243,7 +243,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines
     let cmd = fmt"{nim} e --hint:conf {filename}"
     let (outp, exitCode) = execCmdEx(cmd, options = {poStdErrToStdOut})
     doAssert exitCode == 0
-    var expected = &"Hint: used config file '{filename}' [Conf]\n"
+    var expected = &"Hint: used config file '{filename}' [Conf]\31\n"
     doAssert outp.endsWith "123" & "\n" & expected
 
 
diff --git a/tests/osproc/treadlines.nim b/tests/osproc/treadlines.nim
index 3a8303321..436dd7a10 100644
--- a/tests/osproc/treadlines.nim
+++ b/tests/osproc/treadlines.nim
@@ -1,6 +1,6 @@
 discard """
-  output: '''Error: cannot open 'a.nim'
-Error: cannot open 'b.nim'
+  output: '''Error: cannot open 'a.nim'\31
+Error: cannot open 'b.nim'\31
 '''
   targets: "c"
 """