fix #8871 runnableExamples now preserves source code comments, litterals, and all formatting; other bug fix (#14439)

* fix #8871 runnableExamples now preserves source code comments, litterals, and all formatting
* remove orig deadcode from getAllRunnableExamplesImpl
* fix expected examples
* add test to close https://github.com/nim-lang/Nim/issues/14473
* correctly handle regular comments before 1st token inside runnableExamples
* add test to answer https://github.com/nim-lang/Nim/pull/14439#discussion_r431829199
* update tests

4 files changed, 205 insertions, 57 deletions

diff --git a/compiler/ast.nim b/compiler/ast.nim
index 97aa3a992..d13487e9c 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -1030,6 +1030,7 @@ const
   nkFloatLiterals* = {nkFloatLit..nkFloat128Lit}
   nkLambdaKinds* = {nkLambda, nkDo}
   declarativeDefs* = {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef}
+  routineDefs* = declarativeDefs + {nkMacroDef, nkTemplateDef}
   procDefs* = nkLambdaKinds + declarativeDefs
 
   nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice}
diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index d1e2d42c5..f2dec41b3 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -17,7 +17,7 @@ import
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, cgi, trees, types,
   typesrenderer, astalgo, lineinfos, intsets,
-  pathutils, trees, tables, nimpaths
+  pathutils, trees, tables, nimpaths, renderverbatim
 
 const
   exportSection = skField
@@ -315,7 +315,11 @@ proc genComment(d: PDoc, n: PNode): string =
   result = ""
   var dummyHasToc: bool
   if n.comment.len > 0:
-    renderRstToOut(d[], parseRst(n.comment, toFullPath(d.conf, n.info),
+    var comment2 = n.comment
+    when false:
+      # RFC: to preseve newlines in comments, this would work:
+      comment2 = comment2.replace("\n", "\n\n")
+    renderRstToOut(d[], parseRst(comment2, toFullPath(d.conf, n.info),
                                toLinenumber(n.info), toColumn(n.info),
                                dummyHasToc, d.options, d.conf), result)
 
@@ -494,8 +498,8 @@ proc runAllExamples(d: PDoc) =
       rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
       # removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove
 
-proc prepareExample(d: PDoc; n: PNode): string =
-  ## returns `rdoccmd` for this runnableExamples
+proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
+  ## returns `rdoccmd` and source code for this runnableExamples
   var rdoccmd = ""
   if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid")
   if n.len == 3:
@@ -512,10 +516,11 @@ proc prepareExample(d: PDoc; n: PNode): string =
       docComment,
       newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
   runnableExamples.info = n.info
-
+  let ret = extractRunnableExamplesSource(d.conf, n)
   for a in n.lastSon: runnableExamples.add a
+  # we could also use `ret` instead here, to keep sources verbatim
   writeExample(d, runnableExamples, rdoccmd)
-  result = rdoccmd
+  result = (rdoccmd, ret)
   when false:
     proc extractImports(n: PNode; result: PNode) =
       if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
@@ -529,39 +534,52 @@ proc prepareExample(d: PDoc; n: PNode): string =
     for imp in imports: runnableExamples.add imp
     runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon)
 
-proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previousIsRunnable: var bool) =
+proc renderNimCodeOld(d: PDoc, n: PNode, dest: var Rope) =
+  ## this is a rather hacky way to get rid of the initial indentation
+  ## that the renderer currently produces:
+  # deadcode
+  var i = 0
+  var body = n.lastSon
+  if body.len == 1 and body.kind == nkStmtList and
+      body.lastSon.kind == nkStmtList:
+    body = body.lastSon
+  for b in body:
+    if i > 0: dest.add "\n"
+    inc i
+    nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)
+
+type RunnableState = enum
+  rsStart
+  rsComment
+  rsRunnable
+  rsDone
+
+proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: RunnableState): RunnableState =
   ##[
-  previousIsRunnable: keep track of whether previous sibling was a runnableExample (true if 1st sibling though).
-  This is to ensure this works:
+  Simple state machine to tell whether we render runnableExamples and doc comments.
+  This is to ensure that we can interleave runnableExamples and doc comments freely;
+  the logic is easy to change but currently a doc comment following another doc comment
+  will not render, to avoid rendering in following case:
+
   proc fn* =
     runnableExamples: discard
     ## d1
     runnableExamples: discard
     ## d2
 
-    ## d3 # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
-
-  It also works with:
-  proc fn* =
-    ## d0
-    runnableExamples: discard
-    ## d1
-
-    etc
+    ## internal explanation  # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
+    discard # some code
   ]##
-  # xxx: checkme: owner check instead? this fails with the $nim/nimdoc/tester.nim test
-  # now that we're calling `genRecComment` only from here (to maintain correct order wrt runnableExample)
-  # if n.info.fileIndex != orig.info.fileIndex: return
+
   case n.kind
   of nkCommentStmt:
-    if previousIsRunnable:
+    if state in {rsStart, rsRunnable}:
       dest.add genRecComment(d, n)
-    previousIsRunnable = false
+      return rsComment
   of nkCallKinds:
     if isRunnableExamples(n[0]) and
-        n.len >= 2 and n.lastSon.kind == nkStmtList:
-      previousIsRunnable = true
-      let rdoccmd = prepareExample(d, n)
+        n.len >= 2 and n.lastSon.kind == nkStmtList and state in {rsStart, rsComment, rsRunnable}:
+      let (rdoccmd, code) = prepareExample(d, n)
       var msg = "Example:"
       if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
       dispA(d.conf, dest, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
@@ -569,27 +587,71 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previous
       inc d.listingCounter
       let id = $d.listingCounter
       dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"])
-      # this is a rather hacky way to get rid of the initial indentation
-      # that the renderer currently produces:
-      var i = 0
-      var body = n.lastSon
-      if body.len == 1 and body.kind == nkStmtList and
-          body.lastSon.kind == nkStmtList:
-        body = body.lastSon
-      for b in body:
-        if i > 0: dest.add "\n"
-        inc i
-        nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)
+      when true:
+        var dest2 = ""
+        renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
+        dest.add dest2
+      else:
+        renderNimCodeOld(d, n, dest)
       dest.add(d.config.getOrDefault"doc.listing_end" % id)
-  else: previousIsRunnable = false
+      return rsRunnable
+  else: discard
+  return rsDone
+    # change this to `rsStart` if you want to keep generating doc comments
+    # and runnableExamples that occur after some code in routine
 
-  var previousIsRunnable2 = true
-  for i in 0..<n.safeLen:
-    getAllRunnableExamplesRec(d, n[i], orig, dest, previousIsRunnable2)
+proc getRoutineBody(n: PNode): PNode =
+  ##[
+  nim transforms these quite differently:
+
+  proc someType*(): int =
+    ## foo
+    result = 3
+=>
+  result =
+    ## foo
+    3;
+
+  proc someType*(): int =
+    ## foo
+    3
+=>
+  ## foo
+  result = 3;
+
+  so we normalize the results to get to the statement list containing the
+  (0 or more) doc comments and runnableExamples.
+  (even if using `result = n[bodyPos]`, you'd still to apply similar logic).
+  ]##
+  result = n[^1]
+  case result.kind
+  of nkSym:
+    result = n[^2]
+    case result.kind
+      of nkAsgn:
+        doAssert result[0].kind == nkSym
+        doAssert result.len == 2
+        result = result[1]
+      else: # eg: nkStmtList
+        discard
+  else:
+    discard
 
-proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
-  var previousIsRunnable = true
-  getAllRunnableExamplesRec(d, n, n, dest, previousIsRunnable)
+proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var Rope) =
+  var n = n
+  var state = rsStart
+  template fn(n2) =
+    state = getAllRunnableExamplesImpl(d, n2, dest, state)
+  case n.kind
+  of routineDefs:
+    n = n.getRoutineBody
+    case n.kind
+    of nkCommentStmt, nkCallKinds: fn(n)
+    else:
+      for i in 0..<n.safeLen:
+        fn(n[i])
+        if state == rsDone: return
+  else: fn(n)
 
 proc isVisible(d: PDoc; n: PNode): bool =
   result = false
@@ -765,12 +827,11 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   var literal, plainName = ""
   var kind = tkEof
   var comm: Rope = nil
-  # skipping this (and doing it inside getAllRunnableExamples) would fix order in
-  # case of a runnableExample appearing before a doccomment, but would cause other
-  # issues
-  comm.add genRecComment(d, n)
-  if n.kind in declarativeDefs:
+  if n.kind in routineDefs:
     getAllRunnableExamples(d, n, comm)
+  else:
+    comm.add genRecComment(d, n)
+
   var r: TSrcGen
   # Obtain the plain rendered string for hyperlink titles.
   initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
diff --git a/compiler/msgs.nim b/compiler/msgs.nim
index 1b7be46b8..0e64d9327 100644
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -436,18 +436,25 @@ proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
 proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
   conf.m.fileInfos[fileIdx.int32].lines.add line
 
-proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
-  if i.fileIndex.int32 < 0: return ""
-
-  if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
+proc numLines*(conf: ConfigRef, fileIdx: FileIndex): int =
+  ## xxx there's an off by 1 error that should be fixed; if a file ends with "foo" or "foo\n"
+  ## it will return same number of lines (ie, a trailing empty line is discounted)
+  result = conf.m.fileInfos[fileIdx.int32].lines.len
+  if result == 0:
     try:
-      for line in lines(toFullPathConsiderDirty(conf, i)):
-        addSourceLine conf, i.fileIndex, line.string
+      for line in lines(toFullPathConsiderDirty(conf, fileIdx).string):
+        addSourceLine conf, fileIdx, line.string
     except IOError:
       discard
-  assert i.fileIndex.int32 < conf.m.fileInfos.len
+    result = conf.m.fileInfos[fileIdx.int32].lines.len
+
+proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
+  ## 1-based index (matches editor line numbers); 1st line is for i.line = 1
+  ## last valid line is `numLines` inclusive
+  if i.fileIndex.int32 < 0: return ""
+  let num = numLines(conf, i.fileIndex)
   # can happen if the error points to EOF:
-  if i.line.int > conf.m.fileInfos[i.fileIndex.int32].lines.len: return ""
+  if i.line.int > num: return ""
 
   result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1]
 
diff --git a/compiler/renderverbatim.nim b/compiler/renderverbatim.nim
new file mode 100644
index 000000000..a78433163
--- /dev/null
+++ b/compiler/renderverbatim.nim
@@ -0,0 +1,79 @@
+import strutils
+from xmltree import addEscaped
+
+import ast, options, msgs
+import packages/docutils/highlite
+
+# import compiler/renderer
+import renderer
+
+proc lastNodeRec(n: PNode): PNode =
+  result = n
+  while result.safeLen > 0: result = result[^1]
+
+proc isInIndentationBlock(src: string, indent: int): bool =
+  #[
+  we stop at the first de-indentation; there's an inherent ambiguity with non
+  doc comments since they can have arbitrary indentation, so we just take the
+  practical route and require a runnableExamples to keep its code (including non
+  doc comments) to its indentation level.
+  ]#
+  for j in 0..<indent:
+    if src.len <= j: return true
+    if src[j] != ' ': return false
+  return true
+
+proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
+  ## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
+  ## we'd need to check performance impact to enable it for nimdoc.
+  var first = n.lastSon.info
+  if first.line == n[0].info.line:
+    #[
+    runnableExamples: assert true
+    ]#
+    discard
+  else:
+    #[
+    runnableExamples:
+      # non-doc comment that we want to capture even though `first` points to `assert true`
+      assert true
+    ]#
+    first.line = n[0].info.line + 1
+    # first.col = n[0].info.col + 1 # anything with `col > n[0].col` is part of runnableExamples
+
+  let last = n.lastNodeRec.info
+  var info = first
+  var indent = info.col
+  let numLines = numLines(conf, info.fileIndex).uint16
+  var lastNonemptyPos = 0
+  for line in first.line..numLines: # bugfix, see `testNimDocTrailingExample`
+    info.line = line
+    let src = sourceLine(conf, info)
+    if line > last.line and not isInIndentationBlock(src, indent):
+      break
+    if line > first.line: result.add "\n"
+    if src.len > indent:
+      result.add src[indent..^1]
+      lastNonemptyPos = result.len
+  result.setLen lastNonemptyPos
+
+proc renderNimCode*(result: var string, code: string, isLatex = false) =
+  var toknizr: GeneralTokenizer
+  initGeneralTokenizer(toknizr, code)
+  var buf = ""
+  template append(kind, val) =
+    buf.setLen 0
+    buf.addEscaped(val)
+    let class = tokenClassToStr[kind]
+    if isLatex:
+      result.addf "\\span$1{$2}" % [class, buf]
+    else:
+      result.addf  "<span class=\"$1\">$2</span>" % [class, buf]
+
+  while true:
+    getNextToken(toknizr, langNim)
+    case toknizr.kind
+    of gtEof: break  # End Of File (or string)
+    else:
+      # TODO: avoid alloc; maybe toOpenArray
+      append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))