fix #8871 runnableExamples now preserves source code comments, litterals, and all formatting; other bug fix (#14439)

* fix #8871 runnableExamples now preserves source code comments, litterals, and all formatting
* remove orig deadcode from getAllRunnableExamplesImpl
* fix expected examples
* add test to close https://github.com/nim-lang/Nim/issues/14473
* correctly handle regular comments before 1st token inside runnableExamples
* add test to answer https://github.com/nim-lang/Nim/pull/14439#discussion_r431829199
* update tests

12 files changed, 927 insertions, 76 deletions

diff --git a/compiler/ast.nim b/compiler/ast.nim
index 97aa3a992..d13487e9c 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -1030,6 +1030,7 @@ const
   nkFloatLiterals* = {nkFloatLit..nkFloat128Lit}
   nkLambdaKinds* = {nkLambda, nkDo}
   declarativeDefs* = {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef}
+  routineDefs* = declarativeDefs + {nkMacroDef, nkTemplateDef}
   procDefs* = nkLambdaKinds + declarativeDefs
 
   nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice}
diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index d1e2d42c5..f2dec41b3 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -17,7 +17,7 @@ import
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, cgi, trees, types,
   typesrenderer, astalgo, lineinfos, intsets,
-  pathutils, trees, tables, nimpaths
+  pathutils, trees, tables, nimpaths, renderverbatim
 
 const
   exportSection = skField
@@ -315,7 +315,11 @@ proc genComment(d: PDoc, n: PNode): string =
   result = ""
   var dummyHasToc: bool
   if n.comment.len > 0:
-    renderRstToOut(d[], parseRst(n.comment, toFullPath(d.conf, n.info),
+    var comment2 = n.comment
+    when false:
+      # RFC: to preseve newlines in comments, this would work:
+      comment2 = comment2.replace("\n", "\n\n")
+    renderRstToOut(d[], parseRst(comment2, toFullPath(d.conf, n.info),
                                toLinenumber(n.info), toColumn(n.info),
                                dummyHasToc, d.options, d.conf), result)
 
@@ -494,8 +498,8 @@ proc runAllExamples(d: PDoc) =
       rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
       # removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove
 
-proc prepareExample(d: PDoc; n: PNode): string =
-  ## returns `rdoccmd` for this runnableExamples
+proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
+  ## returns `rdoccmd` and source code for this runnableExamples
   var rdoccmd = ""
   if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid")
   if n.len == 3:
@@ -512,10 +516,11 @@ proc prepareExample(d: PDoc; n: PNode): string =
       docComment,
       newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
   runnableExamples.info = n.info
-
+  let ret = extractRunnableExamplesSource(d.conf, n)
   for a in n.lastSon: runnableExamples.add a
+  # we could also use `ret` instead here, to keep sources verbatim
   writeExample(d, runnableExamples, rdoccmd)
-  result = rdoccmd
+  result = (rdoccmd, ret)
   when false:
     proc extractImports(n: PNode; result: PNode) =
       if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
@@ -529,39 +534,52 @@ proc prepareExample(d: PDoc; n: PNode): string =
     for imp in imports: runnableExamples.add imp
     runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon)
 
-proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previousIsRunnable: var bool) =
+proc renderNimCodeOld(d: PDoc, n: PNode, dest: var Rope) =
+  ## this is a rather hacky way to get rid of the initial indentation
+  ## that the renderer currently produces:
+  # deadcode
+  var i = 0
+  var body = n.lastSon
+  if body.len == 1 and body.kind == nkStmtList and
+      body.lastSon.kind == nkStmtList:
+    body = body.lastSon
+  for b in body:
+    if i > 0: dest.add "\n"
+    inc i
+    nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)
+
+type RunnableState = enum
+  rsStart
+  rsComment
+  rsRunnable
+  rsDone
+
+proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: RunnableState): RunnableState =
   ##[
-  previousIsRunnable: keep track of whether previous sibling was a runnableExample (true if 1st sibling though).
-  This is to ensure this works:
+  Simple state machine to tell whether we render runnableExamples and doc comments.
+  This is to ensure that we can interleave runnableExamples and doc comments freely;
+  the logic is easy to change but currently a doc comment following another doc comment
+  will not render, to avoid rendering in following case:
+
   proc fn* =
     runnableExamples: discard
     ## d1
     runnableExamples: discard
     ## d2
 
-    ## d3 # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
-
-  It also works with:
-  proc fn* =
-    ## d0
-    runnableExamples: discard
-    ## d1
-
-    etc
+    ## internal explanation  # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
+    discard # some code
   ]##
-  # xxx: checkme: owner check instead? this fails with the $nim/nimdoc/tester.nim test
-  # now that we're calling `genRecComment` only from here (to maintain correct order wrt runnableExample)
-  # if n.info.fileIndex != orig.info.fileIndex: return
+
   case n.kind
   of nkCommentStmt:
-    if previousIsRunnable:
+    if state in {rsStart, rsRunnable}:
       dest.add genRecComment(d, n)
-    previousIsRunnable = false
+      return rsComment
   of nkCallKinds:
     if isRunnableExamples(n[0]) and
-        n.len >= 2 and n.lastSon.kind == nkStmtList:
-      previousIsRunnable = true
-      let rdoccmd = prepareExample(d, n)
+        n.len >= 2 and n.lastSon.kind == nkStmtList and state in {rsStart, rsComment, rsRunnable}:
+      let (rdoccmd, code) = prepareExample(d, n)
       var msg = "Example:"
       if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
       dispA(d.conf, dest, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
@@ -569,27 +587,71 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previous
       inc d.listingCounter
       let id = $d.listingCounter
       dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"])
-      # this is a rather hacky way to get rid of the initial indentation
-      # that the renderer currently produces:
-      var i = 0
-      var body = n.lastSon
-      if body.len == 1 and body.kind == nkStmtList and
-          body.lastSon.kind == nkStmtList:
-        body = body.lastSon
-      for b in body:
-        if i > 0: dest.add "\n"
-        inc i
-        nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)
+      when true:
+        var dest2 = ""
+        renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
+        dest.add dest2
+      else:
+        renderNimCodeOld(d, n, dest)
       dest.add(d.config.getOrDefault"doc.listing_end" % id)
-  else: previousIsRunnable = false
+      return rsRunnable
+  else: discard
+  return rsDone
+    # change this to `rsStart` if you want to keep generating doc comments
+    # and runnableExamples that occur after some code in routine
 
-  var previousIsRunnable2 = true
-  for i in 0..<n.safeLen:
-    getAllRunnableExamplesRec(d, n[i], orig, dest, previousIsRunnable2)
+proc getRoutineBody(n: PNode): PNode =
+  ##[
+  nim transforms these quite differently:
+
+  proc someType*(): int =
+    ## foo
+    result = 3
+=>
+  result =
+    ## foo
+    3;
+
+  proc someType*(): int =
+    ## foo
+    3
+=>
+  ## foo
+  result = 3;
+
+  so we normalize the results to get to the statement list containing the
+  (0 or more) doc comments and runnableExamples.
+  (even if using `result = n[bodyPos]`, you'd still to apply similar logic).
+  ]##
+  result = n[^1]
+  case result.kind
+  of nkSym:
+    result = n[^2]
+    case result.kind
+      of nkAsgn:
+        doAssert result[0].kind == nkSym
+        doAssert result.len == 2
+        result = result[1]
+      else: # eg: nkStmtList
+        discard
+  else:
+    discard
 
-proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
-  var previousIsRunnable = true
-  getAllRunnableExamplesRec(d, n, n, dest, previousIsRunnable)
+proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var Rope) =
+  var n = n
+  var state = rsStart
+  template fn(n2) =
+    state = getAllRunnableExamplesImpl(d, n2, dest, state)
+  case n.kind
+  of routineDefs:
+    n = n.getRoutineBody
+    case n.kind
+    of nkCommentStmt, nkCallKinds: fn(n)
+    else:
+      for i in 0..<n.safeLen:
+        fn(n[i])
+        if state == rsDone: return
+  else: fn(n)
 
 proc isVisible(d: PDoc; n: PNode): bool =
   result = false
@@ -765,12 +827,11 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   var literal, plainName = ""
   var kind = tkEof
   var comm: Rope = nil
-  # skipping this (and doing it inside getAllRunnableExamples) would fix order in
-  # case of a runnableExample appearing before a doccomment, but would cause other
-  # issues
-  comm.add genRecComment(d, n)
-  if n.kind in declarativeDefs:
+  if n.kind in routineDefs:
     getAllRunnableExamples(d, n, comm)
+  else:
+    comm.add genRecComment(d, n)
+
   var r: TSrcGen
   # Obtain the plain rendered string for hyperlink titles.
   initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
diff --git a/compiler/msgs.nim b/compiler/msgs.nim
index 1b7be46b8..0e64d9327 100644
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -436,18 +436,25 @@ proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
 proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
   conf.m.fileInfos[fileIdx.int32].lines.add line
 
-proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
-  if i.fileIndex.int32 < 0: return ""
-
-  if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
+proc numLines*(conf: ConfigRef, fileIdx: FileIndex): int =
+  ## xxx there's an off by 1 error that should be fixed; if a file ends with "foo" or "foo\n"
+  ## it will return same number of lines (ie, a trailing empty line is discounted)
+  result = conf.m.fileInfos[fileIdx.int32].lines.len
+  if result == 0:
     try:
-      for line in lines(toFullPathConsiderDirty(conf, i)):
-        addSourceLine conf, i.fileIndex, line.string
+      for line in lines(toFullPathConsiderDirty(conf, fileIdx).string):
+        addSourceLine conf, fileIdx, line.string
     except IOError:
       discard
-  assert i.fileIndex.int32 < conf.m.fileInfos.len
+    result = conf.m.fileInfos[fileIdx.int32].lines.len
+
+proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
+  ## 1-based index (matches editor line numbers); 1st line is for i.line = 1
+  ## last valid line is `numLines` inclusive
+  if i.fileIndex.int32 < 0: return ""
+  let num = numLines(conf, i.fileIndex)
   # can happen if the error points to EOF:
-  if i.line.int > conf.m.fileInfos[i.fileIndex.int32].lines.len: return ""
+  if i.line.int > num: return ""
 
   result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1]
 
diff --git a/compiler/renderverbatim.nim b/compiler/renderverbatim.nim
new file mode 100644
index 000000000..a78433163
--- /dev/null
+++ b/compiler/renderverbatim.nim
@@ -0,0 +1,79 @@
+import strutils
+from xmltree import addEscaped
+
+import ast, options, msgs
+import packages/docutils/highlite
+
+# import compiler/renderer
+import renderer
+
+proc lastNodeRec(n: PNode): PNode =
+  result = n
+  while result.safeLen > 0: result = result[^1]
+
+proc isInIndentationBlock(src: string, indent: int): bool =
+  #[
+  we stop at the first de-indentation; there's an inherent ambiguity with non
+  doc comments since they can have arbitrary indentation, so we just take the
+  practical route and require a runnableExamples to keep its code (including non
+  doc comments) to its indentation level.
+  ]#
+  for j in 0..<indent:
+    if src.len <= j: return true
+    if src[j] != ' ': return false
+  return true
+
+proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
+  ## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
+  ## we'd need to check performance impact to enable it for nimdoc.
+  var first = n.lastSon.info
+  if first.line == n[0].info.line:
+    #[
+    runnableExamples: assert true
+    ]#
+    discard
+  else:
+    #[
+    runnableExamples:
+      # non-doc comment that we want to capture even though `first` points to `assert true`
+      assert true
+    ]#
+    first.line = n[0].info.line + 1
+    # first.col = n[0].info.col + 1 # anything with `col > n[0].col` is part of runnableExamples
+
+  let last = n.lastNodeRec.info
+  var info = first
+  var indent = info.col
+  let numLines = numLines(conf, info.fileIndex).uint16
+  var lastNonemptyPos = 0
+  for line in first.line..numLines: # bugfix, see `testNimDocTrailingExample`
+    info.line = line
+    let src = sourceLine(conf, info)
+    if line > last.line and not isInIndentationBlock(src, indent):
+      break
+    if line > first.line: result.add "\n"
+    if src.len > indent:
+      result.add src[indent..^1]
+      lastNonemptyPos = result.len
+  result.setLen lastNonemptyPos
+
+proc renderNimCode*(result: var string, code: string, isLatex = false) =
+  var toknizr: GeneralTokenizer
+  initGeneralTokenizer(toknizr, code)
+  var buf = ""
+  template append(kind, val) =
+    buf.setLen 0
+    buf.addEscaped(val)
+    let class = tokenClassToStr[kind]
+    if isLatex:
+      result.addf "\\span$1{$2}" % [class, buf]
+    else:
+      result.addf  "<span class=\"$1\">$2</span>" % [class, buf]
+
+  while true:
+    getNextToken(toknizr, langNim)
+    case toknizr.kind
+    of gtEof: break  # End Of File (or string)
+    else:
+      # TODO: avoid alloc; maybe toOpenArray
+      append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))
diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim
index 58272a9b6..15dd32ec7 100644
--- a/nimdoc/tester.nim
+++ b/nimdoc/tester.nim
@@ -1,5 +1,6 @@
 # Small program that runs the test cases for 'nim doc'.
 # To run this, cd to the git repo root, and run "nim c -r nimdoc/tester.nim".
+# to change expected results (after carefully verifying everything), use -d:fixup
 
 import strutils, os
 
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
index 469dde0ae..6e83f718f 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -129,6 +129,8 @@ function main() {
     title="aEnum(): untyped"><wbr />a<wbr />Enum<span class="attachedType"></span></a></li>
   <li><a class="reference" href="#bEnum.t"
     title="bEnum(): untyped"><wbr />b<wbr />Enum<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#fromUtilsGen.t"
+    title="fromUtilsGen(): untyped"><wbr />from<wbr />Utils<wbr />Gen<span class="attachedType"></span></a></li>
 
   </ul>
 </li>
@@ -194,6 +196,15 @@ constructor.
 
 
 </dd>
+<a id="fromUtilsGen.t"></a>
+<dt><pre><span class="Keyword">template</span> <a href="#fromUtilsGen.t"><span class="Identifier">fromUtilsGen</span></a><span class="Other">(</span><span class="Other">)</span><span class="Other">:</span> <span class="Identifier">untyped</span></pre></dt>
+<dd>
+
+this should be shown in utils.html
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Identifier">assert</span><span class="Whitespace"> </span><span class="DecNumber">3</span><span class="Operator">*</span><span class="DecNumber">2</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">6</span></pre>ditto
+
+</dd>
 
 </dl></div>
 
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx
index c848fc26a..b49a777c8 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx
@@ -10,3 +10,4 @@ SomeType	subdir/subdir_b/utils.html#SomeType	utils: SomeType
 someType	subdir/subdir_b/utils.html#someType_2	utils: someType(): SomeType	
 aEnum	subdir/subdir_b/utils.html#aEnum.t	utils: aEnum(): untyped	
 bEnum	subdir/subdir_b/utils.html#bEnum.t	utils: bEnum(): untyped	
+fromUtilsGen	subdir/subdir_b/utils.html#fromUtilsGen.t	utils: fromUtilsGen(): untyped	
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index aa29c0c0d..420b60924 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -115,6 +115,9 @@ function main() {
   <li><a class="reference" href="#B"
     title="B {.inject.} = enum
   bB"><wbr />B<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#Foo"
+    title="Foo = enum
+  enumValueA2"><wbr />Foo<span class="attachedType"></span></a></li>
 
   </ul>
 </li>
@@ -149,8 +152,42 @@ function main() {
     title="baz[T](a, b: T): T"><wbr />baz<span class="attachedType"></span></a></li>
   <li><a class="reference" href="#buzz%2CT%2CT"
     title="buzz[T](a, b: T): T"><wbr />buzz<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#fromUtils3"
+    title="fromUtils3()"><wbr />from<wbr />Utils3<span class="attachedType"></span></a></li>
   <li><a class="reference" href="#isValid%2CT"
     title="isValid[T](x: T): bool"><wbr />is<wbr />Valid<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z1"
+    title="z1(): Foo"><wbr />z1<span class="attachedType">Foo</span></a></li>
+  <li><a class="reference" href="#z2"
+    title="z2()"><wbr />z2<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z3"
+    title="z3()"><wbr />z3<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z4"
+    title="z4()"><wbr />z4<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z5"
+    title="z5(): int"><wbr />z5<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z6"
+    title="z6(): int"><wbr />z6<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z7"
+    title="z7(): int"><wbr />z7<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z8"
+    title="z8(): int"><wbr />z8<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z9"
+    title="z9()"><wbr />z9<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z10"
+    title="z10()"><wbr />z10<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z11"
+    title="z11()"><wbr />z11<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z12"
+    title="z12(): int"><wbr />z12<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z13"
+    title="z13()"><wbr />z13<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#baz"
+    title="baz()"><wbr />baz<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z17"
+    title="z17()"><wbr />z17<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#p1"
+    title="p1()"><wbr />p1<span class="attachedType"></span></a></li>
 
   </ul>
 </li>
@@ -163,18 +200,42 @@ function main() {
   </ul>
 </li>
 <li>
+  <a class="reference reference-toplevel" href="#15" id="65">Iterators</a>
+  <ul class="simple simple-toc-section">
+      <li><a class="reference" href="#fromUtils1.i"
+    title="fromUtils1(): int"><wbr />from<wbr />Utils1<span class="attachedType"></span></a></li>
+
+  </ul>
+</li>
+<li>
   <a class="reference reference-toplevel" href="#17" id="67">Macros</a>
   <ul class="simple simple-toc-section">
       <li><a class="reference" href="#bar.m"
     title="bar(): untyped"><wbr />bar<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z16.m"
+    title="z16()"><wbr />z16<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z18.m"
+    title="z18(): int"><wbr />z18<span class="attachedType"></span></a></li>
 
   </ul>
 </li>
 <li>
   <a class="reference reference-toplevel" href="#18" id="68">Templates</a>
   <ul class="simple simple-toc-section">
-      <li><a class="reference" href="#foo.t%2CSomeType%2CSomeType"
+      <li><a class="reference" href="#fromUtils2.t"
+    title="fromUtils2()"><wbr />from<wbr />Utils2<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z6t.t"
+    title="z6t(): int"><wbr />z6t<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#foo.t%2CSomeType%2CSomeType"
     title="foo(a, b: SomeType)"><wbr />foo<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#myfn.t"
+    title="myfn()"><wbr />myfn<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z14.t"
+    title="z14()"><wbr />z14<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#z15.t"
+    title="z15()"><wbr />z15<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#testNimDocTrailingExample.t"
+    title="testNimDocTrailingExample()"><wbr />test<wbr />Nim<wbr />Doc<wbr />Trailing<wbr />Example<span class="attachedType"></span></a></li>
 
   </ul>
 </li>
@@ -187,13 +248,17 @@ function main() {
   
   <p class="module-desc">This is the top level module.
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">import</span>
-  <span class="Identifier">subdir</span> <span class="Operator">/</span> <span class="Identifier">subdir_b</span> <span class="Operator">/</span> <span class="Identifier">utils</span>
-
-<span class="Identifier">doAssert</span> <span class="Identifier">bar</span><span class="Other">(</span><span class="DecNumber">3</span><span class="Other">,</span> <span class="DecNumber">4</span><span class="Other">)</span> <span class="Operator">==</span> <span class="DecNumber">7</span>
-<span class="Identifier">foo</span><span class="Other">(</span><span class="Identifier">enumValueA</span><span class="Other">,</span> <span class="Identifier">enumValueB</span><span class="Other">)</span>
-<span class="Keyword">for</span> <span class="Identifier">x</span> <span class="Keyword">in</span> <span class="StringLit">&quot;xx&quot;</span><span class="Other">:</span>
-  <span class="Keyword">discard</span></pre></p>
+<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">subdir</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">subdir_b</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">utils</span><span class="Whitespace">
+</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="Identifier">bar</span><span class="Punctuation">(</span><span class="DecNumber">3</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="DecNumber">4</span><span class="Punctuation">)</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">7</span><span class="Whitespace">
+</span><span class="Identifier">foo</span><span class="Punctuation">(</span><span class="Identifier">enumValueA</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="Identifier">enumValueB</span><span class="Punctuation">)</span><span class="Whitespace">
+</span><span class="Comment"># bug #11078</span><span class="Whitespace">
+</span><span class="Keyword">for</span><span class="Whitespace"> </span><span class="Identifier">x</span><span class="Whitespace"> </span><span class="Keyword">in</span><span class="Whitespace"> </span><span class="StringLit">&quot;xx&quot;</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">discard</span></pre>top2
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="StringLit">&quot;in top2&quot;</span></pre>top2 after
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="StringLit">&quot;in top3&quot;</span></pre>top3 after
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Identifier">assert</span><span class="Whitespace"> </span><span class="DecNumber">3</span><span class="Operator">*</span><span class="DecNumber">2</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">6</span></pre></p>
   <div class="section" id="6">
 <h1><a class="toc-backref" href="#6">Imports</a></h1>
 <dl class="item">
@@ -218,6 +283,14 @@ The enum A.
 The enum B.
 
 </dd>
+<a id="Foo"></a>
+<dt><pre><a href="testproject.html#Foo"><span class="Identifier">Foo</span></a> <span class="Other">=</span> <span class="Keyword">enum</span>
+  <span class="Identifier">enumValueA2</span></pre></dt>
+<dd>
+
+
+
+</dd>
 
 </dl></div>
 <div class="section" id="8">
@@ -295,6 +368,15 @@ This is deprecated without message.
 This is deprecated with a message.
 
 </dd>
+<a id="fromUtils3"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#fromUtils3"><span class="Identifier">fromUtils3</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+came form utils but should be shown where <tt class="docutils literal"><span class="pre">fromUtilsGen</span></tt> is called
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>
+
+</dd>
 <a id="isValid,T"></a>
 <dt><pre><span class="Keyword">proc</span> <a href="#isValid%2CT"><span class="Identifier">isValid</span></a><span class="Other">[</span><span class="Identifier">T</span><span class="Other">]</span><span class="Other">(</span><span class="Identifier">x</span><span class="Other">:</span> <span class="Identifier">T</span><span class="Other">)</span><span class="Other">:</span> <span class="Identifier">bool</span></pre></dt>
 <dd>
@@ -302,6 +384,148 @@ This is deprecated with a message.
 
 
 </dd>
+<a id="z1"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z1"><span class="Identifier">z1</span></a><span class="Other">(</span><span class="Other">)</span><span class="Other">:</span> <a href="testproject.html#Foo"><span class="Identifier">Foo</span></a> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+cz1
+
+</dd>
+<a id="z2"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z2"><span class="Identifier">z2</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+cz2
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="StringLit">&quot;in cz2&quot;</span></pre>
+
+</dd>
+<a id="z3"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z3"><span class="Identifier">z3</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+cz3
+
+</dd>
+<a id="z4"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z4"><span class="Identifier">z4</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+cz4
+
+</dd>
+<a id="z5"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z5"><span class="Identifier">z5</span></a><span class="Other">(</span><span class="Other">)</span><span class="Other">:</span> <span class="Identifier">int</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+cz5
+
+</dd>
+<a id="z6"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z6"><span class="Identifier">z6</span></a><span class="Other">(</span><span class="Other">)</span><span class="Other">:</span> <span class="Identifier">int</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+cz6
+
+</dd>
+<a id="z7"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z7"><span class="Identifier">z7</span></a><span class="Other">(</span><span class="Other">)</span><span class="Other">:</span> <span class="Identifier">int</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+cz7
+
+</dd>
+<a id="z8"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z8"><span class="Identifier">z8</span></a><span class="Other">(</span><span class="Other">)</span><span class="Other">:</span> <span class="Identifier">int</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+cz8
+
+</dd>
+<a id="z9"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z9"><span class="Identifier">z9</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Operator">+</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">2</span></pre>
+
+</dd>
+<a id="z10"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z10"><span class="Identifier">z10</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+
+<p><strong class="examples_text">Example: cmd: -d:foobar</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>cz10
+
+</dd>
+<a id="z11"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z11"><span class="Identifier">z11</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>
+
+</dd>
+<a id="z12"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z12"><span class="Identifier">z12</span></a><span class="Other">(</span><span class="Other">)</span><span class="Other">:</span> <span class="Identifier">int</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>
+
+</dd>
+<a id="z13"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z13"><span class="Identifier">z13</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+cz13
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span></pre>
+
+</dd>
+<a id="baz"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#baz"><span class="Identifier">baz</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+
+
+</dd>
+<a id="z17"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#z17"><span class="Identifier">z17</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+cz17 rest
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>rest
+
+</dd>
+<a id="p1"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#p1"><span class="Identifier">p1</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+cp1
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Comment"># regular comments work here</span></pre>c4
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Comment"># c5 regular comments before 1st token work</span><span class="Whitespace">
+</span><span class="Comment"># regular comment</span><span class="Whitespace">
+</span><span class="LongComment">#[
+nested regular comment
+]#</span><span class="Whitespace">
+</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="DecNumber">2</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">2</span><span class="Whitespace"> </span><span class="Comment"># c8</span><span class="Whitespace">
+</span><span class="Comment">## this is a non-nested doc comment</span><span class="Whitespace">
+
+</span><span class="LongComment">##[
+this is a nested doc comment
+]##</span><span class="Whitespace">
+</span><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="StringLit">&quot;c9&quot;</span><span class="Whitespace">
+</span><span class="Comment"># also work after</span></pre>
+
+</dd>
 
 </dl></div>
 <div class="section" id="13">
@@ -316,6 +540,22 @@ My someFunc. Stuff in <tt class="docutils literal"><span class="pre">quotes</spa
 </dd>
 
 </dl></div>
+<div class="section" id="15">
+<h1><a class="toc-backref" href="#15">Iterators</a></h1>
+<dl class="item">
+<a id="fromUtils1.i"></a>
+<dt><pre><span class="Keyword">iterator</span> <a href="#fromUtils1.i"><span class="Identifier">fromUtils1</span></a><span class="Other">(</span><span class="Other">)</span><span class="Other">:</span> <span class="Identifier">int</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Comment"># ok1</span><span class="Whitespace">
+</span><span class="Identifier">assert</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace">
+</span><span class="Comment"># ok2</span></pre>
+
+</dd>
+
+</dl></div>
 <div class="section" id="17">
 <h1><a class="toc-backref" href="#17">Macros</a></h1>
 <dl class="item">
@@ -326,11 +566,43 @@ My someFunc. Stuff in <tt class="docutils literal"><span class="pre">quotes</spa
 
 
 </dd>
+<a id="z16.m"></a>
+<dt><pre><span class="Keyword">macro</span> <a href="#z16.m"><span class="Identifier">z16</span></a><span class="Other">(</span><span class="Other">)</span></pre></dt>
+<dd>
+
+
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>cz16 after
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="DecNumber">2</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">1</span><span class="Whitespace"> </span><span class="Operator">+</span><span class="Whitespace"> </span><span class="DecNumber">1</span></pre>
+
+</dd>
+<a id="z18.m"></a>
+<dt><pre><span class="Keyword">macro</span> <a href="#z18.m"><span class="Identifier">z18</span></a><span class="Other">(</span><span class="Other">)</span><span class="Other">:</span> <span class="Identifier">int</span></pre></dt>
+<dd>
+
+cz18
+
+</dd>
 
 </dl></div>
 <div class="section" id="18">
 <h1><a class="toc-backref" href="#18">Templates</a></h1>
 <dl class="item">
+<a id="fromUtils2.t"></a>
+<dt><pre><span class="Keyword">template</span> <a href="#fromUtils2.t"><span class="Identifier">fromUtils2</span></a><span class="Other">(</span><span class="Other">)</span></pre></dt>
+<dd>
+
+ok3
+
+</dd>
+<a id="z6t.t"></a>
+<dt><pre><span class="Keyword">template</span> <a href="#z6t.t"><span class="Identifier">z6t</span></a><span class="Other">(</span><span class="Other">)</span><span class="Other">:</span> <span class="Identifier">int</span></pre></dt>
+<dd>
+
+cz6t
+
+</dd>
 <a id="foo.t,SomeType,SomeType"></a>
 <dt><pre><span class="Keyword">template</span> <a href="#foo.t%2CSomeType%2CSomeType"><span class="Identifier">foo</span></a><span class="Other">(</span><span class="Identifier">a</span><span class="Other">,</span> <span class="Identifier">b</span><span class="Other">:</span> <a href="subdir/subdir_b/utils.html#SomeType"><span class="Identifier">SomeType</span></a><span class="Other">)</span></pre></dt>
 <dd>
@@ -338,6 +610,61 @@ My someFunc. Stuff in <tt class="docutils literal"><span class="pre">quotes</spa
 This does nothing 
 
 </dd>
+<a id="myfn.t"></a>
+<dt><pre><span class="Keyword">template</span> <a href="#myfn.t"><span class="Identifier">myfn</span></a><span class="Other">(</span><span class="Other">)</span></pre></dt>
+<dd>
+
+
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">std</span><span class="Operator">/</span><span class="Identifier">strutils</span><span class="Whitespace">
+</span><span class="Comment">## issue #8871 preserve formatting</span><span class="Whitespace">
+</span><span class="Comment">## line doc comment</span><span class="Whitespace">
+</span><span class="Comment"># bar</span><span class="Whitespace">
+</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace">
+</span><span class="LongComment">##[
+foo
+bar
+]##</span><span class="Whitespace">
+
+</span><span class="Identifier">doAssert</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">not</span><span class="Whitespace"> </span><span class="StringLit">&quot;foo&quot;</span><span class="Operator">.</span><span class="Identifier">startsWith</span><span class="Whitespace"> </span><span class="StringLit">&quot;ba&quot;</span><span class="Whitespace">
+</span><span class="Keyword">block</span><span class="Punctuation">:</span><span class="Whitespace">
+  </span><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="HexNumber">0xff</span><span class="Whitespace"> </span><span class="Comment"># elu par cette crapule</span><span class="Whitespace">
+</span><span class="Comment"># should be in</span></pre>should be still in
+
+</dd>
+<a id="z14.t"></a>
+<dt><pre><span class="Keyword">template</span> <a href="#z14.t"><span class="Identifier">z14</span></a><span class="Other">(</span><span class="Other">)</span></pre></dt>
+<dd>
+
+cz14
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span></pre>
+
+</dd>
+<a id="z15.t"></a>
+<dt><pre><span class="Keyword">template</span> <a href="#z15.t"><span class="Identifier">z15</span></a><span class="Other">(</span><span class="Other">)</span></pre></dt>
+<dd>
+
+cz15
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span></pre>
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">3</span></pre>
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">4</span></pre>ok5 ok5b
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Identifier">assert</span><span class="Whitespace"> </span><span class="Identifier">true</span></pre>in or out?
+
+</dd>
+<a id="testNimDocTrailingExample.t"></a>
+<dt><pre><span class="Keyword">template</span> <a href="#testNimDocTrailingExample.t"><span class="Identifier">testNimDocTrailingExample</span></a><span class="Other">(</span><span class="Other">)</span></pre></dt>
+<dd>
+
+
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="DecNumber">2</span></pre>
+
+</dd>
 
 </dl></div>
 
diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx
index 106c9cb76..76193de87 100644
--- a/nimdoc/testproject/expected/testproject.idx
+++ b/nimdoc/testproject/expected/testproject.idx
@@ -2,13 +2,41 @@ C_A	testproject.html#C_A	testproject: C_A
 C_B	testproject.html#C_B	testproject: C_B	
 C_C	testproject.html#C_C	testproject: C_C	
 C_D	testproject.html#C_D	testproject: C_D	
-foo	testproject.html#foo.t,SomeType,SomeType	testproject: foo(a, b: SomeType)	
 bar	testproject.html#bar,T,T	testproject: bar[T](a, b: T): T	
 baz	testproject.html#baz,T,T	testproject: baz[T](a, b: T): T	
 buzz	testproject.html#buzz,T,T	testproject: buzz[T](a, b: T): T	
-bar	testproject.html#bar.m	testproject: bar(): untyped	
 aVariable	testproject.html#aVariable	testproject: aVariable	
 A	testproject.html#A	testproject: A	
 B	testproject.html#B	testproject: B	
 someFunc	testproject.html#someFunc	testproject: someFunc()	
+fromUtils1	testproject.html#fromUtils1.i	testproject: fromUtils1(): int	
+fromUtils2	testproject.html#fromUtils2.t	testproject: fromUtils2()	
+fromUtils3	testproject.html#fromUtils3	testproject: fromUtils3()	
 isValid	testproject.html#isValid,T	testproject: isValid[T](x: T): bool	
+enumValueA2	testproject.html#enumValueA2	Foo.enumValueA2	
+Foo	testproject.html#Foo	testproject: Foo	
+z1	testproject.html#z1	testproject: z1(): Foo	
+z2	testproject.html#z2	testproject: z2()	
+z3	testproject.html#z3	testproject: z3()	
+z4	testproject.html#z4	testproject: z4()	
+z5	testproject.html#z5	testproject: z5(): int	
+z6	testproject.html#z6	testproject: z6(): int	
+z6t	testproject.html#z6t.t	testproject: z6t(): int	
+z7	testproject.html#z7	testproject: z7(): int	
+z8	testproject.html#z8	testproject: z8(): int	
+z9	testproject.html#z9	testproject: z9()	
+z10	testproject.html#z10	testproject: z10()	
+z11	testproject.html#z11	testproject: z11()	
+z12	testproject.html#z12	testproject: z12(): int	
+z13	testproject.html#z13	testproject: z13()	
+baz	testproject.html#baz	testproject: baz()	
+z17	testproject.html#z17	testproject: z17()	
+p1	testproject.html#p1	testproject: p1()	
+bar	testproject.html#bar.m	testproject: bar(): untyped	
+z16	testproject.html#z16.m	testproject: z16()	
+z18	testproject.html#z18.m	testproject: z18(): int	
+foo	testproject.html#foo.t,SomeType,SomeType	testproject: foo(a, b: SomeType)	
+myfn	testproject.html#myfn.t	testproject: myfn()	
+z14	testproject.html#z14.t	testproject: z14()	
+z15	testproject.html#z15.t	testproject: z15()	
+testNimDocTrailingExample	testproject.html#testNimDocTrailingExample.t	testproject: testNimDocTrailingExample()	
diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html
index 18a7ca2c2..df118d8a6 100644
--- a/nimdoc/testproject/expected/theindex.html
+++ b/nimdoc/testproject/expected/theindex.html
@@ -97,6 +97,8 @@ function main() {
           </ul></dd>
 <dt><a name="baz" href="#baz"><span>baz:</span></a></dt><dd><ul class="simple">
 <li><a class="reference external"
+          data-doc-search-tag="testproject: baz()" href="testproject.html#baz">testproject: baz()</a></li>
+          <li><a class="reference external"
           data-doc-search-tag="testproject: baz[T](a, b: T): T" href="testproject.html#baz%2CT%2CT">testproject: baz[T](a, b: T): T</a></li>
           </ul></dd>
 <dt><a name="bEnum" href="#bEnum"><span>bEnum:</span></a></dt><dd><ul class="simple">
@@ -127,6 +129,10 @@ function main() {
 <li><a class="reference external"
           data-doc-search-tag="SomeType.enumValueA" href="subdir/subdir_b/utils.html#enumValueA">SomeType.enumValueA</a></li>
           </ul></dd>
+<dt><a name="enumValueA2" href="#enumValueA2"><span>enumValueA2:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="Foo.enumValueA2" href="testproject.html#enumValueA2">Foo.enumValueA2</a></li>
+          </ul></dd>
 <dt><a name="enumValueB" href="#enumValueB"><span>enumValueB:</span></a></dt><dd><ul class="simple">
 <li><a class="reference external"
           data-doc-search-tag="SomeType.enumValueB" href="subdir/subdir_b/utils.html#enumValueB">SomeType.enumValueB</a></li>
@@ -135,14 +141,42 @@ function main() {
 <li><a class="reference external"
           data-doc-search-tag="SomeType.enumValueC" href="subdir/subdir_b/utils.html#enumValueC">SomeType.enumValueC</a></li>
           </ul></dd>
+<dt><a name="Foo" href="#Foo"><span>Foo:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: Foo" href="testproject.html#Foo">testproject: Foo</a></li>
+          </ul></dd>
 <dt><a name="foo" href="#foo"><span>foo:</span></a></dt><dd><ul class="simple">
 <li><a class="reference external"
           data-doc-search-tag="testproject: foo(a, b: SomeType)" href="testproject.html#foo.t%2CSomeType%2CSomeType">testproject: foo(a, b: SomeType)</a></li>
           </ul></dd>
+<dt><a name="fromUtils1" href="#fromUtils1"><span>fromUtils1:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: fromUtils1(): int" href="testproject.html#fromUtils1.i">testproject: fromUtils1(): int</a></li>
+          </ul></dd>
+<dt><a name="fromUtils2" href="#fromUtils2"><span>fromUtils2:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: fromUtils2()" href="testproject.html#fromUtils2.t">testproject: fromUtils2()</a></li>
+          </ul></dd>
+<dt><a name="fromUtils3" href="#fromUtils3"><span>fromUtils3:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: fromUtils3()" href="testproject.html#fromUtils3">testproject: fromUtils3()</a></li>
+          </ul></dd>
+<dt><a name="fromUtilsGen" href="#fromUtilsGen"><span>fromUtilsGen:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="utils: fromUtilsGen(): untyped" href="subdir/subdir_b/utils.html#fromUtilsGen.t">utils: fromUtilsGen(): untyped</a></li>
+          </ul></dd>
 <dt><a name="isValid" href="#isValid"><span>isValid:</span></a></dt><dd><ul class="simple">
 <li><a class="reference external"
           data-doc-search-tag="testproject: isValid[T](x: T): bool" href="testproject.html#isValid%2CT">testproject: isValid[T](x: T): bool</a></li>
           </ul></dd>
+<dt><a name="myfn" href="#myfn"><span>myfn:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: myfn()" href="testproject.html#myfn.t">testproject: myfn()</a></li>
+          </ul></dd>
+<dt><a name="p1" href="#p1"><span>p1:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: p1()" href="testproject.html#p1">testproject: p1()</a></li>
+          </ul></dd>
 <dt><a name="someFunc" href="#someFunc"><span>someFunc:</span></a></dt><dd><ul class="simple">
 <li><a class="reference external"
           data-doc-search-tag="testproject: someFunc()" href="testproject.html#someFunc">testproject: someFunc()</a></li>
@@ -155,6 +189,86 @@ function main() {
 <li><a class="reference external"
           data-doc-search-tag="utils: someType(): SomeType" href="subdir/subdir_b/utils.html#someType_2">utils: someType(): SomeType</a></li>
           </ul></dd>
+<dt><a name="testNimDocTrailingExample" href="#testNimDocTrailingExample"><span>testNimDocTrailingExample:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: testNimDocTrailingExample()" href="testproject.html#testNimDocTrailingExample.t">testproject: testNimDocTrailingExample()</a></li>
+          </ul></dd>
+<dt><a name="z1" href="#z1"><span>z1:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z1(): Foo" href="testproject.html#z1">testproject: z1(): Foo</a></li>
+          </ul></dd>
+<dt><a name="z10" href="#z10"><span>z10:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z10()" href="testproject.html#z10">testproject: z10()</a></li>
+          </ul></dd>
+<dt><a name="z11" href="#z11"><span>z11:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z11()" href="testproject.html#z11">testproject: z11()</a></li>
+          </ul></dd>
+<dt><a name="z12" href="#z12"><span>z12:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z12(): int" href="testproject.html#z12">testproject: z12(): int</a></li>
+          </ul></dd>
+<dt><a name="z13" href="#z13"><span>z13:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z13()" href="testproject.html#z13">testproject: z13()</a></li>
+          </ul></dd>
+<dt><a name="z14" href="#z14"><span>z14:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z14()" href="testproject.html#z14.t">testproject: z14()</a></li>
+          </ul></dd>
+<dt><a name="z15" href="#z15"><span>z15:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z15()" href="testproject.html#z15.t">testproject: z15()</a></li>
+          </ul></dd>
+<dt><a name="z16" href="#z16"><span>z16:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z16()" href="testproject.html#z16.m">testproject: z16()</a></li>
+          </ul></dd>
+<dt><a name="z17" href="#z17"><span>z17:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z17()" href="testproject.html#z17">testproject: z17()</a></li>
+          </ul></dd>
+<dt><a name="z18" href="#z18"><span>z18:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z18(): int" href="testproject.html#z18.m">testproject: z18(): int</a></li>
+          </ul></dd>
+<dt><a name="z2" href="#z2"><span>z2:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z2()" href="testproject.html#z2">testproject: z2()</a></li>
+          </ul></dd>
+<dt><a name="z3" href="#z3"><span>z3:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z3()" href="testproject.html#z3">testproject: z3()</a></li>
+          </ul></dd>
+<dt><a name="z4" href="#z4"><span>z4:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z4()" href="testproject.html#z4">testproject: z4()</a></li>
+          </ul></dd>
+<dt><a name="z5" href="#z5"><span>z5:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z5(): int" href="testproject.html#z5">testproject: z5(): int</a></li>
+          </ul></dd>
+<dt><a name="z6" href="#z6"><span>z6:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z6(): int" href="testproject.html#z6">testproject: z6(): int</a></li>
+          </ul></dd>
+<dt><a name="z6t" href="#z6t"><span>z6t:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z6t(): int" href="testproject.html#z6t.t">testproject: z6t(): int</a></li>
+          </ul></dd>
+<dt><a name="z7" href="#z7"><span>z7:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z7(): int" href="testproject.html#z7">testproject: z7(): int</a></li>
+          </ul></dd>
+<dt><a name="z8" href="#z8"><span>z8:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z8(): int" href="testproject.html#z8">testproject: z8(): int</a></li>
+          </ul></dd>
+<dt><a name="z9" href="#z9"><span>z9:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: z9()" href="testproject.html#z9">testproject: z9()</a></li>
+          </ul></dd>
 </dl>
     <div class="row">
       <div class="twelve-columns footer">
diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim
index 0576f194f..ad8f3d162 100644
--- a/nimdoc/testproject/subdir/subdir_b/utils.nim
+++ b/nimdoc/testproject/subdir/subdir_b/utils.nim
@@ -21,6 +21,8 @@ More text.
 
 ]##
 
+# BUG: module link in theindex.html is wrong, see #14434
+
 type
   SomeType* = enum
     enumValueA,
@@ -48,3 +50,23 @@ template bEnum*(): untyped =
     ## Stuff in `quotes` here.
     ## [Some link](https://nim-lang.org)
     discard
+
+template fromUtilsGen*(): untyped =
+  ## this should be shown in utils.html
+  runnableExamples:
+    assert 3*2 == 6
+  ## ditto
+
+  iterator fromUtils1*(): int =
+    runnableExamples:
+      # ok1
+      assert 1 == 1
+      # ok2
+    yield 15
+
+  template fromUtils2*() =
+    ## ok3
+
+  proc fromUtils3*() =
+    ## came form utils but should be shown where `fromUtilsGen` is called
+    runnableExamples: discard 1
diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim
index d1fcf58cd..39bbe3dc5 100644
--- a/nimdoc/testproject/testproject.nim
+++ b/nimdoc/testproject/testproject.nim
@@ -9,17 +9,22 @@ runnableExamples:
   # bug #11078
   for x in "xx": discard
 
+when true:
+  ## top2
+  runnableExamples:
+    discard "in top2"
+  ## top2 after
+
+runnableExamples:
+  discard "in top3"
+## top3 after
+
 const
   C_A* = 0x7FF0000000000000'f64
   C_B* = 0o377'i8
   C_C* = 0o277'i8
   C_D* = 0o177777'i16
 
-template foo*(a, b: SomeType) =
-  ## This does nothing
-  ##
-  discard
-
 proc bar*[T](a, b: T): T =
   result = a + b
 
@@ -33,13 +38,207 @@ proc buzz*[T](a, b: T): T {.deprecated: "since v0.20".} =
 
 import std/macros
 
-macro bar*(): untyped =
-  result = newStmtList()
-
 var aVariable*: array[1, int]
 
+# bug #9432
 aEnum()
 bEnum()
+fromUtilsGen()
 
-# bug #9432
 proc isValid*[T](x: T): bool = x.len > 0
+
+when true:
+  # these cases appear redundant but they're actually (almost) all different at
+  # AST level and needed to ensure docgen keeps working, eg because of issues
+  # like D20200526T163511
+  type
+    Foo* = enum
+      enumValueA2
+
+  proc z1*(): Foo =
+    ## cz1
+    Foo.default
+
+  proc z2*() =
+    ## cz2
+    runnableExamples:
+      discard "in cz2"
+
+  proc z3*() =
+    ## cz3
+
+  proc z4*() =
+    ## cz4
+    discard
+
+when true:
+  # tests for D20200526T163511
+  proc z5*(): int =
+    ## cz5
+    return 1
+
+  proc z6*(): int =
+    ## cz6
+    1
+
+  template z6t*(): int =
+    ## cz6t
+    1
+
+  proc z7*(): int =
+    ## cz7
+    result = 1
+
+  proc z8*(): int =
+    ## cz8
+    block:
+      discard
+      1+1
+
+when true:
+  # interleaving 0 or more runnableExamples and doc comments, issue #9227
+  proc z9*() =
+    runnableExamples: doAssert 1 + 1 == 2
+
+  proc z10*() =
+    runnableExamples "-d:foobar":
+      discard 1
+    ## cz10
+
+  proc z11*() =
+    runnableExamples:
+      discard 1
+    discard
+
+  proc z12*(): int =
+    runnableExamples:
+      discard 1
+    12
+
+  proc z13*() =
+    ## cz13
+    runnableExamples:
+      discard
+
+  proc baz*() = discard
+
+  proc bazNonExported() =
+    ## out (not exported)
+    runnableExamples:
+      # BUG: this currently this won't be run since not exported
+      # but probably should
+      doAssert false
+
+  proc z17*() =
+    # BUG: a comment before 1st doc comment currently doesn't prevent
+    # doc comment from being docgen'd; probably should be fixed
+    ## cz17
+    ## rest
+    runnableExamples:
+      discard 1
+    ## rest
+    # this comment separates docgen'd doc comments
+    ## out
+
+when true: # capture non-doc comments correctly even before 1st token
+  proc p1*() =
+    ## cp1
+    runnableExamples: doAssert 1 == 1 # regular comments work here
+    ## c4
+    runnableExamples:
+      # c5 regular comments before 1st token work
+      # regular comment
+      #[
+      nested regular comment
+      ]#
+      doAssert 2 == 2 # c8
+      ## this is a non-nested doc comment
+
+      ##[
+      this is a nested doc comment
+      ]##
+      discard "c9"
+      # also work after
+    # this should be out
+
+when true: # (most) macros
+  macro bar*(): untyped =
+    result = newStmtList()
+
+  macro z16*() =
+    runnableExamples: discard 1
+    ## cz16
+    ## after
+    runnableExamples:
+      doAssert 2 == 1 + 1
+    # BUG: we should probably render `cz16\nafter` by keeping newline instead or
+    # what it currently renders as: `cz16 after`
+
+  macro z18*(): int =
+    ## cz18
+    newLit 0
+
+when true: # (most) templates
+  template foo*(a, b: SomeType) =
+    ## This does nothing
+    ##
+    discard
+
+  template myfn*() =
+    runnableExamples:
+      import std/strutils
+      ## issue #8871 preserve formatting
+      ## line doc comment
+      # bar
+      doAssert "'foo" == "'foo"
+      ##[
+      foo
+      bar
+      ]##
+
+      doAssert: not "foo".startsWith "ba"
+      block:
+        discard 0xff # elu par cette crapule
+      # should be in
+    ## should be still in
+
+    # out
+    ## out
+
+  template z14*() =
+    ## cz14
+    runnableExamples:
+      discard
+
+  template z15*() =
+    ## cz15
+    runnableExamples:
+      discard
+    runnableExamples: discard 3
+    runnableExamples: discard 4
+    ## ok5
+    ## ok5b
+    runnableExamples: assert true
+
+    ## in or out?
+    # this is an edge case; a newline separate last runnableExamples from 
+    # next doc comment but AST isnt' aware of it; this could change in future
+    discard 8
+    ## out
+    runnableExamples: discard 1
+
+when true: # issue #14473
+  import std/[sequtils]
+  template doit(): untyped =
+    ## doit
+    ## return output only
+    toSeq([1,2])
+  echo doit() # using doAssert or similar to avoid echo would "hide" the original bug
+
+when true:
+  template testNimDocTrailingExample*() =
+    # this must be last entry in this file, it checks against a bug (that got fixed)
+    # where runnableExamples would not show if there was not at least 2 "\n" after
+    # the last character of runnableExamples
+    runnableExamples:
+      discard 2
\ No newline at end of file