runnableExamples: correctly handle multiline string litterals (#14492)

* runnableExamples: correctly handle multiline string litterals

* address comments: improve doc comments + variable namings

5 files changed, 168 insertions, 5 deletions

diff --git a/compiler/renderverbatim.nim b/compiler/renderverbatim.nim
index 974454456..d20ee1549 100644
--- a/compiler/renderverbatim.nim
+++ b/compiler/renderverbatim.nim
@@ -4,8 +4,10 @@ from xmltree import addEscaped
 import ast, options, msgs
 import packages/docutils/highlite
 
-# import compiler/renderer
-import renderer
+const isDebug = false
+when isDebug:
+  import renderer
+  import astalgo
 
 proc lastNodeRec(n: PNode): PNode =
   result = n
@@ -23,6 +25,66 @@ proc isInIndentationBlock(src: string, indent: int): bool =
     if src[j] != ' ': return false
   return true
 
+type LineData = object
+  ## keep track of which lines are starting inside a multiline doc comment.
+  ## We purposefully avoid re-doing parsing which is already done (we get a PNode)
+  ## so we don't worry about whether we're inside (nested) doc comments etc.
+  ## But we sill need some logic to disambiguate different multiline styles.
+  conf: ConfigRef
+  lineFirst: int
+  lines: seq[bool]
+    ## lines[index] is true if line `lineFirst+index` starts inside a multiline string
+    ## Using a HashSet (extra dependency) would simplify but not by much.
+
+proc tripleStrLitStartsAtNextLine(conf: ConfigRef, n: PNode): bool =
+  # enabling TLineInfo.offsetA,offsetB would probably make this easier
+  const tripleQuote = "\"\"\""
+  let src = sourceLine(conf, n.info)
+  let col = n.info.col
+  doAssert src.continuesWith(tripleQuote, col) # sanity check
+  var i = col + 3
+  var onlySpace = true
+  while true:
+    if src.len <= i:
+      doAssert src.len == i
+      return onlySpace
+    elif src.continuesWith(tripleQuote, i) and (src.len == i+3 or src[i+3] != '\"'):
+      return false # triple lit is in 1 line
+    elif src[i] != ' ': onlySpace = false
+    i.inc
+
+proc visitMultilineStrings(ldata: var LineData, n: PNode) =
+  var cline = ldata.lineFirst
+
+  template setLine() =
+    let index = cline - ldata.lineFirst
+    if ldata.lines.len < index+1: ldata.lines.setLen index+1
+    ldata.lines[index] = true
+
+  case n.kind
+  of nkTripleStrLit:
+    # same logic should be applied for any multiline token
+    # we could also consider nkCommentStmt but right now we just assume doc comments,
+    # unlike triple string litterals, don't de-indent from runnableExamples.
+    cline = n.info.line.int
+    if tripleStrLitStartsAtNextLine(ldata.conf, n):
+      cline.inc
+      setLine()
+    for ai in n.strVal:
+      case ai
+      of '\n':
+        cline.inc
+        setLine()
+      else: discard
+  else:
+    for i in 0..<n.safeLen:
+      visitMultilineStrings(ldata, n[i])
+
+proc startOfLineInsideTriple(ldata: LineData, line: int): bool =
+  let index = line - ldata.lineFirst
+  if index >= ldata.lines.len: false
+  else: ldata.lines[index]
+
 proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
   ## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
   ## we'd need to check performance impact to enable it for nimdoc.
@@ -39,20 +101,31 @@ proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
       assert true
     ]#
     first.line = n[0].info.line + 1
-    # first.col = n[0].info.col + 1 # anything with `col > n[0].col` is part of runnableExamples
 
   let last = n.lastNodeRec.info
   var info = first
   var indent = info.col
   let numLines = numLines(conf, info.fileIndex).uint16
   var lastNonemptyPos = 0
+
+  var ldata = LineData(lineFirst: first.line.int, conf: conf)
+  visitMultilineStrings(ldata, n[^1])
+  when isDebug:
+    debug(n)
+    for i in 0..<ldata.lines.len:
+      echo (i+ldata.lineFirst, ldata.lines[i])
+
   for line in first.line..numLines: # bugfix, see `testNimDocTrailingExample`
     info.line = line
     let src = sourceLine(conf, info)
-    if line > last.line and not isInIndentationBlock(src, indent):
+    let special = startOfLineInsideTriple(ldata, line.int)
+    if line > last.line and not special and not isInIndentationBlock(src, indent):
       break
     if line > first.line: result.add "\n"
-    if src.len > indent:
+    if special:
+      result.add src
+      lastNonemptyPos = result.len
+    elif src.len > indent:
       result.add src[indent..^1]
       lastNonemptyPos = result.len
   result.setLen lastNonemptyPos
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index 4fd5d8ad3..7bd2e64a3 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -198,6 +198,8 @@ function main() {
     title="low[T: Ordinal | enum | range](x: T): T"><wbr />low<span class="attachedType"></span></a></li>
   <li><a class="reference" href="#low2%2CT"
     title="low2[T: Ordinal | enum | range](x: T): T"><wbr />low2<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#tripleStrLitTest"
+    title="tripleStrLitTest()"><wbr />triple<wbr />Str<wbr />Lit<wbr />Test<span class="attachedType"></span></a></li>
 
   </ul>
 </li>
@@ -595,6 +597,49 @@ the c printf. etc.
 <pre class="listing"><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="StringLit">&quot;in low2&quot;</span></pre>
 
 </dd>
+<a id="tripleStrLitTest"></a>
+<dt><pre><span class="Keyword">proc</span> <a href="#tripleStrLitTest"><span class="Identifier">tripleStrLitTest</span></a><span class="Other">(</span><span class="Other">)</span> <span><span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span></span><span class="pragmawrap"><span class="Other">{.</span><span class="pragma"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span><span class="Other">.}</span></span></pre></dt>
+<dd>
+
+
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Comment">## mullitline string litterals are tricky as their indentation can span</span><span class="Whitespace">
+</span><span class="Comment">## below that of the runnableExamples</span><span class="Whitespace">
+</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s1a</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;
+should appear at indent 0
+  at indent 2
+at indent 0
+&quot;&quot;&quot;</span><span class="Whitespace">
+</span><span class="Comment"># make sure this works too</span><span class="Whitespace">
+</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s1b</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;start at same line
+  at indent 2
+at indent 0
+&quot;&quot;&quot;</span><span class="Whitespace"> </span><span class="Comment"># comment after</span><span class="Whitespace">
+</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s2</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;sandwich &quot;&quot;&quot;</span><span class="Whitespace">
+</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s3</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;&quot;&quot;&quot;</span><span class="Whitespace">
+</span><span class="Keyword">when</span><span class="Whitespace"> </span><span class="Identifier">false</span><span class="Punctuation">:</span><span class="Whitespace">
+  </span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s5</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;
+        in s5 &quot;&quot;&quot;</span><span class="Whitespace">
+
+</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s3b</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="Punctuation">[</span><span class="LongStringLit">&quot;&quot;&quot;
+%!? #[...] # inside a multiline ...
+&quot;&quot;&quot;</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="StringLit">&quot;foo&quot;</span><span class="Punctuation">]</span><span class="Whitespace">
+
+</span><span class="Comment">## make sure handles trailing spaces</span><span class="Whitespace">
+</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s4</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot; 
+&quot;&quot;&quot;</span><span class="Whitespace">
+
+</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s5</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot; x
+&quot;&quot;&quot;</span><span class="Whitespace">
+</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s6</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot; &quot;&quot;
+&quot;&quot;&quot;</span><span class="Whitespace">
+</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s7</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;</span><span class="Whitespace">
+</span><span class="Keyword">let</span><span class="Whitespace"> </span><span class="Identifier">s8</span><span class="Whitespace"> </span><span class="Operator">=</span><span class="Whitespace"> </span><span class="Punctuation">[</span><span class="LongStringLit">&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;&quot;</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="LongStringLit">&quot;&quot;&quot;
+  &quot;&quot;&quot;</span><span class="Whitespace"> </span><span class="Punctuation">]</span><span class="Whitespace">
+</span><span class="Keyword">discard</span><span class="Whitespace">
+</span><span class="Comment"># should be in</span></pre>
+
+</dd>
 
 </dl></div>
 <div class="section" id="13">
diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx
index dd26c03d9..46798f314 100644
--- a/nimdoc/testproject/expected/testproject.idx
+++ b/nimdoc/testproject/expected/testproject.idx
@@ -37,6 +37,7 @@ c_printf	testproject.html#c_printf,cstring	testproject: c_printf(frmt: cstring):
 c_nonexistant	testproject.html#c_nonexistant,cstring	testproject: c_nonexistant(frmt: cstring): cint	
 low	testproject.html#low,T	testproject: low[T: Ordinal | enum | range](x: T): T	
 low2	testproject.html#low2,T	testproject: low2[T: Ordinal | enum | range](x: T): T	
+tripleStrLitTest	testproject.html#tripleStrLitTest	testproject: tripleStrLitTest()	
 bar	testproject.html#bar.m	testproject: bar(): untyped	
 z16	testproject.html#z16.m	testproject: z16()	
 z18	testproject.html#z18.m	testproject: z18(): int	
diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html
index 1e9e4374f..14d07bfd9 100644
--- a/nimdoc/testproject/expected/theindex.html
+++ b/nimdoc/testproject/expected/theindex.html
@@ -213,6 +213,10 @@ function main() {
 <li><a class="reference external"
           data-doc-search-tag="testproject: testNimDocTrailingExample()" href="testproject.html#testNimDocTrailingExample.t">testproject: testNimDocTrailingExample()</a></li>
           </ul></dd>
+<dt><a name="tripleStrLitTest" href="#tripleStrLitTest"><span>tripleStrLitTest:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: tripleStrLitTest()" href="testproject.html#tripleStrLitTest">testproject: tripleStrLitTest()</a></li>
+          </ul></dd>
 <dt><a name="z1" href="#z1"><span>z1:</span></a></dt><dd><ul class="simple">
 <li><a class="reference external"
           data-doc-search-tag="testproject: z1(): Foo" href="testproject.html#z1">testproject: z1(): Foo</a></li>
diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim
index 45bbca223..b05754d51 100644
--- a/nimdoc/testproject/testproject.nim
+++ b/nimdoc/testproject/testproject.nim
@@ -207,6 +207,46 @@ when true: # tests RST inside comments
     runnableExamples:
       discard "in low2"
 
+when true: # multiline string litterals
+  proc tripleStrLitTest*() =
+    runnableExamples:
+      ## mullitline string litterals are tricky as their indentation can span
+      ## below that of the runnableExamples
+      let s1a = """
+should appear at indent 0
+  at indent 2
+at indent 0
+"""
+      # make sure this works too
+      let s1b = """start at same line
+  at indent 2
+at indent 0
+""" # comment after
+      let s2 = """sandwich """
+      let s3 = """"""
+      when false:
+        let s5 = """
+        in s5 """
+
+      let s3b = ["""
+%!? #[...] # inside a multiline ...
+""", "foo"]
+
+      ## make sure handles trailing spaces
+      let s4 = """ 
+"""
+
+      let s5 = """ x
+"""
+      let s6 = """ ""
+"""
+      let s7 = """"""""""
+      let s8 = ["""""""""", """
+  """ ]
+      discard
+      # should be in
+    # should be out
+
 when true: # (most) macros
   macro bar*(): untyped =
     result = newStmtList()