rst indentation fixes (ref #17340) (#17715)

5 files changed, 272 insertions, 43 deletions

diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index c2385d517..7c4b92f88 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -1532,6 +1532,55 @@ proc parseUntilNewline(p: var RstParser, father: PRstNode) =
     of tkEof, tkIndent: break
 
 proc parseSection(p: var RstParser, result: PRstNode) {.gcsafe.}
+
+proc tokenAfterNewline(p: RstParser, start: int): int =
+  result = start
+  while true:
+    case p.tok[result].kind
+    of tkEof:
+      break
+    of tkIndent:
+      inc result
+      break
+    else: inc result
+
+proc tokenAfterNewline(p: RstParser): int {.inline.} =
+  result = tokenAfterNewline(p, p.idx)
+
+proc getWrappableIndent(p: RstParser): int =
+  ## Gets baseline indentation for bodies of field lists and directives.
+  ## Handles situations like this (with possible de-indent in [case.3])::
+  ##
+  ##   :field:   definition                                          [case.1]
+  ##
+  ##   currInd   currentTok(p).col
+  ##   |         |
+  ##   v         v
+  ##
+  ##   .. Note:: defItem:                                            [case.2]
+  ##                 definition
+  ##
+  ##                 ^
+  ##                 |
+  ##                 nextIndent
+  ##
+  ##   .. Note:: - point1                                            [case.3]
+  ##       - point 2
+  ##
+  ##       ^
+  ##       |
+  ##       nextIndent
+  if currentTok(p).kind == tkIndent:
+    result = currentTok(p).ival
+  else:
+    var nextIndent = p.tok[tokenAfterNewline(p)-1].ival
+    if nextIndent <= currInd(p):          # parse only this line     [case.1]
+      result = currentTok(p).col
+    elif nextIndent >= currentTok(p).col: # may be a definition list [case.2]
+      result = currentTok(p).col
+    else:
+      result = nextIndent                 #                          [case.3]
+
 proc parseField(p: var RstParser): PRstNode =
   ## Returns a parsed rnField node.
   ##
@@ -1541,13 +1590,12 @@ proc parseField(p: var RstParser): PRstNode =
   var fieldname = newRstNode(rnFieldName)
   parseUntil(p, fieldname, ":", false)
   var fieldbody = newRstNode(rnFieldBody)
-  if currentTok(p).kind != tkIndent: parseLine(p, fieldbody)
-  if currentTok(p).kind == tkIndent:
-    var indent = currentTok(p).ival
-    if indent > col:
-      pushInd(p, indent)
-      parseSection(p, fieldbody)
-      popInd(p)
+  if currentTok(p).kind == tkWhite: inc p.idx
+  let indent = getWrappableIndent(p)
+  if indent > col:
+    pushInd(p, indent)
+    parseSection(p, fieldbody)
+    popInd(p)
   result.add(fieldname)
   result.add(fieldbody)
 
@@ -1652,20 +1700,6 @@ proc countTitles(p: var RstParser, n: PRstNode) =
         if p.s.hTitleCnt >= 2:
           break
 
-proc tokenAfterNewline(p: RstParser, start: int): int =
-  result = start
-  while true:
-    case p.tok[result].kind
-    of tkEof:
-      break
-    of tkIndent:
-      inc result
-      break
-    else: inc result
-
-proc tokenAfterNewline(p: RstParser): int {.inline.} =
-  result = tokenAfterNewline(p, p.idx)
-
 proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool =
   ## check that underline/overline length is enough for the heading.
   ## No support for Unicode.
@@ -1752,7 +1786,7 @@ proc whichSection(p: RstParser): RstNodeKind =
       return rnCodeBlock
     elif currentTok(p).symbol == "::":
       return rnLiteralBlock
-    elif currentTok(p).symbol == ".."  and predNL(p) and
+    elif currentTok(p).symbol == ".."  and
        nextTok(p).kind in {tkWhite, tkIndent}:
      return rnDirective
   case currentTok(p).kind
@@ -1780,10 +1814,9 @@ proc whichSection(p: RstParser): RstNodeKind =
     elif match(p, tokenAfterNewline(p), "aI") and
         isAdornmentHeadline(p, tokenAfterNewline(p)):
       result = rnHeadline
-    elif predNL(p) and
-        currentTok(p).symbol in ["+", "*", "-"] and nextTok(p).kind == tkWhite:
+    elif currentTok(p).symbol in ["+", "*", "-"] and nextTok(p).kind == tkWhite:
       result = rnBulletList
-    elif match(p, p.idx, ":w:E") and predNL(p):
+    elif match(p, p.idx, ":w:E"):
       # (currentTok(p).symbol == ":")
       result = rnFieldList
     elif match(p, p.idx, "(e) ") or match(p, p.idx, "e) ") or
@@ -2350,9 +2383,11 @@ proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode
       parseLine(p, args)
   result.add(args)
   if hasOptions in flags:
-    if currentTok(p).kind == tkIndent and currentTok(p).ival >= 3 and
+    if currentTok(p).kind == tkIndent and currentTok(p).ival > currInd(p) and
         nextTok(p).symbol == ":":
+      pushInd(p, currentTok(p).ival)
       options = parseFields(p)
+      popInd(p)
   result.add(options)
 
 proc indFollows(p: RstParser): bool =
@@ -2363,11 +2398,9 @@ proc parseBlockContent(p: var RstParser, father: var PRstNode,
   ## parse the final content part of explicit markup blocks (directives,
   ## footnotes, etc). Returns true if succeeded.
   if currentTok(p).kind != tkIndent or indFollows(p):
-    var nextIndent = p.tok[tokenAfterNewline(p)-1].ival
-    if nextIndent <= currInd(p):  # parse only this line
-      nextIndent = currentTok(p).col
-    pushInd(p, nextIndent)
-    var content = contentParser(p)
+    let blockIndent = getWrappableIndent(p)
+    pushInd(p, blockIndent)
+    let content = contentParser(p)
     popInd(p)
     father.add content
     result = true
diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim
index 394cc2698..00f3f2b35 100644
--- a/lib/packages/docutils/rstast.nim
+++ b/lib/packages/docutils/rstast.nim
@@ -350,7 +350,7 @@ proc renderRstToJson*(node: PRstNode): string =
 proc renderRstToStr*(node: PRstNode, indent=0): string =
   ## Writes the parsed RST `node` into a compact string
   ## representation in the format (one line per every sub-node):
-  ## ``indent - kind - text - level - order - anchor (if non-zero)``
+  ## ``indent - kind - [text|level|order|adType] - anchor (if non-zero)``
   ## (suitable for debugging of RST parsing).
   if node == nil:
     result.add " ".repeat(indent) & "[nil]\n"
@@ -358,21 +358,23 @@ proc renderRstToStr*(node: PRstNode, indent=0): string =
   result.add " ".repeat(indent) & $node.kind
   case node.kind
   of rnLeaf, rnSmiley:
-    result.add (if node.text == "": "" else: "\t'" & node.text & "'")
+    result.add (if node.text == "": "" else: "  '" & node.text & "'")
   of rnEnumList:
-    result.add "\tlabelFmt=" & node.labelFmt
+    result.add "  labelFmt=" & node.labelFmt
   of rnLineBlockItem:
     var txt: string
-    if node.lineIndent == "\n": txt = "\t(blank line)"
-    else: txt = "\tlineIndent=" & $node.lineIndent.len
+    if node.lineIndent == "\n": txt = "  (blank line)"
+    else: txt = "  lineIndent=" & $node.lineIndent.len
     result.add txt
+  of rnAdmonition:
+    result.add "  adType=" & node.adType
   of rnHeadline, rnOverline, rnMarkdownHeadline:
-    result.add "\tlevel=" & $node.level
+    result.add "  level=" & $node.level
   of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem:
-    result.add (if node.order == 0:   "" else: "\torder=" & $node.order)
+    result.add (if node.order == 0:   "" else: "  order=" & $node.order)
   else:
     discard
-  result.add (if node.anchor == "": "" else: "\tanchor='" & node.anchor & "'")
+  result.add (if node.anchor == "": "" else: "  anchor='" & node.anchor & "'")
   result.add "\n"
   for son in node.sons:
     result.add renderRstToStr(son, indent=indent+2)
diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html
index 32abc0f80..5c434193e 100644
--- a/nimdoc/rst2html/expected/rst_examples.html
+++ b/nimdoc/rst2html/expected/rst_examples.html
@@ -119,8 +119,8 @@ window.addEventListener('DOMContentLoaded', main);
   <div class="nine columns" id="content">
   <div id="tocRoot"></div>
   
-  <p class="module-desc"><table class="docinfo" frame="void" rules="none"><col class="docinfo-name" /><col class="docinfo-content" /><tbody valign="top"><tr><th class="docinfo-name">Authors:</th><td> Andreas Rumpf, Zahary Karadjov</td></tr>
-<tr><th class="docinfo-name">Version:</th><td> |nimversion|</td></tr>
+  <p class="module-desc"><table class="docinfo" frame="void" rules="none"><col class="docinfo-name" /><col class="docinfo-content" /><tbody valign="top"><tr><th class="docinfo-name">Authors:</th><td>Andreas Rumpf, Zahary Karadjov</td></tr>
+<tr><th class="docinfo-name">Version:</th><td>|nimversion|</td></tr>
 </tbody></table><blockquote><p>&quot;Complexity&quot; seems to be a lot like &quot;energy&quot;: you can transfer it from the end-user to one/some of the other players, but the total amount seems to remain pretty much constant for a given task. -- Ran</p></blockquote>
 
 <h1><a class="toc-backref" id="about-this-document" href="#about-this-document">About this document</a></h1><p><strong>Note</strong>: This document is a draft! Several of Nim's features may need more precise wording. This manual is constantly evolving into a proper specification.</p>
diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim
index 0645e4150..2398b92a8 100644
--- a/tests/stdlib/trst.nim
+++ b/tests/stdlib/trst.nim
@@ -1,6 +1,8 @@
 discard """
   output: '''
 
+[Suite] RST indentation
+
 [Suite] RST include directive
 '''
 """
@@ -9,9 +11,190 @@ discard """
 
 import ../../lib/packages/docutils/rstgen
 import ../../lib/packages/docutils/rst
-import unittest
+import ../../lib/packages/docutils/rstast
+import unittest, strutils
+import std/private/miscdollars
 import os
 
+proc toAst(input: string,
+            rstOptions: RstParseOptions = {roSupportMarkdown, roNimFile},
+            error: ref string = nil,
+            warnings: ref seq[string] = nil): string =
+  ## If `error` is nil then no errors should be generated.
+  ## The same goes for `warnings`.
+  proc testMsgHandler(filename: string, line, col: int, msgkind: MsgKind,
+                      arg: string) =
+    let mc = msgkind.whichMsgClass
+    let a = $msgkind % arg
+    var message: string
+    toLocation(message, filename, line, col + ColRstOffset)
+    message.add " $1: $2" % [$mc, a]
+    if mc == mcError:
+      doAssert error != nil, "unexpected RST error '" & message & "'"
+      error[] = message
+      # we check only first error because subsequent ones may be meaningless
+      raise newException(EParseError, message)
+    else:
+      doAssert warnings != nil, "unexpected RST warning '" & message & "'"
+      warnings[].add message
+  try:
+    const filen = "input"
+
+    proc myFindFile(filename: string): string =
+      # we don't find any files in online mode:
+      result = ""
+
+    var dummyHasToc = false
+    var rst = rstParse(input, filen, line=LineRstInit, column=ColRstInit,
+                       dummyHasToc, rstOptions, myFindFile, testMsgHandler)
+    result = renderRstToStr(rst)
+  except EParseError:
+    discard
+
+suite "RST indentation":
+  test "nested bullet lists":
+    let input = dedent """
+      * - bullet1
+        - bullet2
+      * - bullet3
+        - bullet4
+      """
+    let output = input.toAst
+    check(output == dedent"""
+      rnBulletList
+        rnBulletItem
+          rnBulletList
+            rnBulletItem
+              rnInner
+                rnLeaf  'bullet1'
+            rnBulletItem
+              rnInner
+                rnLeaf  'bullet2'
+        rnBulletItem
+          rnBulletList
+            rnBulletItem
+              rnInner
+                rnLeaf  'bullet3'
+            rnBulletItem
+              rnInner
+                rnLeaf  'bullet4'
+      """)
+
+  test "nested markup blocks":
+    let input = dedent"""
+      #) .. Hint:: .. Error:: none
+      #) .. Warning:: term0
+                        Definition0
+      #) some
+         paragraph1
+      #) term1
+           Definition1
+         term2
+           Definition2
+    """
+    check(input.toAst == dedent"""
+      rnEnumList  labelFmt=1)
+        rnEnumItem
+          rnAdmonition  adType=hint
+            [nil]
+            [nil]
+            rnAdmonition  adType=error
+              [nil]
+              [nil]
+              rnLeaf  'none'
+        rnEnumItem
+          rnAdmonition  adType=warning
+            [nil]
+            [nil]
+            rnDefList
+              rnDefItem
+                rnDefName
+                  rnLeaf  'term0'
+                rnDefBody
+                  rnInner
+                    rnLeaf  'Definition0'
+        rnEnumItem
+          rnInner
+            rnLeaf  'some'
+            rnLeaf  ' '
+            rnLeaf  'paragraph1'
+        rnEnumItem
+          rnDefList
+            rnDefItem
+              rnDefName
+                rnLeaf  'term1'
+              rnDefBody
+                rnInner
+                  rnLeaf  'Definition1'
+            rnDefItem
+              rnDefName
+                rnLeaf  'term2'
+              rnDefBody
+                rnInner
+                  rnLeaf  'Definition2'
+      """)
+
+  test "code-block parsing":
+    let input1 = dedent"""
+      .. code-block:: nim
+          :test: "nim c $1"
+      
+        template additive(typ: typedesc) =
+          discard
+      """
+    let input2 = dedent"""
+      .. code-block:: nim
+        :test: "nim c $1"
+      
+        template additive(typ: typedesc) =
+          discard
+      """
+    let input3 = dedent"""
+      .. code-block:: nim
+         :test: "nim c $1"
+         template additive(typ: typedesc) =
+           discard
+      """
+    let inputWrong = dedent"""
+      .. code-block:: nim
+       :test: "nim c $1"
+      
+         template additive(typ: typedesc) =
+           discard
+      """
+    let ast = dedent"""
+      rnCodeBlock
+        rnDirArg
+          rnLeaf  'nim'
+        rnFieldList
+          rnField
+            rnFieldName
+              rnLeaf  'test'
+            rnFieldBody
+              rnInner
+                rnLeaf  '"'
+                rnLeaf  'nim'
+                rnLeaf  ' '
+                rnLeaf  'c'
+                rnLeaf  ' '
+                rnLeaf  '$'
+                rnLeaf  '1'
+                rnLeaf  '"'
+          rnField
+            rnFieldName
+              rnLeaf  'default-language'
+            rnFieldBody
+              rnLeaf  'Nim'
+        rnLiteralBlock
+          rnLeaf  'template additive(typ: typedesc) =
+        discard'
+      """
+    check input1.toAst == ast
+    check input2.toAst == ast
+    check input3.toAst == ast
+    # "template..." should be parsed as a definition list attached to ":test:":
+    check inputWrong.toAst != ast
+
 suite "RST include directive":
   test "Include whole":
     "other.rst".writeFile("**test1**")
diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim
index ed5d72226..29747f4e8 100644
--- a/tests/stdlib/trstgen.nim
+++ b/tests/stdlib/trstgen.nim
@@ -1118,6 +1118,17 @@ Test1
       """
     check "<pre class=\"line-nums\">55\n56\n</pre>" in input.toHtml
 
+  test "Nim code-block indentation":
+    let input = dedent """
+      .. code-block:: nim
+        :number-lines: 55
+
+       x
+      """
+    let output = input.toHtml
+    check "<pre class=\"line-nums\">55\n</pre>" in output
+    check "<span class=\"Identifier\">x</span>" in output
+
   test "RST admonitions":
     # check that all admonitions are implemented
     let input0 = dedent """
@@ -1466,7 +1477,7 @@ Test1
             """<table class="docinfo" frame="void" rules="none">""" &
             """<col class="docinfo-name" /><col class="docinfo-content" />""" &
             """<tbody valign="top"><tr><th class="docinfo-name">field:</th>""" &
-            """<td> text</td></tr>""" & "\n</tbody></table>")
+            """<td>text</td></tr>""" & "\n</tbody></table>")
 
   test "Field list: body after newline":
     let output = dedent """