RST heading improvements (fix #17091) (#17195)

3 files changed, 107 insertions, 49 deletions

diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index f764b65b0..83e3ef6ff 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -8,9 +8,13 @@
 #
 
 ## ==================================
-## rst: Nim-flavored reStructuredText
+##                rst
 ## ==================================
 ##
+## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+## Nim-flavored reStructuredText
+## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+##
 ## This module implements a `reStructuredText`:idx: (RST) parser.
 ## A large subset is implemented with some limitations_ and
 ## `Nim-specific features`_.
@@ -410,7 +414,14 @@ proc getTokens(buffer: string, skipPounds: bool, tokens: var TokenSeq): int =
     tokens[0].kind = tkIndent
 
 type
-  LevelMap = array[char, int]
+  LevelInfo = object
+    symbol: char         # adornment character
+    hasOverline: bool    # has also overline (besides underline)?
+    line: int            # the last line of this style occurrence
+                         # (for error message)
+    hasPeers: bool       # has headings on the same level of hierarchy?
+  LevelMap = seq[LevelInfo]   # Saves for each possible title adornment
+                              # style its level in the current document.
   Substitution = object
     key*: string
     value*: PRstNode
@@ -433,7 +444,10 @@ type
 
   SharedState = object
     options: RstParseOptions    # parsing options
-    uLevel, oLevel: int         # counters for the section levels
+    hLevels: LevelMap           # hierarchy of heading styles
+    hTitleCnt: int              # =0 if no title, =1 if only main title,
+                                # =2 if both title and subtitle are present
+    hCurLevel: int              # current section level
     subs: seq[Substitution]     # substitutions
     refs: seq[Substitution]     # references
     anchors: seq[AnchorSubst]   # internal target substitutions
@@ -443,14 +457,6 @@ type
     lineFootnoteSymRef: seq[int]  # footnote line, their reference [*]_
     footnotes: seq[FootnoteSubst] # correspondence b/w footnote label,
                                   # number, order of occurrence
-    underlineToLevel: LevelMap  # Saves for each possible title adornment
-                                # character its level in the
-                                # current document.
-                                # This is for single underline adornments.
-    overlineToLevel: LevelMap   # Saves for each possible title adornment
-                                # character its level in the current
-                                # document.
-                                # This is for over-underline adornments.
     msgHandler: MsgHandler      # How to handle errors.
     findFile: FindFileHandler   # How to find files.
 
@@ -1408,11 +1414,30 @@ proc parseLiteralBlock(p: var RstParser): PRstNode =
       inc p.idx
   result.add(n)
 
-proc getLevel(map: var LevelMap, lvl: var int, c: char): int =
-  if map[c] == 0:
-    inc lvl
-    map[c] = lvl
-  result = map[c]
+proc getLevel(p: var RstParser, c: char, hasOverline: bool): int =
+  ## Returns (preliminary) heading level corresponding to `c` and
+  ## `hasOverline`. If level does not exist, add it first.
+  for i, hType in p.s.hLevels:
+    if hType.symbol == c and hType.hasOverline == hasOverline:
+      p.s.hLevels[i].line = curLine(p)
+      p.s.hLevels[i].hasPeers = true
+      return i
+  p.s.hLevels.add LevelInfo(symbol: c, hasOverline: hasOverline,
+                            line: curLine(p), hasPeers: false)
+  result = p.s.hLevels.len - 1
+
+proc countTitles(p: var RstParser, n: PRstNode) =
+  ## Fill `p.s.hTitleCnt`
+  for node in n.sons:
+    if node != nil:
+      if node.kind notin {rnOverline, rnSubstitutionDef, rnDefaultRole}:
+        break
+      if node.kind == rnOverline:
+        if p.s.hLevels[p.s.hTitleCnt].hasPeers:
+          break
+        inc p.s.hTitleCnt
+        if p.s.hTitleCnt >= 2:
+          break
 
 proc tokenAfterNewline(p: RstParser): int =
   result = p.idx
@@ -1529,7 +1554,7 @@ proc whichSection(p: RstParser): RstNodeKind =
       result = rnLeaf
   of tkPunct:
     if isMarkdownHeadline(p):
-      result = rnHeadline
+      result = rnMarkdownHeadline
     elif roSupportMarkdown in p.s.options and predNL(p) and
         match(p, p.idx, "| w") and findPipe(p, p.idx+3):
       result = rnMarkdownTable
@@ -1599,7 +1624,8 @@ proc parseParagraph(p: var RstParser, result: PRstNode) =
       elif currentTok(p).ival == currInd(p):
         inc p.idx
         case whichSection(p)
-        of rnParagraph, rnLeaf, rnHeadline, rnOverline, rnDirective:
+        of rnParagraph, rnLeaf, rnHeadline, rnMarkdownHeadline,
+            rnOverline, rnDirective:
           result.add newLeaf(" ")
         of rnLineBlock:
           result.addIfNotNil(parseLineBlock(p))
@@ -1620,20 +1646,60 @@ proc parseParagraph(p: var RstParser, result: PRstNode) =
       parseInline(p, result)
     else: break
 
+proc checkHeadingHierarchy(p: RstParser, lvl: int) =
+  if lvl - p.s.hCurLevel > 1:  # broken hierarchy!
+    proc descr(l: int): string =
+      (if p.s.hLevels[l].hasOverline: "overline " else: "underline ") &
+      repeat(p.s.hLevels[l].symbol, 5)
+    var msg = "(section level inconsistent: "
+    msg.add descr(lvl) & " unexpectedly found, " &
+      "while the following intermediate section level(s) are missing on lines "
+    msg.add $p.s.hLevels[p.s.hCurLevel].line & ".." & $curLine(p) & ":"
+    for l in p.s.hCurLevel+1 .. lvl-1:
+      msg.add " " & descr(l)
+      if l != lvl-1: msg.add ","
+    rstMessage(p, meNewSectionExpected, msg & ")")
+
 proc parseHeadline(p: var RstParser): PRstNode =
-  result = newRstNode(rnHeadline)
   if isMarkdownHeadline(p):
+    result = newRstNode(rnMarkdownHeadline)
+    # Note that level hierarchy is not checked for markdown headings
     result.level = currentTok(p).symbol.len
     assert(nextTok(p).kind == tkWhite)
     inc p.idx, 2
     parseUntilNewline(p, result)
   else:
+    result = newRstNode(rnHeadline)
     parseUntilNewline(p, result)
     assert(currentTok(p).kind == tkIndent)
     assert(nextTok(p).kind == tkAdornment)
     var c = nextTok(p).symbol[0]
     inc p.idx, 2
-    result.level = getLevel(p.s.underlineToLevel, p.s.uLevel, c)
+    result.level = getLevel(p, c, hasOverline=false)
+    checkHeadingHierarchy(p, result.level)
+    p.s.hCurLevel = result.level
+  addAnchor(p, rstnodeToRefname(result), reset=true)
+
+proc parseOverline(p: var RstParser): PRstNode =
+  var c = currentTok(p).symbol[0]
+  inc p.idx, 2
+  result = newRstNode(rnOverline)
+  while true:
+    parseUntilNewline(p, result)
+    if currentTok(p).kind == tkIndent:
+      inc p.idx
+      if prevTok(p).ival > currInd(p):
+        result.add newLeaf(" ")
+      else:
+        break
+    else:
+      break
+  result.level = getLevel(p, c, hasOverline=true)
+  checkHeadingHierarchy(p, result.level)
+  p.s.hCurLevel = result.level
+  if currentTok(p).kind == tkAdornment:
+    inc p.idx
+    if currentTok(p).kind == tkIndent: inc p.idx
   addAnchor(p, rstnodeToRefname(result), reset=true)
 
 type
@@ -1779,26 +1845,6 @@ proc parseTransition(p: var RstParser): PRstNode =
   if currentTok(p).kind == tkIndent: inc p.idx
   if currentTok(p).kind == tkIndent: inc p.idx
 
-proc parseOverline(p: var RstParser): PRstNode =
-  var c = currentTok(p).symbol[0]
-  inc p.idx, 2
-  result = newRstNode(rnOverline)
-  while true:
-    parseUntilNewline(p, result)
-    if currentTok(p).kind == tkIndent:
-      inc p.idx
-      if prevTok(p).ival > currInd(p):
-        result.add newLeaf(" ")
-      else:
-        break
-    else:
-      break
-  result.level = getLevel(p.s.overlineToLevel, p.s.oLevel, c)
-  if currentTok(p).kind == tkAdornment:
-    inc p.idx                # XXX: check?
-    if currentTok(p).kind == tkIndent: inc p.idx
-  addAnchor(p, rstnodeToRefname(result), reset=true)
-
 proc parseBulletList(p: var RstParser): PRstNode =
   result = nil
   if nextTok(p).kind == tkWhite:
@@ -1982,7 +2028,7 @@ proc parseSection(p: var RstParser, result: PRstNode) =
       if p.idx > 0: dec p.idx
       a = parseFields(p)
     of rnTransition: a = parseTransition(p)
-    of rnHeadline: a = parseHeadline(p)
+    of rnHeadline, rnMarkdownHeadline: a = parseHeadline(p)
     of rnOverline: a = parseOverline(p)
     of rnTable: a = parseSimpleTable(p)
     of rnMarkdownTable: a = parseMarkdownTable(p)
@@ -2399,6 +2445,15 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
       var e = getEnv(key)
       if e != "": result = newLeaf(e)
       else: rstMessage(p, mwUnknownSubstitution, key)
+  of rnHeadline, rnOverline:
+    # fix up section levels depending on presence of a title and subtitle
+    if p.s.hTitleCnt == 2:
+      if n.level == 1:    # it's the subtitle
+        n.level = 0
+      elif n.level >= 2:  # normal sections
+        n.level -= 1
+    elif p.s.hTitleCnt == 0:
+      n.level += 1
   of rnRef:
     let refn = rstnodeToRefname(n)
     var y = findRef(p, refn)
@@ -2498,6 +2553,7 @@ proc rstParse*(text, filename: string,
   p.line = line
   p.col = column + getTokens(text, roSkipPounds in options, p.tok)
   let unresolved = parseDoc(p)
+  countTitles(p, unresolved)
   orderFootnotes(p)
   result = resolveSubs(p, unresolved)
   hasToc = p.hasToc
diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim
index 6902b4a8b..c68df7daa 100644
--- a/lib/packages/docutils/rstast.nim
+++ b/lib/packages/docutils/rstast.nim
@@ -18,6 +18,7 @@ type
     rnInner,                  # an inner node or a root
     rnHeadline,               # a headline
     rnOverline,               # an over- and underlined headline
+    rnMarkdownHeadline,       # a Markdown headline
     rnTransition,             # a transition (the ------------- <hr> thingie)
     rnParagraph,              # a paragraph
     rnBulletList,             # a bullet list
@@ -84,9 +85,10 @@ type
     of rnAdmonition:
       adType*: string         ## admonition type: "note", "caution", etc. This
                               ## text will set the style and also be displayed
-    of rnOverline, rnHeadline:
-      level*: int             ## level of headings starting from 1 (document
-                              ## title) to larger ones (minor sub-sections)
+    of rnOverline, rnHeadline, rnMarkdownHeadline:
+      level*: int             ## level of headings starting from 1 (main
+                              ## chapter) to larger ones (minor sub-sections)
+                              ## level=0 means it's document title or subtitle
     of rnFootnote, rnCitation, rnFootnoteRef:
       order*: int             ## footnote order (for auto-symbol footnotes and
                               ## auto-numbered ones without a label)
@@ -363,8 +365,8 @@ proc renderRstToStr*(node: PRstNode, indent=0): string =
     if node.lineIndent == "\n": txt = "\t(blank line)"
     else: txt = "\tlineIndent=" & $node.lineIndent.len
     result.add txt
-  of rnHeadline, rnOverline:
-    result.add (if node.level == 0: "" else: "\tlevel=" & $node.level)
+  of rnHeadline, rnOverline, rnMarkdownHeadline:
+    result.add "\tlevel=" & $node.level
   of rnFootnote, rnCitation, rnFootnoteRef:
     result.add (if node.order == 0:   "" else: "\torder=" & $node.order)
   else:
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index f16fd5717..59a9ba09a 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -797,11 +797,11 @@ proc renderHeadline(d: PDoc, n: PRstNode, result: var string) =
     spaces(max(0, n.level)) & tmp)
 
 proc renderOverline(d: PDoc, n: PRstNode, result: var string) =
-  if d.meta[metaTitle].len == 0:
+  if n.level == 0 and d.meta[metaTitle].len == 0:
     for i in countup(0, len(n)-1):
       renderRstToOut(d, n.sons[i], d.meta[metaTitle])
     d.currentSection = d.meta[metaTitle]
-  elif d.meta[metaSubtitle].len == 0:
+  elif n.level == 0 and d.meta[metaSubtitle].len == 0:
     for i in countup(0, len(n)-1):
       renderRstToOut(d, n.sons[i], d.meta[metaSubtitle])
     d.currentSection = d.meta[metaSubtitle]
@@ -1140,7 +1140,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
   if n == nil: return
   case n.kind
   of rnInner: renderAux(d, n, result)
-  of rnHeadline: renderHeadline(d, n, result)
+  of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, result)
   of rnOverline: renderOverline(d, n, result)
   of rnTransition: renderAux(d, n, "<hr$2 />\n", "\\hrule$2\n", result)
   of rnParagraph: renderAux(d, n, "<p$2>$1</p>\n", "$2\n$1\n\n", result)