rst: add support for markdown tables (#15854)

* rst: add support for markdown tables

* change template into proc

* don't create unnecessary `seq[string]`

6 files changed, 116 insertions, 16 deletions

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 7609c96ee..8f6b7a560 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -127,6 +127,7 @@ template declareClosures =
     of meCannotOpenFile: k = errCannotOpenFile
     of meExpected: k = errXExpected
     of meGridTableNotImplemented: k = errGridTableNotImplemented
+    of meMarkdownIllformedTable: k = errMarkdownIllformedTable
     of meNewSectionExpected: k = errNewSectionExpected
     of meGeneralParseError: k = errGeneralParseError
     of meInvalidDirective: k = errInvalidDirectiveX
diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index 759002312..f9ea90caf 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -30,42 +30,43 @@ type
     errUnknown, errInternal, errIllFormedAstX, errCannotOpenFile,
     errXExpected,
     errGridTableNotImplemented,
+    errMarkdownIllformedTable,
     errGeneralParseError,
     errNewSectionExpected,
     errInvalidDirectiveX,
     errProveInit, # deadcode
     errGenerated,
     errUser,
-    
-    warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape", 
+
+    warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape",
     warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit",
     warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated",
-    warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic", 
-    warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX", 
-    warnLanguageXNotSupported = "LanguageXNotSupported", warnFieldXNotSupported = "FieldXNotSupported", 
+    warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic",
+    warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX",
+    warnLanguageXNotSupported = "LanguageXNotSupported", warnFieldXNotSupported = "FieldXNotSupported",
     warnCommentXIgnored = "CommentXIgnored", warnTypelessParam = "TypelessParam",
-    warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap", 
+    warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap",
     warnUnsafeCode = "UnsafeCode", warnUnusedImportX = "UnusedImport",
     warnInheritFromException = "InheritFromException", warnEachIdentIsTuple = "EachIdentIsTuple",
     warnUnsafeSetLen = "UnsafeSetLen", warnUnsafeDefault = "UnsafeDefault",
     warnProveInit = "ProveInit", warnProveField = "ProveField", warnProveIndex = "ProveIndex",
     warnUnreachableElse = "UnreachableElse", warnUnreachableCode = "UnreachableCode",
     warnStaticIndexCheck = "IndexCheck", warnGcUnsafe = "GcUnsafe", warnGcUnsafe2 = "GcUnsafe2",
-    warnUninit = "Uninit", warnGcMem = "GcMem", warnDestructor = "Destructor", 
-    warnLockLevel = "LockLevel", warnResultShadowed = "ResultShadowed", 
-    warnInconsistentSpacing = "Spacing",  warnCaseTransition = "CaseTransition", 
-    warnCycleCreated = "CycleCreated", warnObservableStores = "ObservableStores", 
+    warnUninit = "Uninit", warnGcMem = "GcMem", warnDestructor = "Destructor",
+    warnLockLevel = "LockLevel", warnResultShadowed = "ResultShadowed",
+    warnInconsistentSpacing = "Spacing",  warnCaseTransition = "CaseTransition",
+    warnCycleCreated = "CycleCreated", warnObservableStores = "ObservableStores",
     warnUser = "User",
 
     hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC",
     hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed",
     hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded",
-    hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX", 
-    hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintCodeBegin = "CodeBegin", 
+    hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX",
+    hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintCodeBegin = "CodeBegin",
     hintCodeEnd = "CodeEnd", hintConf = "Conf", hintPath = "Path",
-    hintConditionAlwaysTrue = "CondTrue", hintConditionAlwaysFalse = "CondFalse", hintName = "Name", 
+    hintConditionAlwaysTrue = "CondTrue", hintConditionAlwaysFalse = "CondFalse", hintName = "Name",
     hintPattern = "Pattern", hintExecuting = "Exec", hintLinking = "Link", hintDependency = "Dependency",
-    hintSource = "Source", hintPerformance = "Performance", hintStackTrace = "StackTrace", 
+    hintSource = "Source", hintPerformance = "Performance", hintStackTrace = "StackTrace",
     hintGCStats = "GCStats", hintGlobalVar = "GlobalVar", hintExpandMacro = "ExpandMacro",
     hintUser = "User", hintUserRaw = "UserRaw", hintExtendedContext = "ExtendedContext",
     hintMsgOrigin = "MsgOrigin", # since 1.3.5
@@ -79,6 +80,7 @@ const
     errCannotOpenFile: "cannot open '$1'",
     errXExpected: "'$1' expected",
     errGridTableNotImplemented: "grid table is not implemented",
+    errMarkdownIllformedTable: "illformed delimiter row of a markdown table",
     errGeneralParseError: "general parse error",
     errNewSectionExpected: "new section expected",
     errInvalidDirectiveX: "invalid directive: '$1'",
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index 56bfe580b..c04a6e07c 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -36,6 +36,7 @@ type
     meCannotOpenFile,
     meExpected,
     meGridTableNotImplemented,
+    meMarkdownIllformedTable,
     meNewSectionExpected,
     meGeneralParseError,
     meInvalidDirective,
@@ -53,6 +54,7 @@ const
     meCannotOpenFile: "cannot open '$1'",
     meExpected: "'$1' expected",
     meGridTableNotImplemented: "grid table is not implemented",
+    meMarkdownIllformedTable: "illformed delimiter row of a markdown table",
     meNewSectionExpected: "new section expected",
     meGeneralParseError: "general parse error",
     meInvalidDirective: "invalid directive: '$1'",
@@ -1091,6 +1093,13 @@ proc isMarkdownHeadline(p: RstParser): bool =
       if p.tok[p.idx+2].kind in {tkWord, tkOther, tkPunct}:
         result = true
 
+proc findPipe(p: RstParser, start: int): bool =
+  var i = start
+  while true:
+    if p.tok[i].symbol == "|": return true
+    if p.tok[i].kind in {tkIndent, tkEof}: return false
+    inc i
+
 proc whichSection(p: RstParser): RstNodeKind =
   case p.tok[p.idx].kind
   of tkAdornment:
@@ -1104,6 +1113,9 @@ proc whichSection(p: RstParser): RstNodeKind =
   of tkPunct:
     if isMarkdownHeadline(p):
       result = rnHeadline
+    elif roSupportMarkdown in p.s.options and predNL(p) and
+        match(p, p.idx, "| w") and findPipe(p, p.idx+3):
+      result = rnMarkdownTable
     elif p.tok[p.idx].symbol == "```":
       result = rnCodeBlock
     elif match(p, tokenAfterNewline(p), "ai"):
@@ -1206,6 +1218,9 @@ proc parseHeadline(p: var RstParser): PRstNode =
 
 type
   IntSeq = seq[int]
+  ColumnLimits = tuple
+    first, last: int
+  ColSeq = seq[ColumnLimits]
 
 proc tokEnd(p: RstParser): int =
   result = p.tok[p.idx].col + len(p.tok[p.idx].symbol) - 1
@@ -1280,6 +1295,63 @@ proc parseSimpleTable(p: var RstParser): PRstNode =
       add(a, b)
     add(result, a)
 
+proc readTableRow(p: var RstParser): ColSeq =
+  if p.tok[p.idx].symbol == "|": inc p.idx
+  while p.tok[p.idx].kind notin {tkIndent, tkEof}:
+    var limits: ColumnLimits
+    limits.first = p.idx
+    while p.tok[p.idx].kind notin {tkIndent, tkEof}:
+      if p.tok[p.idx].symbol == "|" and p.tok[p.idx-1].symbol != "\\": break
+      inc p.idx
+    limits.last = p.idx
+    result.add(limits)
+    if p.tok[p.idx].kind in {tkIndent, tkEof}: break
+    inc p.idx
+  p.idx = tokenAfterNewline(p)
+
+proc getColContents(p: var RstParser, colLim: ColumnLimits): string =
+  for i in colLim.first ..< colLim.last:
+    result.add(p.tok[i].symbol)
+  result.strip
+
+proc isValidDelimiterRow(p: var RstParser, colNum: int): bool =
+  let row = readTableRow(p)
+  if row.len != colNum: return false
+  for limits in row:
+    let content = getColContents(p, limits)
+    if content.len < 3 or not (content.startsWith("--") or content.startsWith(":-")):
+      return false
+  return true
+
+proc parseMarkdownTable(p: var RstParser): PRstNode =
+  var
+    row: ColSeq
+    colNum: int
+    a, b: PRstNode
+    q: RstParser
+  result = newRstNode(rnMarkdownTable)
+
+  proc parseRow(p: var RstParser, cellKind: RstNodeKind, result: PRstNode) =
+    row = readTableRow(p)
+    if colNum == 0: colNum = row.len # table header
+    elif row.len < colNum: row.setLen(colNum)
+    a = newRstNode(rnTableRow)
+    for j in 0 ..< colNum:
+      b = newRstNode(cellKind)
+      initParser(q, p.s)
+      q.col = p.col
+      q.line = p.tok[p.idx].line - 1
+      q.filename = p.filename
+      q.col += getTokens(getColContents(p, row[j]), false, q.tok)
+      b.add(parseDoc(q))
+      a.add(b)
+    result.add(a)
+
+  parseRow(p, rnTableHeaderCell, result)
+  if not isValidDelimiterRow(p, colNum): rstMessage(p, meMarkdownIllformedTable)
+  while predNL(p) and p.tok[p.idx].symbol == "|":
+    parseRow(p, rnTableDataCell, result)
+
 proc parseTransition(p: var RstParser): PRstNode =
   result = newRstNode(rnTransition)
   inc(p.idx)
@@ -1461,6 +1533,7 @@ proc parseSection(p: var RstParser, result: PRstNode) =
     of rnHeadline: a = parseHeadline(p)
     of rnOverline: a = parseOverline(p)
     of rnTable: a = parseSimpleTable(p)
+    of rnMarkdownTable: a = parseMarkdownTable(p)
     of rnOptionList: a = parseOptionList(p)
     else:
       #InternalError("rst.parseSection()")
diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim
index 044ea2c14..8b2159aeb 100644
--- a/lib/packages/docutils/rstast.nim
+++ b/lib/packages/docutils/rstast.nim
@@ -37,7 +37,7 @@ type
     rnLineBlock,              # the | thingie
     rnLineBlockItem,          # sons of the | thing
     rnBlockQuote,             # text just indented
-    rnTable, rnGridTable, rnTableRow, rnTableHeaderCell, rnTableDataCell,
+    rnTable, rnGridTable, rnMarkdownTable, rnTableRow, rnTableHeaderCell, rnTableDataCell,
     rnLabel,                  # used for footnotes and other things
     rnFootnote,               # a footnote
     rnCitation,               # similar to footnote
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index c70998edb..604cc3ca9 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -1094,7 +1094,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
   of rnBlockQuote:
     renderAux(d, n, "<blockquote><p>$1</p></blockquote>\n",
                     "\\begin{quote}$1\\end{quote}\n", result)
-  of rnTable, rnGridTable:
+  of rnTable, rnGridTable, rnMarkdownTable:
     renderAux(d, n,
       "<table border=\"1\" class=\"docutils\">$1</table>",
       "\\begin{table}\\begin{rsttab}{" &
diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim
index 8fdbf3911..d35bc5821 100644
--- a/tests/stdlib/trstgen.nim
+++ b/tests/stdlib/trstgen.nim
@@ -153,3 +153,27 @@ suite "YAML syntax highlighting":
     assert a == """(( <a class="reference external" href="https://nim-lang.org/">Nim</a> ))"""
     assert b == """((<a class="reference external" href="https://nim-lang.org/">Nim</a>))"""
     assert c == """[<a class="reference external" href="https://nim-lang.org/">Nim</a>]"""
+
+  test "Markdown tables":
+    let input1 = """
+| A1 header    | A2 \| not fooled
+| :---         | ----:       |
+| C1           | C2 **bold** | ignored |
+| D1 `code \|` | D2          | also ignored
+| E1 \| text   |
+|              | F2 without pipe
+not in table"""
+    let output1 = rstToHtml(input1, {roSupportMarkdown}, defaultConfig())
+    assert output1 == """<table border="1" class="docutils"><tr><th>A1 header</th><th>A2 | not fooled</th></tr>
+<tr><td>C1</td><td>C2 <strong>bold</strong></td></tr>
+<tr><td>D1 <tt class="docutils literal"><span class="pre">code |</span></tt></td><td>D2</td></tr>
+<tr><td>E1 | text</td><td></td></tr>
+<tr><td></td><td>F2 without pipe</td></tr>
+</table><p>not in table</p>
+"""
+    let input2 = """
+| A1 header | A2 |
+| --- | --- |"""
+    let output2 = rstToHtml(input2, {roSupportMarkdown}, defaultConfig())
+    assert output2 == """<table border="1" class="docutils"><tr><th>A1 header</th><th>A2</th></tr>
+</table>"""