Implement Pandoc Markdown concise link extension (#20304)

* Implement Pandoc Markdown concise link extension

This implements https://github.com/nim-lang/Nim/issues/20127.
Besides reference to headings we also support doing references
to Nim symbols inside Nim modules.

Markdown:
```
Some heading
------------

Ref. [Some heading].
```

Nim:
```
proc someFunction*() ...

... ## Ref. [someFunction]
```

This is substitution for RST syntax like `` `target`_ ``.
All 3 syntax variants of extension from Pandoc Markdown are supported:
`[target]`, `[target][]`, `[description][target]`.

This PR also fixes clashes in existing files, particularly
conflicts with RST footnote feature, which does not work with
this PR (but there is a plan to adopt a popular [Markdown footnote
extension](https://pandoc.org/MANUAL.html#footnotes) to make footnotes work).

Also the PR fixes a bug that Markdown links did not work when `[...]`
section had a line break.

The implementation is straightforward since link resolution did not
change w.r.t. RST implementation, it's almost only about new syntax
addition. The only essential difference is a possibility to add a custom
link description: form `[description][target]` which does not have an
RST equivalent.

* fix nim 1.0 gotcha

11 files changed, 92 insertions, 51 deletions

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 57764410e..61b3a7b43 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -182,7 +182,7 @@ template `^^`(n: NimNode, i: untyped): untyped =
 
 proc `[]`*[T, U: Ordinal](n: NimNode, x: HSlice[T, U]): seq[NimNode] =
   ## Slice operation for NimNode.
-  ## Returns a seq of child of `n` who inclusive range [n[x.a], n[x.b]].
+  ## Returns a seq of child of `n` who inclusive range `[n[x.a], n[x.b]]`.
   let xa = n ^^ x.a
   let L = (n ^^ x.b) - xa + 1
   result = newSeq[NimNode](L)
diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim
index 87fd34913..9a98cb9c5 100644
--- a/lib/impure/db_mysql.nim
+++ b/lib/impure/db_mysql.nim
@@ -180,7 +180,7 @@ iterator fastRows*(db: DbConn, query: SqlQuery,
   ## if you require **ALL** the rows.
   ##
   ## Breaking the fastRows() iterator during a loop will cause the next
-  ## database query to raise an [EDb] exception `Commands out of sync`.
+  ## database query to raise an `EDb` exception `Commands out of sync`.
   rawExec(db, query, args)
   var sqlres = mysql.useResult(PMySQL db)
   if sqlres != nil:
@@ -203,7 +203,7 @@ iterator instantRows*(db: DbConn, query: SqlQuery,
                       args: varargs[string, `$`]): InstantRow
                       {.tags: [ReadDbEffect].} =
   ## Same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within the iterator body.
+  ## on demand using `[]`. Returned handle is valid only within the iterator body.
   rawExec(db, query, args)
   var sqlres = mysql.useResult(PMySQL db)
   if sqlres != nil:
@@ -283,7 +283,7 @@ proc setColumnInfo(columns: var DbColumns; res: PRES; L: int) =
 iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery;
                       args: varargs[string, `$`]): InstantRow =
   ## Same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within the iterator body.
+  ## on demand using `[]`. Returned handle is valid only within the iterator body.
   rawExec(db, query, args)
   var sqlres = mysql.useResult(PMySQL db)
   if sqlres != nil:
diff --git a/lib/impure/db_odbc.nim b/lib/impure/db_odbc.nim
index 0ecd8129f..da1b1e9b5 100644
--- a/lib/impure/db_odbc.nim
+++ b/lib/impure/db_odbc.nim
@@ -168,7 +168,7 @@ proc dbError*(db: var DbConn) {.
     raise e
 
 proc sqlCheck(db: var DbConn, resVal: TSqlSmallInt) {.raises: [DbError]} =
-  ## Wrapper that raises [EDb] if `resVal` is neither SQL_SUCCESS or SQL_NO_DATA
+  ## Wrapper that raises `EDb` if `resVal` is neither SQL_SUCCESS or SQL_NO_DATA
   if resVal notIn [SQL_SUCCESS, SQL_NO_DATA]: dbError(db)
 
 proc sqlGetDBMS(db: var DbConn): string {.
@@ -304,7 +304,7 @@ iterator instantRows*(db: var DbConn, query: SqlQuery,
                       args: varargs[string, `$`]): InstantRow
                 {.tags: [ReadDbEffect, WriteDbEffect].} =
   ## Same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within the iterator body.
+  ## on demand using `[]`. Returned handle is valid only within the iterator body.
   var
     rowRes: Row = @[]
     sz: TSqlLen = 0
diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim
index 82403ab00..e629b7945 100644
--- a/lib/impure/db_postgres.nim
+++ b/lib/impure/db_postgres.nim
@@ -271,7 +271,7 @@ iterator instantRows*(db: DbConn, query: SqlQuery,
                       args: varargs[string, `$`]): InstantRow
                       {.tags: [ReadDbEffect].} =
   ## same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within iterator body.
+  ## on demand using `[]`. Returned handle is valid only within iterator body.
   setupSingeRowQuery(db, query, args)
   fetchinstantRows(db)
 
@@ -279,7 +279,7 @@ iterator instantRows*(db: DbConn, stmtName: SqlPrepared,
                       args: varargs[string, `$`]): InstantRow
                       {.tags: [ReadDbEffect].} =
   ## same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within iterator body.
+  ## on demand using `[]`. Returned handle is valid only within iterator body.
   setupSingeRowQuery(db, stmtName, args)
   fetchinstantRows(db)
 
diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim
index a11f2bbbb..b85e37983 100644
--- a/lib/packages/docutils/dochelpers.nim
+++ b/lib/packages/docutils/dochelpers.nim
@@ -82,7 +82,7 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol =
   ##
   ## This proc should be kept in sync with the `renderTypes` proc from
   ## ``compiler/typesrenderer.nim``.
-  assert linkText.kind in {rnRef, rnInner}
+  assert linkText.kind in {rnRstRef, rnInner}
 
   const NimDefs = ["proc", "func", "macro", "method", "iterator",
                    "template", "converter", "const", "type", "var",
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index 5d61dc8c3..8ea2b56f3 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -24,12 +24,12 @@
 ## using simple plaintext representation.
 ##
 ## This module is also embedded into Nim compiler; the compiler can output
-## the result to HTML [#html]_ or Latex [#latex]_.
+## the result to HTML \[#html] or Latex \[#latex].
 ##
-## .. [#html] commands `nim doc`:cmd: for ``*.nim`` files and
+## \[#html] commands `nim doc`:cmd: for ``*.nim`` files and
 ##    `nim rst2html`:cmd: for ``*.rst`` files
 ##
-## .. [#latex] commands `nim doc2tex`:cmd: for ``*.nim`` and
+## \[#latex] commands `nim doc2tex`:cmd: for ``*.nim`` and
 ##    `nim rst2tex`:cmd: for ``*.rst``.
 ##
 ## If you are new to Markdown/RST please consider reading the following:
@@ -84,8 +84,8 @@
 ##
 ## Additional Nim-specific features:
 ##
-## * directives: ``code-block`` [cmp:Sphinx]_, ``title``,
-##   ``index`` [cmp:Sphinx]_
+## * directives: ``code-block`` \[cmp:Sphinx], ``title``,
+##   ``index`` \[cmp:Sphinx]
 ## * predefined roles
 ##   - ``:nim:`` (default), ``:c:`` (C programming language),
 ##     ``:python:``, ``:yaml:``, ``:java:``, ``:cpp:`` (C++), ``:csharp`` (C#).
@@ -99,9 +99,9 @@
 ##     - ``:cmd:`` for commands and common shells syntax
 ##     - ``:console:`` the same  for interactive sessions
 ##       (commands should be prepended by ``$``)
-##     - ``:program:`` for executable names [cmp:Sphinx]_
+##     - ``:program:`` for executable names \[cmp:Sphinx]
 ##       (one can just use ``:cmd:`` on single word)
-##     - ``:option:`` for command line options [cmp:Sphinx]_
+##     - ``:option:`` for command line options \[cmp:Sphinx]
 ##   - ``:tok:``, a role for highlighting of programming language tokens
 ## * ***triple emphasis*** (bold and italic) using \*\*\*
 ## * ``:idx:`` role for \`interpreted text\` to include the link to this
@@ -115,7 +115,7 @@
 ##   Here the dummy `//` will disappear, while options `compile`:option:
 ##   and `doc`:option: will be left in the final document.
 ##
-## .. [cmp:Sphinx] similar but different from the directives of
+## \[cmp:Sphinx] similar but different from the directives of
 ##    Python `Sphinx directives`_ and `Sphinx roles`_ extensions
 ##
 ## .. _`extra features`:
@@ -1458,7 +1458,7 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
         newSons = n.sons
       result = newRstNode(newKind, newSons)
     else:  # some link that will be resolved in `resolveSubs`
-      newKind = rnRef
+      newKind = rnRstRef
       result = newRstNode(newKind, sons=newSons, info=n.info)
   elif match(p, p.idx, ":w:"):
     # a role:
@@ -1552,7 +1552,7 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) =
     while currentTok(p).kind in {tkWord, tkPunct}:
       if currentTok(p).kind == tkPunct:
         if isInlineMarkupEnd(p, "_", exact=true):
-          reference = newRstNode(rnRef, info=lineInfo(p, saveIdx))
+          reference = newRstNode(rnRstRef, info=lineInfo(p, saveIdx))
           break
         if not validRefnamePunct(currentTok(p).symbol):
           break
@@ -1746,7 +1746,9 @@ proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
     defaultCodeLangNim(p, result)
 
 proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool =
-  var desc, link = ""
+  # Parses Markdown link. If it's Pandoc auto-link then its second
+  # son (target) will be in tokenized format (rnInner with leafs).
+  var desc = newRstNode(rnInner)
   var i = p.idx
 
   var parensStack: seq[char]
@@ -1754,31 +1756,59 @@ proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool =
     parensStack.setLen 0
     inc i # skip begin token
     while true:
-      if p.tok[i].kind in {tkEof, tkIndent}: return false
+      if p.tok[i].kind == tkEof: return false
+      if p.tok[i].kind == tkIndent and p.tok[i+1].kind == tkIndent:
+        return false
       let isClosing = checkParen(p.tok[i], parensStack)
       if p.tok[i].symbol == endToken and not isClosing:
         break
-      dest.add p.tok[i].symbol
+      let symbol = if p.tok[i].kind == tkIndent: " " else: p.tok[i].symbol
+      when dest is string: dest.add symbol
+      else: dest.add newLeaf(symbol)
       inc i
     inc i # skip end token
 
   parse("]", desc)
-  if p.tok[i].symbol != "(": return false
-  let linkIdx = i + 1
-  parse(")", link)
-  # only commit if we detected no syntax error:
-  let protocol = safeProtocol(link)
-  if link == "":
-    result = false
-    rstMessage(p, mwBrokenLink, protocol,
-               p.tok[linkIdx].line, p.tok[linkIdx].col)
-  else:
-    let child = newRstNode(rnHyperlink)
-    child.add desc
-    child.add link
-    father.add child
+  if p.tok[i].symbol == "(":
+    var link = ""
+    let linkIdx = i + 1
+    parse(")", link)
+    # only commit if we detected no syntax error:
+    let protocol = safeProtocol(link)
+    if link == "":
+      result = false
+      rstMessage(p, mwBrokenLink, protocol,
+                 p.tok[linkIdx].line, p.tok[linkIdx].col)
+    else:
+      let child = newRstNode(rnHyperlink)
+      child.add newLeaf(desc.addNodes)
+      child.add link
+      father.add child
+      p.idx = i
+      result = true
+  elif roPreferMarkdown in p.s.options:
+    # Use Pandoc's implicit_header_references extension
+    var n = newRstNode(rnPandocRef)
+    if p.tok[i].symbol == "[":
+      var link = newRstNode(rnInner)
+      let targetIdx = i + 1
+      parse("]", link)
+      n.add desc
+      if link.len != 0:  # [description][target]
+        n.add link
+        n.info = lineInfo(p, targetIdx)
+      else:              # [description=target][]
+        n.add desc
+        n.info = lineInfo(p, p.idx + 1)
+    else:                # [description=target]
+      n.add desc
+      n.add desc  # target is the same as description
+      n.info = lineInfo(p, p.idx + 1)
+    father.add n
     p.idx = i
     result = true
+  else:
+    result = false
 
 proc getFootnoteType(label: PRstNode): (FootnoteType, int) =
   if label.sons.len >= 1 and label.sons[0].kind == rnLeaf and
@@ -3510,6 +3540,13 @@ proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) =
 proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode =
     # Associate this link alias with its target and change node kind to
     # rnHyperlink or rnInternalRef appropriately.
+    var desc, alias: PRstNode
+    if n.kind == rnPandocRef:  # link like [desc][alias]
+      desc = n.sons[0]
+      alias = n.sons[1]
+    else:  # n.kind == rnRstRef, link like `desc=alias`_
+      desc = n
+      alias = n
     type LinkDef = object
       ar: AnchorRule
       priority: int
@@ -3521,14 +3558,13 @@ proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode =
       if result == 0:
         result = cmp(x.target, y.target)
     var foundLinks: seq[LinkDef]
-    let text = newRstNode(rnInner, n.sons)
-    let refn = rstnodeToRefname(n)
+    let refn = rstnodeToRefname(alias)
     var hyperlinks = findRef(s, refn)
     for y in hyperlinks:
       foundLinks.add LinkDef(ar: arHyperlink, priority: refPriority(y.kind),
                              target: y.value, info: y.info,
                              tooltip: "(" & $y.kind & ")")
-    let substRst = findMainAnchorRst(s, text.addNodes, n.info)
+    let substRst = findMainAnchorRst(s, alias.addNodes, n.info)
     for subst in substRst:
       foundLinks.add LinkDef(ar: arInternalRst, priority: subst.priority,
                              target: newLeaf(subst.target.anchor),
@@ -3536,19 +3572,19 @@ proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode =
                              tooltip: "(" & $subst.anchorType & ")")
     # find anchors automatically generated from Nim symbols
     if roNimFile in s.options:
-      let substNim = findMainAnchorNim(s, signature=text, n.info)
+      let substNim = findMainAnchorNim(s, signature=alias, n.info)
       for subst in substNim:
         foundLinks.add LinkDef(ar: arNim, priority: subst.priority,
                                target: newLeaf(subst.refname),
                                info: subst.info, tooltip: subst.tooltip)
     foundLinks.sort(cmp = cmp, order = Descending)
-    let linkText = addNodes(n)
+    let linkText = addNodes(desc)
     if foundLinks.len >= 1:
       let kind = if foundLinks[0].ar == arHyperlink: rnHyperlink
                  elif foundLinks[0].ar == arNim: rnNimdocRef
                  else: rnInternalRef
       result = newRstNode(kind)
-      result.sons = @[text, foundLinks[0].target]
+      result.sons = @[newRstNode(rnInner, desc.sons), foundLinks[0].target]
       if kind == rnNimdocRef: result.tooltip = foundLinks[0].tooltip
       if foundLinks.len > 1:  # report ambiguous link
         var targets = newSeq[string]()
@@ -3585,7 +3621,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
       if e != "": result = newLeaf(e)
       else: rstMessage(s.filenames, s.msgHandler, n.info,
                        mwUnknownSubstitution, key)
-  of rnRef:
+  of rnRstRef, rnPandocRef:
     result = resolveLink(s, n)
   of rnFootnote:
     var (fnType, num) = getFootnoteType(n.sons[0])
diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim
index eac3a5e15..05e4ec39e 100644
--- a/lib/packages/docutils/rstast.nim
+++ b/lib/packages/docutils/rstast.nim
@@ -49,7 +49,10 @@ type
     rnCitation,               # similar to footnote, so use rnFootnote instead
     rnFootnoteGroup,          # footnote group - exists for a purely stylistic
                               # reason: to display a few footnotes as 1 block
-    rnStandaloneHyperlink, rnHyperlink, rnRef, rnInternalRef, rnFootnoteRef,
+    rnStandaloneHyperlink, rnHyperlink,
+    rnRstRef,                 # RST reference like `section name`_
+    rnPandocRef,              # Pandoc Markdown reference like [section name]
+    rnInternalRef, rnFootnoteRef,
     rnNimdocRef,              # reference to automatically generated Nim symbol
     rnDirective,              # a general directive
     rnDirArg,                 # a directive argument (for some directives).
@@ -110,7 +113,7 @@ type
                               ## auto-numbered ones without a label)
     of rnMarkdownBlockQuoteItem:
       quotationDepth*: int    ## number of characters in line prefix
-    of rnRef, rnSubstitutionReferences,
+    of rnRstRef, rnPandocRef, rnSubstitutionReferences,
         rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef:
       info*: TLineInfo        ## To have line/column info for warnings at
                               ## nodes that are post-processed after parsing
@@ -281,7 +284,7 @@ proc renderRstToRst(d: var RenderContext, n: PRstNode, result: var string) =
     inc(d.indent, 2)
     renderRstSons(d, n, result)
     dec(d.indent, 2)
-  of rnRef:
+  of rnRstRef:
     result.add("`")
     renderRstSons(d, n, result)
     result.add("`_")
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index 32d2d3483..00f1ac19d 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -37,7 +37,7 @@
 ##
 ## * The same goes for footnotes/citations links: they point to themselves.
 ##   No backreferences are generated since finding all references of a footnote
-##   can be done by simply searching for [footnoteName].
+##   can be done by simply searching for ``[footnoteName]``.
 
 import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils,
   algorithm, parseutils, std/strbasics
@@ -1372,7 +1372,9 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
           "</div> &ensp; $1\n</div>\n",
       "\\item[\\textsuperscript{[$3]}]$2 $1\n",
       [body, n.anchor.idS, mark, n.anchor])
-  of rnRef:
+  of rnPandocRef:
+    renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false)
+  of rnRstRef:
     renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=false)
   of rnStandaloneHyperlink:
     renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=true)
diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim
index f65ca125e..dacb15e17 100644
--- a/lib/pure/memfiles.nim
+++ b/lib/pure/memfiles.nim
@@ -377,7 +377,7 @@ proc `$`*(ms: MemSlice): string {.inline.} =
   copyMem(addr(result[0]), ms.data, ms.size)
 
 iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline.} =
-  ## Iterates over [optional `eat`] `delim`-delimited slices in MemFile `mfile`.
+  ## Iterates over \[optional `eat`] `delim`-delimited slices in MemFile `mfile`.
   ##
   ## Default parameters parse lines ending in either Unix(\\l) or Windows(\\r\\l)
   ## style on on a line-by-line basis.  I.e., not every line needs the same ending.
diff --git a/lib/std/private/schubfach.nim b/lib/std/private/schubfach.nim
index 5b965aaa7..206153a68 100644
--- a/lib/std/private/schubfach.nim
+++ b/lib/std/private/schubfach.nim
@@ -6,7 +6,7 @@
 # --------------------------------------------------------------------------------------------------
 ##  This file contains an implementation of the Schubfach algorithm as described in
 ##
-##  [1] Raffaello Giulietti, "The Schubfach way to render doubles",
+##  \[1] Raffaello Giulietti, "The Schubfach way to render doubles",
 ##      https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN
 # --------------------------------------------------------------------------------------------------
 
diff --git a/lib/system/comparisons.nim b/lib/system/comparisons.nim
index 7122daa20..daa47fa59 100644
--- a/lib/system/comparisons.nim
+++ b/lib/system/comparisons.nim
@@ -250,7 +250,7 @@ proc max*[T](x: openArray[T]): T =
 
 
 proc clamp*[T](x, a, b: T): T =
-  ## Limits the value `x` within the interval [a, b].
+  ## Limits the value `x` within the interval \[a, b].
   ## This proc is equivalent to but faster than `max(a, min(b, x))`.
   ## 
   ## .. warning:: `a <= b` is assumed and will not be checked (currently).