docgen: implement doc link resolution in current module (#18642)

3 files changed, 268 insertions, 15 deletions

diff --git a/tests/stdlib/tdochelpers.nim b/tests/stdlib/tdochelpers.nim
new file mode 100644
index 000000000..5bcdf32d5
--- /dev/null
+++ b/tests/stdlib/tdochelpers.nim
@@ -0,0 +1,155 @@
+discard """
+  output: '''
+
+[Suite] Integration with Nim
+'''
+"""
+
+# tests for dochelpers.nim module
+
+import ../../lib/packages/docutils/[rstast, rst, dochelpers]
+import unittest
+
+proc rstParseTest(text: string): PRstNode =
+  proc testMsgHandler(filename: string, line, col: int, msgkind: MsgKind,
+                      arg: string) =
+    doAssert msgkind == mwBrokenLink
+  let r = rstParse(text, "-input-", LineRstInit, ColRstInit,
+                   {roPreferMarkdown, roSupportMarkdown, roNimFile},
+                   msgHandler=testMsgHandler)
+  result = r.node
+
+suite "Integration with Nim":
+  test "simple symbol parsing (shortest form)":
+    let input1 = "g_".rstParseTest
+    check input1.toLangSymbol == LangSymbol(symKind: "", name: "g")
+
+  test "simple symbol parsing (group of words)":
+    let input1 = "`Y`_".rstParseTest
+    check input1.toLangSymbol == LangSymbol(symKind: "", name: "Y")
+
+    # this means not a statement 'type', it's a backticked identifier `type`:
+    let input2 = "`type`_".rstParseTest
+    check input2.toLangSymbol == LangSymbol(symKind: "", name: "type")
+
+    let input3 = "`[]`_".rstParseTest
+    check input3.toLangSymbol == LangSymbol(symKind: "", name: "[]")
+
+    let input4 = "`X Y Z`_".rstParseTest
+    check input4.toLangSymbol == LangSymbol(symKind: "", name: "Xyz")
+
+  test "simple proc parsing":
+    let input1 = "proc f".rstParseTest
+    check input1.toLangSymbol == LangSymbol(symKind: "proc", name: "f")
+
+  test "another backticked name":
+    let input1 = """`template \`type\``_""".rstParseTest
+    check input1.toLangSymbol == LangSymbol(symKind: "template", name: "type")
+
+  test "simple proc parsing with parameters":
+    let input1 = "`proc f*()`_".rstParseTest
+    let input2 = "`proc f()`_".rstParseTest
+    let expected = LangSymbol(symKind: "proc", name: "f",
+                              parametersProvided: true)
+    check input1.toLangSymbol == expected
+    check input2.toLangSymbol == expected
+
+  test "symbol parsing with 1 parameter":
+    let input = "`f(G[int])`_".rstParseTest
+    let expected = LangSymbol(symKind: "", name: "f",
+                              parameters: @[("G[int]", "")],
+                              parametersProvided: true)
+    check input.toLangSymbol == expected
+
+  test "more proc parsing":
+    let input1 = "`proc f[T](x:G[T]):M[T]`_".rstParseTest
+    let input2 = "`proc f[ T ] ( x: G [T] ): M[T]`_".rstParseTest
+    let input3 = "`proc f*[T](x: G[T]): M[T]`_".rstParseTest
+    let expected = LangSymbol(symKind: "proc",
+                              name: "f",
+                              generics: "[T]",
+                              parameters: @[("x", "G[T]")],
+                              parametersProvided: true,
+                              outType: "M[T]")
+    check(input1.toLangSymbol == expected)
+    check(input2.toLangSymbol == expected)
+    check(input3.toLangSymbol == expected)
+
+  test "advanced proc parsing with Nim identifier normalization":
+    let input = """`proc binarySearch*[T, K](a: openArray[T]; key: K;
+                    cmp: proc (x: T; y: K): int)`_""".rstParseTest
+    let expected = LangSymbol(symKind: "proc",
+                              name: "binarysearch",
+                              generics: "[T,K]",
+                              parameters: @[
+                                ("a", "openarray[T]"),
+                                ("key", "K"),
+                                ("cmp", "proc(x:T;y:K):int")],
+                              parametersProvided: true,
+                              outType: "")
+    check(input.toLangSymbol == expected)
+
+  test "the same without proc":
+    let input = """`binarySearch*[T, K](a: openArray[T]; key: K;
+                    cmp: proc (x: T; y: K): int {.closure.})`_""".rstParseTest
+    let expected = LangSymbol(symKind: "",
+                              name: "binarysearch",
+                              generics: "[T,K]",
+                              parameters: @[
+                                ("a", "openarray[T]"),
+                                ("key", "K"),
+                                ("cmp", "proc(x:T;y:K):int")],
+                              parametersProvided: true,
+                              outType: "")
+    check(input.toLangSymbol == expected)
+
+  test "operator $ with and without backticks":
+    let input1 = """`func \`$\`*[T](a: \`open Array\`[T]): string`_""".
+                  rstParseTest
+    let input2 = """`func $*[T](a: \`open Array\`[T]): string`_""".
+                  rstParseTest
+    let expected = LangSymbol(symKind: "func",
+                              name: "$",
+                              generics: "[T]",
+                              parameters: @[("a", "openarray[T]")],
+                              parametersProvided: true,
+                              outType: "string")
+    check(input1.toLangSymbol == expected)
+    check(input2.toLangSymbol == expected)
+
+  test "operator [] with and without backticks":
+    let input1 = """`func \`[]\`[T](a: \`open Array\`[T], idx: int): T`_""".
+                  rstParseTest
+    let input2 = """`func [][T](a: \`open Array\`[T], idx: int): T`_""".
+                  rstParseTest
+    let expected = LangSymbol(symKind: "func",
+                              name: "[]",
+                              generics: "[T]",
+                              parameters: @[("a", "openarray[T]"),
+                                            ("idx", "int")],
+                              parametersProvided: true,
+                              outType: "T")
+    check(input1.toLangSymbol == expected)
+    check(input2.toLangSymbol == expected)
+
+  test "postfix symbol specifier #1":
+    let input = """`walkDir iterator`_""".
+                  rstParseTest
+    let expected = LangSymbol(symKind: "iterator",
+                              name: "walkdir")
+    check(input.toLangSymbol == expected)
+
+  test "postfix symbol specifier #2":
+    let input1 = """`\`[]\`[T](a: \`open Array\`[T], idx: int): T func`_""".
+                  rstParseTest
+    let input2 = """`[][T](a: \`open Array\`[T], idx: int): T func`_""".
+                  rstParseTest
+    let expected = LangSymbol(symKind: "func",
+                              name: "[]",
+                              generics: "[T]",
+                              parameters: @[("a", "openarray[T]"),
+                                            ("idx", "int")],
+                              parametersProvided: true,
+                              outType: "T")
+    check(input1.toLangSymbol == expected)
+    check(input2.toLangSymbol == expected)
diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim
index e9f67324c..e4609a671 100644
--- a/tests/stdlib/trst.nim
+++ b/tests/stdlib/trst.nim
@@ -17,9 +17,7 @@ discard """
 
 # tests for rst module
 
-import ../../lib/packages/docutils/rstgen
-import ../../lib/packages/docutils/rst
-import ../../lib/packages/docutils/rstast
+import ../../lib/packages/docutils/[rstgen, rst, rstast]
 import unittest, strutils
 import std/private/miscdollars
 import os
@@ -61,6 +59,52 @@ proc toAst(input: string,
       result = e.msg
 
 suite "RST parsing":
+  test "References are whitespace-neutral and case-insensitive":
+    # refname is 'lexical-analysis', the same for all the 3 variants:
+    check(dedent"""
+        Lexical Analysis
+        ================
+
+        Ref. `Lexical Analysis`_ or `Lexical analysis`_ or `lexical analysis`_.
+        """.toAst ==
+      dedent"""
+        rnInner
+          rnHeadline  level=1
+            rnLeaf  'Lexical'
+            rnLeaf  ' '
+            rnLeaf  'Analysis'
+          rnParagraph
+            rnLeaf  'Ref'
+            rnLeaf  '.'
+            rnLeaf  ' '
+            rnInternalRef
+              rnInner
+                rnLeaf  'Lexical'
+                rnLeaf  ' '
+                rnLeaf  'Analysis'
+              rnLeaf  'lexical-analysis'
+            rnLeaf  ' '
+            rnLeaf  'or'
+            rnLeaf  ' '
+            rnInternalRef
+              rnInner
+                rnLeaf  'Lexical'
+                rnLeaf  ' '
+                rnLeaf  'analysis'
+              rnLeaf  'lexical-analysis'
+            rnLeaf  ' '
+            rnLeaf  'or'
+            rnLeaf  ' '
+            rnInternalRef
+              rnInner
+                rnLeaf  'lexical'
+                rnLeaf  ' '
+                rnLeaf  'analysis'
+              rnLeaf  'lexical-analysis'
+            rnLeaf  '.'
+            rnLeaf  ' '
+      """)
+
   test "option list has priority over definition list":
     check(dedent"""
         --defusages
@@ -376,9 +420,9 @@ suite "Warnings":
     let output = input.toAst(warnings=warnings)
     check(warnings[] == @[
         "input(3, 14) Warning: broken link 'citation-som'",
-        "input(5, 7) Warning: broken link 'a-broken-link'",
+        "input(5, 7) Warning: broken link 'a broken Link'",
         "input(7, 15) Warning: unknown substitution 'undefined subst'",
-        "input(9, 6) Warning: broken link 'shortdotlink'"
+        "input(9, 6) Warning: broken link 'short.link'"
         ])
 
   test "With include directive and blank lines at the beginning":
@@ -391,7 +435,7 @@ suite "Warnings":
     let input = ".. include:: other.rst"
     var warnings = new seq[string]
     let output = input.toAst(warnings=warnings)
-    check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenlink'"]
+    check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenLink'"]
     check(output == dedent"""
       rnInner
         rnParagraph
@@ -404,6 +448,59 @@ suite "Warnings":
       """)
     removeFile("other.rst")
 
+  test "warnings for ambiguous links (references + anchors)":
+    # Reference like `x`_ generates a link alias x that may clash with others
+    let input = dedent"""
+      Manual reference: `foo <#foo,string,string>`_
+
+      .. _foo:
+
+      Paragraph.
+
+      Ref foo_
+      """
+    var warnings = new seq[string]
+    let output = input.toAst(warnings=warnings)
+    check(warnings[] == @[
+      dedent """
+      input(7, 5) Warning: ambiguous doc link `foo`
+        clash:
+          (3, 8): (manual directive anchor)
+          (1, 45): (implicitly-generated hyperlink alias)"""
+    ])
+    # reference should be resolved to the manually set anchor:
+    check(output ==
+      dedent"""
+        rnInner
+          rnParagraph
+            rnLeaf  'Manual'
+            rnLeaf  ' '
+            rnLeaf  'reference'
+            rnLeaf  ':'
+            rnLeaf  ' '
+            rnHyperlink
+              rnInner
+                rnLeaf  'foo'
+              rnInner
+                rnLeaf  '#'
+                rnLeaf  'foo'
+                rnLeaf  ','
+                rnLeaf  'string'
+                rnLeaf  ','
+                rnLeaf  'string'
+          rnParagraph  anchor='foo'
+            rnLeaf  'Paragraph'
+            rnLeaf  '.'
+          rnParagraph
+            rnLeaf  'Ref'
+            rnLeaf  ' '
+            rnInternalRef
+              rnInner
+                rnLeaf  'foo'
+              rnLeaf  'foo'
+            rnLeaf  ' '
+      """)
+
 suite "RST include directive":
   test "Include whole":
     "other.rst".writeFile("**test1**")
diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim
index 867677404..45cf1a68e 100644
--- a/tests/stdlib/trstgen.nim
+++ b/tests/stdlib/trstgen.nim
@@ -1054,8 +1054,9 @@ Test1
       Paragraph2 ref `internal anchor`_.
       """
     let output9 = input9.toHtml
-    #doAssert "id=\"internal-anchor\"" in output9
-    #doAssert "internal anchor" notin output9
+    # _`internal anchor` got erased:
+    check "href=\"#internal-anchor\"" notin output9
+    check "href=\"#citation-another\"" in output9
     doAssert output9.count("<hr class=\"footnote\">" &
                            "<div class=\"footnote-group\">") == 1
     doAssert output9.count("<div class=\"footnote-label\">") == 3
@@ -1330,12 +1331,12 @@ Test1
     """
     let output1 = input1.toHtml
     # "target101" should be erased and changed to "section-xyz":
-    doAssert "href=\"#target101\"" notin output1
-    doAssert "id=\"target101\""    notin output1
-    doAssert "href=\"#target102\"" notin output1
-    doAssert "id=\"target102\""    notin output1
-    doAssert "id=\"section-xyz\""     in output1
-    doAssert "href=\"#section-xyz\""  in output1
+    check "href=\"#target101\"" notin output1
+    check "id=\"target101\""    notin output1
+    check "href=\"#target102\"" notin output1
+    check "id=\"target102\""    notin output1
+    check "id=\"section-xyz\""     in output1
+    check "href=\"#section-xyz\""  in output1
 
     let input2 = dedent """
       .. _target300:
@@ -1405,7 +1406,7 @@ Test1
     let output1 = input1.toHtml
     doAssert "id=\"secdot1\"" in output1
     doAssert "id=\"Z2minusothercolonsecplusc-2\"" in output1
-    doAssert "id=\"linkdot1-2021\"" in output1
+    check "id=\"linkdot1-2021\"" in output1
     let ref1 = "<a class=\"reference internal\" href=\"#secdot1\">sec.1</a>"
     let ref2 = "<a class=\"reference internal\" href=\"#Z2minusothercolonsecplusc-2\">2-other:sec+c_2</a>"
     let ref3 = "<a class=\"reference internal\" href=\"#linkdot1-2021\">link.1_2021</a>"