Add `doctype: RST|Markdown|RstMarkdown` pragma (#20252)

* Add `doctype: RST|Markdown|RstMarkdown` pragma

Implements https://github.com/nim-lang/RFCs/issues/68 ,
see also discussion in https://github.com/nim-lang/Nim/issues/17987

The permitted values:
* `markdown`, which is default. It still contains nearly all of
  the RST supported but it is assumed that in time we will give up
  most or all RST features in this mode
* `rst`, without any extensions
* `RstMarkdown` — compatibility with Nim 1.x. It's basically RST
  with those Markdown features enabled that don't conflict with RST.

* Apply suggestions from code review

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

* Additional fix in spirit of review

* Fix test after #20188

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

7 files changed, 158 insertions, 4 deletions

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 1c03762fd..875089c49 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -293,6 +293,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   if preferMarkdown:
     options.incl roPreferMarkdown
   if not standaloneDoc: options.incl roNimFile
+  # (options can be changed dynamically in `setDoctype` by `{.doctype.}`)
   result.sharedState = newRstSharedState(
       options, filename.string,
       docgenFindFile, compilerMsgHandler)
@@ -1121,6 +1122,44 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem =
           param["types"].add %($kind)
         result.json["signature"]["genericParams"].add param
 
+proc setDoctype(d: PDoc, n: PNode) =
+  ## Processes `{.doctype.}` pragma changing Markdown/RST parsing options.
+  if n == nil:
+    return
+  if n.len != 2:
+    localError(d.conf, n.info, errUser,
+      "doctype pragma takes exactly 1 argument"
+    )
+    return
+  var dt = ""
+  case n[1].kind
+  of nkStrLit:
+    dt = toLowerAscii(n[1].strVal)
+  of nkIdent:
+    dt = toLowerAscii(n[1].ident.s)
+  else:
+    localError(d.conf, n.info, errUser,
+      "unknown argument type $1 provided to doctype" % [$n[1].kind]
+    )
+    return
+  case dt
+  of "markdown":
+    d.sharedState.options.incl roSupportMarkdown
+    d.sharedState.options.incl roPreferMarkdown
+  of "rstmarkdown":
+    d.sharedState.options.incl roSupportMarkdown
+    d.sharedState.options.excl roPreferMarkdown
+  of "rst":
+    d.sharedState.options.excl roSupportMarkdown
+    d.sharedState.options.excl roPreferMarkdown
+  else:
+    localError(d.conf, n.info, errUser,
+      (
+        "unknown doctype value \"$1\", should be from " &
+        "\"RST\", \"Markdown\", \"RstMarkdown\""
+      ) % [dt]
+    )
+
 proc checkForFalse(n: PNode): bool =
   result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0
 
@@ -1238,6 +1277,8 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
   of nkPragma:
     let pragmaNode = findPragma(n, wDeprecated)
     d.modDeprecationMsg.add(genDeprecationMsg(d, pragmaNode))
+    let doctypeNode = findPragma(n, wDoctype)
+    setDoctype(d, doctypeNode)
   of nkCommentStmt: d.modDescPre.add(genComment(d, n))
   of nkProcDef, nkFuncDef:
     when useEffectSystem: documentRaises(d.cache, n)
@@ -1373,6 +1414,9 @@ proc add(d: PDoc; j: JsonItem) =
 
 proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
   case n.kind
+  of nkPragma:
+    let doctypeNode = findPragma(n, wDoctype)
+    setDoctype(d, doctypeNode)
   of nkCommentStmt:
     if includeComments:
       d.add JsonItem(rst: genComment(d, n), rstField: "comment",
diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim
index 2e86b6c44..d5c728418 100644
--- a/compiler/pragmas.nim
+++ b/compiler/pragmas.nim
@@ -55,7 +55,7 @@ const
     wDeprecated,
     wPragma, wEmit, wUnroll,
     wLinearScanEnd, wPatterns, wTrMacros, wEffects, wNoForward, wReorder, wComputedGoto,
-    wExperimental, wThis, wUsed, wInvariant, wAssume, wAssert}
+    wExperimental, wDoctype, wThis, wUsed, wInvariant, wAssume, wAssert}
   stmtPragmasTopLevel* = {wChecks, wObjChecks, wFieldChecks, wRangeChecks,
     wBoundChecks, wOverflowChecks, wNilChecks, wStaticBoundchecks,
     wStyleChecks, wAssertions,
@@ -1216,6 +1216,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         if not isTopLevel(c):
           localError(c.config, n.info, "'experimental' pragma only valid as toplevel statement or in a 'push' environment")
         processExperimental(c, it)
+      of wDoctype:
+        if not isTopLevel(c):
+          localError(c.config, n.info, "\"doctype\" pragma only valid as top-level statement")
       of wNoRewrite:
         noVal(c, it)
       of wBase:
diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim
index ded02abe8..71b2e6384 100644
--- a/compiler/wordrecg.nim
+++ b/compiler/wordrecg.nim
@@ -80,7 +80,7 @@ type
     wLocalPassc = "localPassC", wBorrow = "borrow", wDiscardable = "discardable",
     wFieldChecks = "fieldChecks", wSubsChar = "subschar", wAcyclic = "acyclic",
     wShallow = "shallow", wUnroll = "unroll", wLinearScanEnd = "linearScanEnd",
-    wComputedGoto = "computedGoto", wExperimental = "experimental",
+    wComputedGoto = "computedGoto", wExperimental = "experimental", wDoctype = "doctype",
     wWrite = "write", wGensym = "gensym", wInject = "inject", wDirty = "dirty",
     wInheritable = "inheritable", wThreadVar = "threadvar", wEmit = "emit",
     wAsmNoStackFrame = "asmNoStackFrame", wImplicitStatic = "implicitStatic",
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index b3adbf9de..9bc3c604a 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -586,7 +586,7 @@ type
     filenameToIdx*: Table[string, FileIndex]
     idxToFilename*: seq[string]
   RstSharedState = object
-    options: RstParseOptions    # parsing options
+    options*: RstParseOptions   # parsing options
     hLevels: LevelMap           # hierarchy of heading styles
     hTitleCnt: int              # =0 if no title, =1 if only main title,
                                 # =2 if both title and subtitle are present
diff --git a/nimdoc/test_doctype/expected/test_doctype.html b/nimdoc/test_doctype/expected/test_doctype.html
new file mode 100644
index 000000000..416541d4d
--- /dev/null
+++ b/nimdoc/test_doctype/expected/test_doctype.html
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!--  This file is generated by Nim. -->
+<html xmlns="https://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>nimdoc/test_doctype/test_doctype</title>
+
+<!-- Favicon -->
+<link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
+<link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">
+
+<!-- Google fonts -->
+<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
+<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
+
+<!-- CSS -->
+<link rel="stylesheet" type="text/css" href="nimdoc.out.css">
+
+<!-- JS -->
+<script type="text/javascript" src="dochack.js"></script>
+</head>
+<body>
+  <div class="document" id="documentId">
+    <div class="container">
+      <h1 class="title">nimdoc/test_doctype/test_doctype</h1>
+      <div class="row">
+  <div class="three columns">
+    <div class="theme-select-wrapper">
+      <label for="theme-select">Theme:&nbsp;</label>
+      <select id="theme-select" onchange="setTheme(this.value)">
+        <option value="auto">🌗 Match OS</option>
+        <option value="dark">🌑 Dark</option>
+        <option value="light">🌕 Light</option>
+      </select>
+    </div>
+    <div id="global-links">
+      <ul class="simple">
+        <li><a href="theindex.html">Index</a></li>
+      </ul>
+    </div>
+    <div id="searchInputDiv">
+      Search: <input type="search" id="searchInput" onkeyup="search()"/>
+    </div>
+    <div>
+      Group by:
+      <select onchange="groupBy(this.value)">
+        <option value="section">Section</option>
+        <option value="type">Type</option>
+      </select>
+    </div>
+    <ul class="simple simple-toc" id="toc-list">
+  <li><a class="reference" id="check_toc" href="#check">Check</a></li>
+<li><a class="reference" id="text_toc" href="#text">text</a></li>
+
+</ul>
+
+  </div>
+  <div class="nine columns" id="content">
+    
+    <div id="tocRoot"></div>
+    
+    <p class="module-desc"><p>Check</p>
+<p><pre class="listing">
+<span class="Identifier">text</span></pre></p>
+
+<h1><a class="toc-backref" id="check" href="#check">Check</a></h1>
+<h1><a class="toc-backref" id="text" href="#text">text</a></h1></p>
+    
+  </div>
+</div>
+
+      <div class="twelve-columns footer">
+        <span class="nim-sprite"></span>
+        <br>
+        <small style="color: var(--hint);">Made with Nim. Generated: 1970-01-02 03:46:40 UTC</small>
+      </div>
+    </div>
+  </div>
+  
+</body>
+</html>
diff --git a/nimdoc/test_doctype/test_doctype.nim b/nimdoc/test_doctype/test_doctype.nim
new file mode 100644
index 000000000..28d35fc1e
--- /dev/null
+++ b/nimdoc/test_doctype/test_doctype.nim
@@ -0,0 +1,15 @@
+# Markdown (default) interpretes this as text + a Markdown code block:
+
+## Check
+## ~~~~~
+## text
+## ~~~~~
+
+{.doctype: RST.}
+
+# Now RST interpretes this as 2 headings:
+
+## Check
+## ~~~~~
+## text
+## ~~~~~
diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim
index 32dce0709..ef82ae1b9 100644
--- a/nimdoc/tester.nim
+++ b/nimdoc/tester.nim
@@ -36,7 +36,7 @@ proc testNimDoc(prjDir, docsDir: string; switches: NimSwitches; fixup = false) =
   if nimBuildIndexSwitches != "":
     exec("$1 buildIndex $2" % [nimExe, nimBuildIndexSwitches])
 
-  for expected in walkDirRec(prjDir / "expected/"):
+  for expected in walkDirRec(prjDir / "expected/", checkDir=true):
     let produced = expected.replace('\\', '/').replace("/expected/", "/$1/" % [docsDir])
     if not fileExists(produced):
       echo "FAILURE: files not found: ", produced
@@ -79,5 +79,14 @@ let
                                             "$1/$2" % [test2Dir, test2DocsDir]])
 testNimDoc(test2Dir, test2DocsDir, test2Switches, fixup)
 
+# Test `nim doc` on file with `{.doctype.}` pragma
+let
+  test3PrjDir = "test_doctype"
+  test3PrjName = "test_doctype"
+  test3Dir = baseDir / test3PrjDir
+  test3DocsDir = "htmldocs"
+  test3Switches = NimSwitches(doc: @["$1/$2.nim" % [test3Dir, test3PrjName]])
+testNimDoc(test3Dir, test3DocsDir, test3Switches, fixup)
+
 if failures > 0:
   quit "$# failures occurred; see note in nimdoc/tester.nim regarding -d:nimTestsNimdocFixup" %  $failures