Add `doctype: RST|Markdown|RstMarkdown` pragma (#20252)

* Add `doctype: RST|Markdown|RstMarkdown` pragma

Implements https://github.com/nim-lang/RFCs/issues/68 ,
see also discussion in https://github.com/nim-lang/Nim/issues/17987

The permitted values:
* `markdown`, which is default. It still contains nearly all of
  the RST supported but it is assumed that in time we will give up
  most or all RST features in this mode
* `rst`, without any extensions
* `RstMarkdown` — compatibility with Nim 1.x. It's basically RST
  with those Markdown features enabled that don't conflict with RST.

* Apply suggestions from code review

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

* Additional fix in spirit of review

* Fix test after #20188

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

3 files changed, 49 insertions, 2 deletions

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 1c03762fd..875089c49 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -293,6 +293,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   if preferMarkdown:
     options.incl roPreferMarkdown
   if not standaloneDoc: options.incl roNimFile
+  # (options can be changed dynamically in `setDoctype` by `{.doctype.}`)
   result.sharedState = newRstSharedState(
       options, filename.string,
       docgenFindFile, compilerMsgHandler)
@@ -1121,6 +1122,44 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem =
           param["types"].add %($kind)
         result.json["signature"]["genericParams"].add param
 
+proc setDoctype(d: PDoc, n: PNode) =
+  ## Processes `{.doctype.}` pragma changing Markdown/RST parsing options.
+  if n == nil:
+    return
+  if n.len != 2:
+    localError(d.conf, n.info, errUser,
+      "doctype pragma takes exactly 1 argument"
+    )
+    return
+  var dt = ""
+  case n[1].kind
+  of nkStrLit:
+    dt = toLowerAscii(n[1].strVal)
+  of nkIdent:
+    dt = toLowerAscii(n[1].ident.s)
+  else:
+    localError(d.conf, n.info, errUser,
+      "unknown argument type $1 provided to doctype" % [$n[1].kind]
+    )
+    return
+  case dt
+  of "markdown":
+    d.sharedState.options.incl roSupportMarkdown
+    d.sharedState.options.incl roPreferMarkdown
+  of "rstmarkdown":
+    d.sharedState.options.incl roSupportMarkdown
+    d.sharedState.options.excl roPreferMarkdown
+  of "rst":
+    d.sharedState.options.excl roSupportMarkdown
+    d.sharedState.options.excl roPreferMarkdown
+  else:
+    localError(d.conf, n.info, errUser,
+      (
+        "unknown doctype value \"$1\", should be from " &
+        "\"RST\", \"Markdown\", \"RstMarkdown\""
+      ) % [dt]
+    )
+
 proc checkForFalse(n: PNode): bool =
   result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0
 
@@ -1238,6 +1277,8 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
   of nkPragma:
     let pragmaNode = findPragma(n, wDeprecated)
     d.modDeprecationMsg.add(genDeprecationMsg(d, pragmaNode))
+    let doctypeNode = findPragma(n, wDoctype)
+    setDoctype(d, doctypeNode)
   of nkCommentStmt: d.modDescPre.add(genComment(d, n))
   of nkProcDef, nkFuncDef:
     when useEffectSystem: documentRaises(d.cache, n)
@@ -1373,6 +1414,9 @@ proc add(d: PDoc; j: JsonItem) =
 
 proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
   case n.kind
+  of nkPragma:
+    let doctypeNode = findPragma(n, wDoctype)
+    setDoctype(d, doctypeNode)
   of nkCommentStmt:
     if includeComments:
       d.add JsonItem(rst: genComment(d, n), rstField: "comment",
diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim
index 2e86b6c44..d5c728418 100644
--- a/compiler/pragmas.nim
+++ b/compiler/pragmas.nim
@@ -55,7 +55,7 @@ const
     wDeprecated,
     wPragma, wEmit, wUnroll,
     wLinearScanEnd, wPatterns, wTrMacros, wEffects, wNoForward, wReorder, wComputedGoto,
-    wExperimental, wThis, wUsed, wInvariant, wAssume, wAssert}
+    wExperimental, wDoctype, wThis, wUsed, wInvariant, wAssume, wAssert}
   stmtPragmasTopLevel* = {wChecks, wObjChecks, wFieldChecks, wRangeChecks,
     wBoundChecks, wOverflowChecks, wNilChecks, wStaticBoundchecks,
     wStyleChecks, wAssertions,
@@ -1216,6 +1216,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         if not isTopLevel(c):
           localError(c.config, n.info, "'experimental' pragma only valid as toplevel statement or in a 'push' environment")
         processExperimental(c, it)
+      of wDoctype:
+        if not isTopLevel(c):
+          localError(c.config, n.info, "\"doctype\" pragma only valid as top-level statement")
       of wNoRewrite:
         noVal(c, it)
       of wBase:
diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim
index ded02abe8..71b2e6384 100644
--- a/compiler/wordrecg.nim
+++ b/compiler/wordrecg.nim
@@ -80,7 +80,7 @@ type
     wLocalPassc = "localPassC", wBorrow = "borrow", wDiscardable = "discardable",
     wFieldChecks = "fieldChecks", wSubsChar = "subschar", wAcyclic = "acyclic",
     wShallow = "shallow", wUnroll = "unroll", wLinearScanEnd = "linearScanEnd",
-    wComputedGoto = "computedGoto", wExperimental = "experimental",
+    wComputedGoto = "computedGoto", wExperimental = "experimental", wDoctype = "doctype",
     wWrite = "write", wGensym = "gensym", wInject = "inject", wDirty = "dirty",
     wInheritable = "inheritable", wThreadVar = "threadvar", wEmit = "emit",
     wAsmNoStackFrame = "asmNoStackFrame", wImplicitStatic = "implicitStatic",