the documentation generator now supports system.runnableExamples

9 files changed, 132 insertions, 58 deletions

diff --git a/changelog.md b/changelog.md
index 49cd4123a..14352374c 100644
--- a/changelog.md
+++ b/changelog.md
@@ -94,3 +94,7 @@ This now needs to be written as:
   - [``poly``](https://github.com/lcrees/polynumeric)
   - [``pdcurses``](https://github.com/lcrees/pdcurses)
   - [``romans``](https://github.com/lcrees/romans)
+
+- Added ``system.runnableExamples`` to make examples in Nim's documentation easier
+  to write and test. The examples are tested as the last step of
+  ``nim doc``.
diff --git a/compiler/ast.nim b/compiler/ast.nim
index 787cb4997..5bf4184c9 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -639,7 +639,7 @@ type
     mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl,
     mNHint, mNWarning, mNError,
     mInstantiationInfo, mGetTypeInfo, mNGenSym,
-    mNimvm, mIntDefine, mStrDefine
+    mNimvm, mIntDefine, mStrDefine, mRunnableExamples
 
 # things that we can evaluate safely at compile time, even if not asked for it:
 const
diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim
index 2050a746b..4879ce5c3 100644
--- a/compiler/condsyms.nim
+++ b/compiler/condsyms.nim
@@ -110,3 +110,4 @@ proc initDefines*() =
   when false: defineSymbol("nimHasOpt")
   defineSymbol("nimNoArrayToCstringConversion")
   defineSymbol("nimNewRoof")
+  defineSymbol("nimHasRunnableExamples")
diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 8978052e2..4a3674812 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -204,10 +204,85 @@ proc getPlainDocstring(n: PNode): string =
   if n.comment != nil and startsWith(n.comment, "##"):
     result = n.comment
   if result.len < 1:
-    if n.kind notin {nkEmpty..nkNilLit}:
-      for i in countup(0, len(n)-1):
-        result = getPlainDocstring(n.sons[i])
-        if result.len > 0: return
+    for i in countup(0, safeLen(n)-1):
+      result = getPlainDocstring(n.sons[i])
+      if result.len > 0: return
+
+proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRenderFlags = {}) =
+  var r: TSrcGen
+  var literal = ""
+  initTokRender(r, n, renderFlags)
+  var kind = tkEof
+  while true:
+    getNextTok(r, kind, literal)
+    case kind
+    of tkEof:
+      break
+    of tkComment:
+      dispA(result, "<span class=\"Comment\">$1</span>", "\\spanComment{$1}",
+            [rope(esc(d.target, literal))])
+    of tokKeywordLow..tokKeywordHigh:
+      dispA(result, "<span class=\"Keyword\">$1</span>", "\\spanKeyword{$1}",
+            [rope(literal)])
+    of tkOpr:
+      dispA(result, "<span class=\"Operator\">$1</span>", "\\spanOperator{$1}",
+            [rope(esc(d.target, literal))])
+    of tkStrLit..tkTripleStrLit:
+      dispA(result, "<span class=\"StringLit\">$1</span>",
+            "\\spanStringLit{$1}", [rope(esc(d.target, literal))])
+    of tkCharLit:
+      dispA(result, "<span class=\"CharLit\">$1</span>", "\\spanCharLit{$1}",
+            [rope(esc(d.target, literal))])
+    of tkIntLit..tkUInt64Lit:
+      dispA(result, "<span class=\"DecNumber\">$1</span>",
+            "\\spanDecNumber{$1}", [rope(esc(d.target, literal))])
+    of tkFloatLit..tkFloat128Lit:
+      dispA(result, "<span class=\"FloatNumber\">$1</span>",
+            "\\spanFloatNumber{$1}", [rope(esc(d.target, literal))])
+    of tkSymbol:
+      dispA(result, "<span class=\"Identifier\">$1</span>",
+            "\\spanIdentifier{$1}", [rope(esc(d.target, literal))])
+    of tkSpaces, tkInvalid:
+      add(result, literal)
+    of tkCurlyDotLe:
+      dispA(result, """<span class="Other pragmabegin">$1</span><div class="pragma">""",
+                    "\\spanOther{$1}",
+                  [rope(esc(d.target, literal))])
+    of tkCurlyDotRi:
+      dispA(result, "</div><span class=\"Other pragmaend\">$1</span>",
+                    "\\spanOther{$1}",
+                  [rope(esc(d.target, literal))])
+    of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi,
+       tkBracketDotLe, tkBracketDotRi, tkParDotLe,
+       tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot,
+       tkAccent, tkColonColon,
+       tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr:
+      dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}",
+            [rope(esc(d.target, literal))])
+
+proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
+  case n.kind
+  of nkCallKinds:
+    if n[0].kind == nkSym and n[0].sym.magic == mRunnableExamples and
+        n.len >= 2 and n.lastSon.kind == nkStmtList:
+      dispA(dest, "\n<strong class=\"examples_text\">$1</strong>\n",
+          "\n\\textbf{$1}\n", [rope"Examples:"])
+      inc d.listingCounter
+      let id = $d.listingCounter
+      dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"])
+      # this is a rather hacky way to get rid of the initial indentation
+      # that the renderer currently produces:
+      var i = 0
+      var body = n.lastSon
+      if body.len == 1 and body.kind == nkStmtList: body = body.lastSon
+      for b in body:
+        if i > 0: dest.add "\n"
+        inc i
+        nodeToHighlightedHtml(d, b, dest, {})
+      dest.add(d.config.getOrDefault"doc.listing_end" % id)
+  else: discard
+  for i in 0 ..< n.safeLen:
+    getAllRunnableExamples(d, n[i], dest)
 
 when false:
   proc findDocComment(n: PNode): PNode =
@@ -379,11 +454,12 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
   let
     name = getName(d, nameNode)
     nameRope = name.rope
-    plainDocstring = getPlainDocstring(n) # call here before genRecComment!
+  var plainDocstring = getPlainDocstring(n) # call here before genRecComment!
   var result: Rope = nil
   var literal, plainName = ""
   var kind = tkEof
   var comm = genRecComment(d, n)  # call this here for the side-effect!
+  getAllRunnableExamples(d, n, comm)
   var r: TSrcGen
   # Obtain the plain rendered string for hyperlink titles.
   initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
@@ -395,53 +471,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
     plainName.add(literal)
 
   # Render the HTML hyperlink.
-  initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments})
-  while true:
-    getNextTok(r, kind, literal)
-    case kind
-    of tkEof:
-      break
-    of tkComment:
-      dispA(result, "<span class=\"Comment\">$1</span>", "\\spanComment{$1}",
-            [rope(esc(d.target, literal))])
-    of tokKeywordLow..tokKeywordHigh:
-      dispA(result, "<span class=\"Keyword\">$1</span>", "\\spanKeyword{$1}",
-            [rope(literal)])
-    of tkOpr:
-      dispA(result, "<span class=\"Operator\">$1</span>", "\\spanOperator{$1}",
-            [rope(esc(d.target, literal))])
-    of tkStrLit..tkTripleStrLit:
-      dispA(result, "<span class=\"StringLit\">$1</span>",
-            "\\spanStringLit{$1}", [rope(esc(d.target, literal))])
-    of tkCharLit:
-      dispA(result, "<span class=\"CharLit\">$1</span>", "\\spanCharLit{$1}",
-            [rope(esc(d.target, literal))])
-    of tkIntLit..tkUInt64Lit:
-      dispA(result, "<span class=\"DecNumber\">$1</span>",
-            "\\spanDecNumber{$1}", [rope(esc(d.target, literal))])
-    of tkFloatLit..tkFloat128Lit:
-      dispA(result, "<span class=\"FloatNumber\">$1</span>",
-            "\\spanFloatNumber{$1}", [rope(esc(d.target, literal))])
-    of tkSymbol:
-      dispA(result, "<span class=\"Identifier\">$1</span>",
-            "\\spanIdentifier{$1}", [rope(esc(d.target, literal))])
-    of tkSpaces, tkInvalid:
-      add(result, literal)
-    of tkCurlyDotLe:
-      dispA(result, """<span class="Other pragmabegin">$1</span><div class="pragma">""",
-                    "\\spanOther{$1}",
-                  [rope(esc(d.target, literal))])
-    of tkCurlyDotRi:
-      dispA(result, "</div><span class=\"Other pragmaend\">$1</span>",
-                    "\\spanOther{$1}",
-                  [rope(esc(d.target, literal))])
-    of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi,
-       tkBracketDotLe, tkBracketDotRi, tkParDotLe,
-       tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot,
-       tkAccent, tkColonColon,
-       tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr:
-      dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}",
-            [rope(esc(d.target, literal))])
+  nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments, renderDocComments})
 
   inc(d.id)
   let
@@ -609,10 +639,7 @@ proc generateJson*(d: PDoc, n: PNode) =
   else: discard
 
 proc genTagsItem(d: PDoc, n, nameNode: PNode, k: TSymKind): string =
-  var
-    name = getName(d, nameNode)
-
-  result = name & "\n"
+  result = getName(d, nameNode) & "\n"
 
 proc generateTags*(d: PDoc, n: PNode, r: var Rope) =
   case n.kind
diff --git a/compiler/sem.nim b/compiler/sem.nim
index 3608bc11c..495321de4 100644
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -570,6 +570,18 @@ proc myProcess(context: PPassContext, n: PNode): PNode =
         result = ast.emptyNode
       #if gCmd == cmdIdeTools: findSuggest(c, n)
 
+proc testExamples(c: PContext) =
+  let inp = toFullPath(c.module.info)
+  let outp = inp.changeFileExt"" & "_examples.nim"
+  renderModule(c.runnableExamples, inp, outp)
+  let backend = if isDefined("js"): "js"
+                elif isDefined("cpp"): "cpp"
+                elif isDefined("objc"): "objc"
+                else: "c"
+  if os.execShellCmd("nim " & backend & " -r " & outp) != 0:
+    quit "[Examples] failed"
+  removeFile(outp)
+
 proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
   var c = PContext(context)
   if gCmd == cmdIdeTools and not c.suggestionsMade:
@@ -584,5 +596,6 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
     result.add(c.module.ast)
   popOwner(c)
   popProcCon(c)
+  if c.runnableExamples != nil: testExamples(c)
 
 const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
diff --git a/compiler/semdata.nim b/compiler/semdata.nim
index 5057260a4..8affee649 100644
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -136,6 +136,7 @@ type
       # the generic type has been constructed completely. See
       # tests/destructor/topttree.nim for an example that
       # would otherwise fail.
+    runnableExamples*: PNode
 
 proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair =
   result.genericSym = s
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index d600b1c48..380b367bc 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -1847,6 +1847,17 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
       analyseIfAddressTakenInCall(c, result)
       if callee.magic != mNone:
         result = magicsAfterOverloadResolution(c, result, flags)
+  of mRunnableExamples:
+    if gCmd == cmdDoc and n.len >= 2 and n.lastSon.kind == nkStmtList:
+      if sfMainModule in c.module.flags:
+        let inp = toFullPath(c.module.info)
+        if c.runnableExamples == nil:
+          c.runnableExamples = newTree(nkStmtList,
+            newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp))))
+        c.runnableExamples.add newTree(nkBlockStmt, emptyNode, n.lastSon)
+      result = n
+    else:
+      result = emptyNode
   else:
     result = semDirectOp(c, n, flags)
 
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index 1272affdc..f156c440b 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -46,7 +46,7 @@ type
     target*: OutputTarget
     config*: StringTableRef
     splitAfter*: int          # split too long entries in the TOC
-    listingCounter: int
+    listingCounter*: int
     tocPart*: seq[TocEntry]
     hasToc*: bool
     theIndex: string # Contents of the index file to be dumped at the end.
diff --git a/lib/system.nim b/lib/system.nim
index 1b53bf9f5..323ff00e6 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -3992,3 +3992,20 @@ when defined(windows) and appType == "console" and defined(nimSetUtf8CodePage):
   proc setConsoleOutputCP(codepage: cint): cint {.stdcall, dynlib: "kernel32",
     importc: "SetConsoleOutputCP".}
   discard setConsoleOutputCP(65001) # 65001 - utf-8 codepage
+
+
+when defined(nimHasRunnableExamples):
+  proc runnableExamples*(body: untyped) {.magic: "RunnableExamples".}
+    ## A section you should use to mark `runnable example`:idx: code with.
+    ##
+    ## - In normal debug and release builds code within
+    ##   a ``runnableExamples`` section is ignored.
+    ## - The documentation generator is aware of these examples and considers them
+    ##   part of the ``##`` doc comment. As the last step of documentation
+    ##   generation the examples are put into an ``$file_example.nim`` file,
+    ##   compiled and tested. The collected examples are
+    ##   put into their own module to ensure the examples do not refer to
+    ##   non-exported symbols.
+else:
+  template runnableExamples*(body: untyped) =
+    discard