Merge branch 'pr_adds_code_directive_to_rst' into pr_bigbreak_adds_code_directive_to_rst

Conflicts:
	doc/tut2.txt
	lib/packages/docutils/rstgen.nim
	lib/system.nim

2 files changed, 178 insertions, 25 deletions

diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index b21d51c93..07014c754 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -39,7 +39,8 @@ type
     meInvalidDirective,
     mwRedefinitionOfLabel,
     mwUnknownSubstitution,
-    mwUnsupportedLanguage
+    mwUnsupportedLanguage,
+    mwUnsupportedField
   
   TMsgHandler* = proc (filename: string, line, col: int, msgKind: TMsgKind,
                        arg: string) {.nimcall.} ## what to do in case of an error
@@ -55,7 +56,8 @@ const
     meInvalidDirective: "invalid directive: '$1'",
     mwRedefinitionOfLabel: "redefinition of label '$1'", 
     mwUnknownSubstitution: "unknown substitution '$1'",
-    mwUnsupportedLanguage: "language '$1' not supported"
+    mwUnsupportedLanguage: "language '$1' not supported",
+    mwUnsupportedField: "field '$1' not supported"
   ]
 
 proc rstnodeToRefname*(n: PRstNode): string
@@ -850,13 +852,13 @@ proc parseComment(p: var TRstParser): PRstNode =
 
 type 
   TDirKind = enum             # must be ordered alphabetically!
-    dkNone, dkAuthor, dkAuthors, dkCodeBlock, dkContainer, dkContents,
+    dkNone, dkAuthor, dkAuthors, dkCode, dkCodeBlock, dkContainer, dkContents,
     dkFigure, dkImage, dkInclude, dkIndex, dkRaw, dkTitle
 
 const 
-  DirIds: array[0..11, string] = ["", "author", "authors", "code-block", 
-    "container", "contents", "figure", "image", "include", "index", "raw", 
-    "title"]
+  DirIds: array[0..12, string] = ["", "author", "authors", "code",
+    "code-block", "container", "contents", "figure", "image", "include",
+    "index", "raw", "title"]
 
 proc getDirKind(s: string): TDirKind = 
   let i = find(DirIds, s)
@@ -876,7 +878,10 @@ proc parseUntilNewline(p: var TRstParser, father: PRstNode) =
     of tkEof, tkIndent: break
   
 proc parseSection(p: var TRstParser, result: PRstNode)
-proc parseField(p: var TRstParser): PRstNode = 
+proc parseField(p: var TRstParser): PRstNode =
+  ## Returns a parsed rnField node.
+  ##
+  ## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody.
   result = newRstNode(rnField)
   var col = p.tok[p.idx].col
   var fieldname = newRstNode(rnFieldName)
@@ -892,7 +897,11 @@ proc parseField(p: var TRstParser): PRstNode =
   add(result, fieldname)
   add(result, fieldbody)
 
-proc parseFields(p: var TRstParser): PRstNode = 
+proc parseFields(p: var TRstParser): PRstNode =
+  ## Parses fields for a section or directive block.
+  ##
+  ## This proc may return nil if the parsing doesn't find anything of value,
+  ## otherwise it will return a node of rnFieldList type with children.
   result = nil
   var atStart = p.idx == 0 and p.tok[0].symbol == ":"
   if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx + 1].symbol == ":") or
@@ -908,6 +917,18 @@ proc parseFields(p: var TRstParser): PRstNode =
       else: 
         break 
   
+proc getFieldValue*(n: PRstNode): string =
+  ## Returns the value of a specific ``rnField`` node.
+  ##
+  ## This proc will assert if the node is not of the expected type. The empty
+  ## string will be returned as a minimum. Any value in the rst will be
+  ## stripped form leading/trailing whitespace.
+  assert n.kind == rnField
+  assert n.len == 2
+  assert n.sons[0].kind == rnFieldName
+  assert n.sons[1].kind == rnFieldBody
+  result = addNodes(n.sons[1]).strip
+
 proc getFieldValue(n: PRstNode, fieldname: string): string = 
   result = ""
   if n.sons[1] == nil: return 
@@ -1387,7 +1408,16 @@ type
   TDirFlags = set[TDirFlag]
   TSectionParser = proc (p: var TRstParser): PRstNode {.nimcall.}
 
-proc parseDirective(p: var TRstParser, flags: TDirFlags): PRstNode = 
+proc parseDirective(p: var TRstParser, flags: TDirFlags): PRstNode =
+  ## Parses arguments and options for a directive block.
+  ##
+  ## A directive block will always have three sons: the arguments for the
+  ## directive (rnDirArg), the options (rnFieldList) and the block
+  ## (rnLineBlock). This proc parses the two first nodes, the block is left to
+  ## the outer `parseDirective` call.
+  ##
+  ## Both rnDirArg and rnFieldList children nodes might be nil, so you need to
+  ## check them before accessing.
   result = newRstNode(rnDirective)
   var args: PRstNode = nil
   var options: PRstNode = nil
@@ -1421,6 +1451,9 @@ proc indFollows(p: TRstParser): bool =
   
 proc parseDirective(p: var TRstParser, flags: TDirFlags, 
                     contentParser: TSectionParser): PRstNode = 
+  ## Returns a generic rnDirective tree.
+  ##
+  ## The children are rnDirArg, rnFieldList and rnLineBlock. Any might be nil.
   result = parseDirective(p, flags)
   if not isNil(contentParser) and indFollows(p): 
     pushInd(p, p.tok[p.idx].ival)
@@ -1474,7 +1507,23 @@ proc dirInclude(p: var TRstParser): PRstNode =
       #  InternalError("Too many binary zeros in include file")
       result = parseDoc(q)
 
-proc dirCodeBlock(p: var TRstParser): PRstNode = 
+proc dirCodeBlock(p: var TRstParser, nimrodExtension = false): PRstNode =
+  ## Parses a code block.
+  ##
+  ## Code blocks are rnDirective trees with a `kind` of rnCodeBlock. See the
+  ## description of ``parseDirective`` for further structure information.
+  ##
+  ## Code blocks can come in two forms, the standard `code directive
+  ## <http://docutils.sourceforge.net/docs/ref/rst/directives.html#code>`_ and
+  ## the nimrod extension ``.. code-block::``. If the block is an extension, we
+  ## want the default language syntax highlighting to be Nimrod, so we create a
+  ## fake internal field to comminicate with the generator. The field is named
+  ## ``default-language``, which is unlikely to collide with a field specified
+  ## by any random rst input file.
+  ##
+  ## As an extension this proc will process the ``file`` extension field and if
+  ## present will replace the code block with the contents of the referenced
+  ## file.
   result = parseDirective(p, {hasArg, hasOptions}, parseLiteralBlock)
   var filename = strip(getFieldValue(result, "file"))
   if filename != "": 
@@ -1483,6 +1532,20 @@ proc dirCodeBlock(p: var TRstParser): PRstNode =
     var n = newRstNode(rnLiteralBlock)
     add(n, newRstNode(rnLeaf, readFile(path)))
     result.sons[2] = n
+
+  # Extend the field block if we are using our custom extension.
+  if nimrodExtension:
+    # Create a field block if the input block didn't have any.
+    if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList)
+    assert result.sons[1].kind == rnFieldList
+    # Hook the extra field and specify the Nimrod language as value.
+    var extraNode = newRstNode(rnField)
+    extraNode.add(newRstNode(rnFieldName))
+    extraNode.add(newRstNode(rnFieldBody))
+    extraNode.sons[0].add(newRstNode(rnLeaf, "default-language"))
+    extraNode.sons[1].add(newRstNode(rnLeaf, "Nimrod"))
+    result.sons[1].add(extraNode)
+
   result.kind = rnCodeBlock
 
 proc dirContainer(p: var TRstParser): PRstNode = 
@@ -1566,7 +1629,8 @@ proc parseDotDot(p: var TRstParser): PRstNode =
         result = dirRaw(p)
       else:
         rstMessage(p, meInvalidDirective, d)
-    of dkCodeBlock: result = dirCodeBlock(p)
+    of dkCode: result = dirCodeBlock(p)
+    of dkCodeBlock: result = dirCodeBlock(p, nimrodExtension = true)
     of dkIndex: result = dirIndex(p)
     else: rstMessage(p, meInvalidDirective, d)
     popInd(p)
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index 02b0afd2f..cfbc4a708 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -24,7 +24,7 @@
 ## generate `LaTeX documents <https://en.wikipedia.org/wiki/LaTeX>`_ too.
 
 import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils,
-  algorithm
+  algorithm, parseutils
 
 const
   HtmlExt = "html"
@@ -63,6 +63,19 @@ type
   
   PDoc = var TRstGenerator ## Alias to type less.
 
+  CodeBlockParams = object ## Stores code block params.
+    numberLines: bool ## True if the renderer has to show line numbers.
+    startLine: int ## The starting line of the code block, by default 1.
+    langStr: string ## Input string used to specify the language.
+    lang: TSourceLanguage ## Type of highlighting, by default none.
+
+
+proc init(p: var CodeBlockParams) =
+  ## Default initialisation of CodeBlockParams to sane values.
+  p.startLine = 1
+  p.lang = langNone
+  p.langStr = ""
+
 proc initRstGenerator*(g: var TRstGenerator, target: TOutputTarget,
                        config: StringTableRef, filename: string,
                        options: TRstParseOptions,
@@ -761,26 +774,102 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) =
         height="17" hspace="2" vspace="2" />""",
     "\\includegraphics{$1}", [n.text])
   
+proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) =
+  ## Parses useful fields which can appear before a code block.
+  ##
+  ## This supports the special ``default-language`` internal string generated
+  ## by the ``rst`` module to communicate a specific default language.
+  case n.getArgument.toLower
+  of "number-lines":
+    params.numberLines = true
+    # See if the field has a parameter specifying a different line than 1.
+    var number: int
+    if parseInt(n.getFieldValue, number) > 0:
+      params.startLine = number
+  of "file":
+    # The ``file`` option is a Nimrod extension to the official spec, it acts
+    # like it would for other directives like ``raw`` or ``cvs-table``. This
+    # field is dealt with in ``rst.nim`` which replaces the existing block with
+    # the referenced file, so we only need to ignore it here to avoid incorrect
+    # warning messages.
+    discard
+  of "default-language":
+    params.langStr = n.getFieldValue.strip
+    params.lang = params.langStr.getSourceLanguage
+  else:
+    d.msgHandler(d.filename, 1, 0, mwUnsupportedField, n.getArgument)
+
+proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams =
+  ## Iterates over all code block fields and returns processed params.
+  ##
+  ## Also processes the argument of the directive as the default language. This
+  ## is done last so as to override any internal communication field variables.
+  result.init
+  if n.isNil:
+    return
+  assert n.kind == rnCodeBlock
+  assert(not n.sons[2].isNil)
+
+  # Parse the field list for rendering parameters if there are any.
+  if not n.sons[1].isNil:
+    for son in n.sons[1].sons: d.parseCodeBlockField(son, result)
+
+  # Parse the argument and override the language.
+  result.langStr = strip(getArgument(n))
+  if result.langStr != "":
+    result.lang = getSourceLanguage(result.langStr)
+
+proc buildLinesHTMLTable(params: CodeBlockParams, code: string):
+    tuple[beginTable, endTable: string] =
+  ## Returns the necessary tags to start/end a code block in HTML.
+  ##
+  ## If the numberLines has not been used, the tags will default to a simple
+  ## <pre> pair. Otherwise it will build a table and insert an initial column
+  ## with all the line numbers, which requires you to pass the `code` to detect
+  ## how many lines have to be generated (and starting at which point!).
+  if not params.numberLines:
+    result = ("<pre>", "</pre>")
+    return
+
+  var codeLines = 1 + code.strip.countLines
+  assert codeLines > 0
+  result.beginTable = """<table><tbody><tr><td class="blob-line-nums"><pre>"""
+  var line = params.startLine
+  while codeLines > 0:
+    result.beginTable.add($line & "\n")
+    line.inc
+    codeLines.dec
+  result.beginTable.add("</pre></td><td><pre>")
+  result.endTable = "</pre></td></tr></tbody></table>"
+
 proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
+  ## Renders a code block, appending it to `result`.
+  ##
+  ## If the code block uses the ``number-lines`` option, a table will be
+  ## generated with two columns, the first being a list of numbers and the
+  ## second the code block itself. The code block can use syntax highlighting,
+  ## which depends on the directive argument specified by the rst input, and
+  ## may also come from the parser through the internal ``default-language``
+  ## option to differentiate between a plain code block and nimrod's code block
+  ## extension.
+  assert n.kind == rnCodeBlock
   if n.sons[2] == nil: return
+  var params = d.parseCodeBlockParams(n)
   var m = n.sons[2].sons[0]
   assert m.kind == rnLeaf
-  var langstr = strip(getArgument(n))
-  var lang: TSourceLanguage
-  if langstr == "":
-    lang = langNim         # default language
-  else:
-    lang = getSourceLanguage(langstr)
-  
-  dispA(d.target, result, "<pre>", "\\begin{rstpre}\n", [])
-  if lang == langNone:
-    d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, langstr)
+
+  let (blockStart, blockEnd) = params.buildLinesHTMLTable(m.text)
+
+  dispA(d.target, result, blockStart, "\\begin{rstpre}\n", [])
+  if params.lang == langNone:
+    if len(params.langStr) > 0:
+      d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr)
     for letter in m.text: escChar(d.target, result, letter)
   else:
     var g: TGeneralTokenizer
     initGeneralTokenizer(g, m.text)
     while true: 
-      getNextToken(g, lang)
+      getNextToken(g, params.lang)
       case g.kind
       of gtEof: break 
       of gtNone, gtWhitespace: 
@@ -790,8 +879,8 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
           esc(d.target, substr(m.text, g.start, g.length+g.start-1)),
           tokenClassToStr[g.kind]])
     deinitGeneralTokenizer(g)
-  dispA(d.target, result, "</pre>", "\n\\end{rstpre}\n")
-  
+  dispA(d.target, result, blockEnd, "\n\\end{rstpre}\n")
+
 proc renderContainer(d: PDoc, n: PRstNode, result: var string) = 
   var tmp = ""
   renderRstToOut(d, n.sons[2], tmp)