From 76235348f8ccb1363100bdc686f0fafae5dacc5f Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 9 May 2012 01:50:08 +0200 Subject: extracted documentation generator --- packages/docutils/rst.nim | 17 +- packages/docutils/rstgen.nim | 608 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 611 insertions(+), 14 deletions(-) (limited to 'packages/docutils') diff --git a/packages/docutils/rst.nim b/packages/docutils/rst.nim index cd385ccac..2a497af9a 100755 --- a/packages/docutils/rst.nim +++ b/packages/docutils/rst.nim @@ -38,7 +38,8 @@ type meGeneralParseError, meInvalidDirective, mwRedefinitionOfLabel, - mwUnknownSubstitution + mwUnknownSubstitution, + mwUnsupportedLanguage TMsgHandler* = proc (filename: string, line, col: int, msgKind: TMsgKind, arg: string) ## what to do in case of an error @@ -53,7 +54,8 @@ const meGeneralParseError: "general parse error", meInvalidDirective: "invalid directive: '$1'", mwRedefinitionOfLabel: "redefinition of label '$1'", - mwUnknownSubstitution: "unknown substitution '$1'" + mwUnknownSubstitution: "unknown substitution '$1'", + mwUnsupportedLanguage: "language '$1' not supported" ] proc rstnodeToRefname*(n: PRstNode): string @@ -281,17 +283,6 @@ type EParseError* = object of EInvalidValue -when false: - proc tokInfo(p: TRstParser, tok: TToken): TLineInfo = - result = newLineInfo(p.filename, p.line + tok.line, p.col + tok.col) - - proc rstMessage(p: TRstParser, msgKind: TMsgKind, arg: string) = - GlobalError(tokInfo(p, p.tok[p.idx]), msgKind, arg) - - proc rstMessage(p: TRstParser, msgKind: TMsgKind) = - GlobalError(tokInfo(p, p.tok[p.idx]), msgKind, p.tok[p.idx].symbol) - - proc whichMsgClass*(k: TMsgKind): TMsgClass = ## returns which message class `k` belongs to. case ($k)[1] diff --git a/packages/docutils/rstgen.nim b/packages/docutils/rstgen.nim index 93544124f..c2a267005 100644 --- a/packages/docutils/rstgen.nim +++ b/packages/docutils/rstgen.nim @@ -9,14 +9,61 @@ ## This module implements a generator of HTML/Latex from `reStructuredText`:idx. -import strutils, strtabs, rstast +import strutils, os, hashes, strtabs, rstast, rst, highlite + +const + HtmlExt = "html" + IndexExt* = ".idx" type TOutputTarget* = enum ## which document type to generate outHtml, # output is HTML outLatex # output is Latex + + TTocEntry{.final.} = object + n*: PRstNode + refname*, header*: string + + TMetaEnum* = enum + metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion + TRstGenerator* = object of TObject + target*: TOutputTarget + config*: PStringTable + splitAfter*: int # split too long entries in the TOC + tocPart*: seq[TTocEntry] + hasToc*: bool + theIndex: string + options*: TRstParseOptions + findFile*: TFindFileHandler + msgHandler*: TMsgHandler + filename*: string + meta*: array[TMetaEnum, string] + + PDoc = var TRstGenerator +proc initRstGenerator*(g: var TRstGenerator, target: TOutputTarget, + config: PStringTable, filename: string, + options: TRstParseOptions, + findFile: TFindFileHandler, + msgHandler: TMsgHandler) = + g.config = config + g.target = target + g.tocPart = @[] + g.filename = filename + g.splitAfter = 20 + g.theIndex = "" + g.options = options + g.findFile = findFile + g.msgHandler = msgHandler + + let s = config["split.item.toc"] + if s != "": g.splitAfter = parseInt(s) + for i in low(g.meta)..high(g.meta): g.meta[i] = "" + +proc writeIndexFile*(g: var TRstGenerator, outfile: string) = + if g.theIndex.len > 0: writeFile(outfile, g.theIndex) + proc addXmlChar(dest: var string, c: Char) = case c of '&': add(dest, "&") @@ -84,4 +131,563 @@ proc esc*(target: TOutputTarget, s: string, splitAfter = -1): string = j = k + 1 else: for i in countup(0, len(s) - 1): escChar(target, result, s[i]) + + +proc disp(target: TOutputTarget, xml, tex: string): string = + if target != outLatex: result = xml + else: result = tex + +proc dispF(target: TOutputTarget, xml, tex: string, + args: openArray[string]): string = + if target != outLatex: result = xml % args + else: result = tex % args + +proc dispA(target: TOutputTarget, dest: var string, + xml, tex: string, args: openarray[string]) = + if target != outLatex: addf(dest, xml, args) + else: addf(dest, tex, args) + +proc renderRstToOut*(d: PDoc, n: PRstNode, result: var string) + +proc renderAux(d: PDoc, n: PRstNode, result: var string) = + for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], result) + +proc renderAux(d: PDoc, n: PRstNode, frmtA, frmtB: string, result: var string) = + var tmp = "" + for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], tmp) + if d.target != outLatex: + result.addf(frmtA, [tmp]) + else: + result.addf(frmtB, [tmp]) + +# ---------------- index handling -------------------------------------------- + +proc setIndexTerm*(d: PDoc, id, term: string) = + d.theIndex.add(term) + d.theIndex.add('\t') + let htmlFile = changeFileExt(extractFilename(d.filename), HtmlExt) + d.theIndex.add(htmlFile) + d.theIndex.add('#') + d.theIndex.add(id) + d.theIndex.add("\n") + +proc hash(n: PRstNode): int = + if n.kind == rnLeaf: + result = hash(n.text) + elif n.len > 0: + result = hash(n.sons[0]) + for i in 1 .. $2", "$2\\label{$1}", + [id, term]) + +type + TIndexEntry {.pure, final.} = object + keyword: string + link: string + +proc cmp(a, b: TIndexEntry): int = + result = cmpIgnoreStyle(a.keyword, b.keyword) + +proc `<-`(a: var TIndexEntry, b: TIndexEntry) = + shallowCopy a.keyword, b.keyword + shallowCopy a.link, b.link + +proc sortIndex(a: var openArray[TIndexEntry]) = + # we use shellsort here; fast and simple + let N = len(a) + var h = 1 + while true: + h = 3 * h + 1 + if h > N: break + while true: + h = h div 3 + for i in countup(h, N - 1): + var v: TIndexEntry + v <- a[i] + var j = i + while cmp(a[j-h], v) >= 0: + a[j] <- a[j-h] + j = j-h + if j < h: break + a[j] <- v + if h == 1: break + +proc mergeIndexes*(dir: string): string = + ## merges all index files in `dir` and returns the generated index as HTML. + ## The result is no full HTML for flexibility. + var a: seq[TIndexEntry] + newSeq(a, 15_000) + setLen(a, 0) + var L = 0 + for kind, path in walkDir(dir): + if kind == pcFile and path.endsWith(IndexExt): + for line in lines(path): + let s = line.find('\t') + if s < 0: continue + setLen(a, L+1) + a[L].keyword = line.substr(0, s-1) + a[L].link = line.substr(s+1) + inc L + sortIndex(a) + result = "" + var i = 0 + while i < L: + result.addf("
$1
\n") + i = j + +# ---------------------------------------------------------------------------- + +proc renderHeadline(d: PDoc, n: PRstNode, result: var string) = + var tmp = "" + for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp) + var refname = rstnodeToRefname(n) + if d.hasToc: + var length = len(d.tocPart) + setlen(d.tocPart, length + 1) + d.tocPart[length].refname = refname + d.tocPart[length].n = n + d.tocPart[length].header = tmp + + dispA(d.target, result, + "$3", + "\\rsth$4{$3}\\label{$2}\n", [$n.level, + d.tocPart[length].refname, tmp, + $chr(n.level - 1 + ord('A'))]) + else: + dispA(d.target, result, "$3", + "\\rsth$4{$3}\\label{$2}\n", [ + $n.level, refname, tmp, + $chr(n.level - 1 + ord('A'))]) + +proc renderOverline(d: PDoc, n: PRstNode, result: var string) = + if d.meta[metaTitle].len == 0: + for i in countup(0, len(n)-1): + renderRstToOut(d, n.sons[i], d.meta[metaTitle]) + elif d.meta[metaSubtitle].len == 0: + for i in countup(0, len(n)-1): + renderRstToOut(d, n.sons[i], d.meta[metaSubtitle]) + else: + var tmp = "" + for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp) + dispA(d.target, result, "
$3
", + "\\rstov$4{$3}\\label{$2}\n", [$n.level, + rstnodeToRefname(n), tmp, $chr(n.level - 1 + ord('A'))]) + + +proc renderTocEntry(d: PDoc, e: TTocEntry, result: var string) = + dispA(d.target, result, + "
  • $2
    • \n", + "\\item\\label{$1_toc} $2\\ref{$1}\n", [e.refname, e.header]) + +proc renderTocEntries*(d: PDoc, j: var int, lvl: int, result: var string) = + var tmp = "" + while j <= high(d.tocPart): + var a = abs(d.tocPart[j].n.level) + if a == lvl: + renderTocEntry(d, d.tocPart[j], tmp) + inc(j) + elif a > lvl: + renderTocEntries(d, j, a, tmp) + else: + break + if lvl > 1: + dispA(d.target, result, "", + "\\begin{enumerate}$1\\end{enumerate}", [tmp]) + else: + result.add(tmp) + +proc renderImage(d: PDoc, n: PRstNode, result: var string) = + var options = "" + var s = getFieldValue(n, "scale") + if s != "": dispA(d.target, options, " scale=\"$1\"", " scale=$1", [strip(s)]) + + s = getFieldValue(n, "height") + if s != "": dispA(d.target, options, " height=\"$1\"", " height=$1", [strip(s)]) + + s = getFieldValue(n, "width") + if s != "": dispA(d.target, options, " width=\"$1\"", " width=$1", [strip(s)]) + + s = getFieldValue(n, "alt") + if s != "": dispA(d.target, options, " alt=\"$1\"", "", [strip(s)]) + + s = getFieldValue(n, "align") + if s != "": dispA(d.target, options, " align=\"$1\"", "", [strip(s)]) + + if options.len > 0: options = dispF(d.target, "$1", "[$1]", [options]) + + dispA(d.target, result, "", "\\includegraphics$2{$1}", + [getArgument(n), options]) + if len(n) >= 3: renderRstToOut(d, n.sons[2], result) + +proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = + dispA(d.target, result, + """""", + "\\includegraphics{$1}", [n.text]) + +proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) = + if n.sons[2] == nil: return + var m = n.sons[2].sons[0] + assert m.kind == rnLeaf + var langstr = strip(getArgument(n)) + var lang: TSourceLanguage + if langstr == "": + lang = langNimrod # default language + else: + lang = getSourceLanguage(langstr) + + dispA(d.target, result, "
    ", "\\begin{rstpre}\n")
+  if lang == langNone:
+    d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, langstr)
+    result.add(m.text)
+  else:
+    var g: TGeneralTokenizer
+    initGeneralTokenizer(g, m.text)
+    while true: 
+      getNextToken(g, lang)
+      case g.kind
+      of gtEof: break 
+      of gtNone, gtWhitespace: 
+        add(result, substr(m.text, g.start, g.length + g.start - 1))
+      else:
+        dispA(d.target, result, "$1", "\\span$2{$1}", [
+          esc(d.target, substr(m.text, g.start, g.length+g.start-1)),
+          tokenClassToStr[g.kind]])
+    deinitGeneralTokenizer(g)
+  dispA(d.target, result, "
    ", "\n\\end{rstpre}\n") + +proc renderContainer(d: PDoc, n: PRstNode, result: var string) = + var tmp = "" + renderRstToOut(d, n.sons[2], tmp) + var arg = strip(getArgument(n)) + if arg == "": + dispA(d.target, result, "
    $1
    ", "$1", [tmp]) + else: + dispA(d.target, result, "
    $2
    ", "$2", [arg, tmp]) + +proc texColumns(n: PRstNode): string = + result = "" + for i in countup(1, len(n)): add(result, "|X") + +proc renderField(d: PDoc, n: PRstNode, result: var string) = + var b = false + if d.target == outLatex: + var fieldname = addNodes(n.sons[0]) + var fieldval = esc(d.target, strip(addNodes(n.sons[1]))) + if cmpIgnoreStyle(fieldname, "author") == 0: + if d.meta[metaAuthor].len == 0: + d.meta[metaAuthor] = fieldval + b = true + elif cmpIgnoreStyle(fieldName, "version") == 0: + if d.meta[metaVersion].len == 0: + d.meta[metaVersion] = fieldval + b = true + if not b: + renderAux(d, n, "$1\n", "$1", result) + +proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = + if n == nil: return + case n.kind + of rnInner: renderAux(d, n, result) + of rnHeadline: renderHeadline(d, n, result) + of rnOverline: renderOverline(d, n, result) + of rnTransition: renderAux(d, n, "
    \n", "\\hrule\n", result) + of rnParagraph: renderAux(d, n, "

    $1

    \n", "$1\n\n", result) + of rnBulletList: + renderAux(d, n, "\n", + "\\begin{itemize}$1\\end{itemize}\n", result) + of rnBulletItem, rnEnumItem: + renderAux(d, n, "
  • $1
    • \n", "\\item $1\n", result) + of rnEnumList: + renderAux(d, n, "
      $1
    \n", + "\\begin{enumerate}$1\\end{enumerate}\n", result) + of rnDefList: + renderAux(d, n, "
    $1
    \n", + "\\begin{description}$1\\end{description}\n", result) + of rnDefItem: renderAux(d, n, result) + of rnDefName: renderAux(d, n, "
    $1
    \n", "\\item[$1] ", result) + of rnDefBody: renderAux(d, n, "
    $1
    \n", "$1\n", result) + of rnFieldList: + var tmp = "" + for i in countup(0, len(n) - 1): + renderRstToOut(d, n.sons[i], tmp) + if tmp.len != 0: + dispA(d.target, result, + "" & + "" & + "" & + "$1" & + "
    ", + "\\begin{description}$1\\end{description}\n", + [tmp]) + of rnField: renderField(d, n, result) + of rnFieldName: + renderAux(d, n, "$1:", "\\item[$1:]", result) + of rnFieldBody: + renderAux(d, n, "$1", " $1\n", result) + of rnIndex: + renderRstToOut(d, n.sons[2], result) + of rnOptionList: + renderAux(d, n, "$1
    ", + "\\begin{description}\n$1\\end{description}\n", result) + of rnOptionListItem: + renderAux(d, n, "$1\n", "$1", result) + of rnOptionGroup: + renderAux(d, n, "$1", "\\item[$1]", result) + of rnDescription: + renderAux(d, n, "$1\n", " $1\n", result) + of rnOption, rnOptionString, rnOptionArgument: + doAssert false, "renderRstToOut" + of rnLiteralBlock: + renderAux(d, n, "
    $1
    \n", + "\\begin{rstpre}\n$1\n\\end{rstpre}\n", result) + of rnQuotedLiteralBlock: + doAssert false, "renderRstToOut" + of rnLineBlock: + renderAux(d, n, "

    $1

    ", "$1\n\n", result) + of rnLineBlockItem: + renderAux(d, n, "$1
    ", "$1\\\\\n", result) + of rnBlockQuote: + renderAux(d, n, "

    $1

    \n", + "\\begin{quote}$1\\end{quote}\n", result) + of rnTable, rnGridTable: + renderAux(d, n, + "$1
    ", + "\\begin{table}\\begin{rsttab}{" & + texColumns(n) & "|}\n\\hline\n$1\\end{rsttab}\\end{table}", result) + of rnTableRow: + if len(n) >= 1: + if d.target == outLatex: + var tmp = "" + renderRstToOut(d, n.sons[0], tmp) + for i in countup(1, len(n) - 1): + result.add(" & ") + renderRstToOut(d, n.sons[i], result) + result.add("\\\\\n\\hline\n") + else: + result.add("") + renderAux(d, n, result) + result.add("\n") + of rnTableDataCell: + renderAux(d, n, "$1", "$1", result) + of rnTableHeaderCell: + renderAux(d, n, "$1", "\\textbf{$1}", result) + of rnLabel: + doAssert false, "renderRstToOut" # used for footnotes and other + of rnFootnote: + doAssert false, "renderRstToOut" # a footnote + of rnCitation: + doAssert false, "renderRstToOut" # similar to footnote + of rnRef: + var tmp = "" + renderAux(d, n, tmp) + dispA(d.target, result, "$1", + "$1\\ref{$2}", [tmp, rstnodeToRefname(n)]) + of rnStandaloneHyperlink: + renderAux(d, n, + "$1", + "\\href{$1}{$1}", result) + of rnHyperlink: + var tmp0 = "" + var tmp1 = "" + renderRstToOut(d, n.sons[0], tmp0) + renderRstToOut(d, n.sons[1], tmp1) + dispA(d.target, result, "$1", + "\\href{$2}{$1}", + [tmp0, tmp1]) + of rnDirArg, rnRaw: renderAux(d, n, result) + of rnRawHtml: + if d.target != outLatex: + result.add addNodes(lastSon(n)) + of rnRawLatex: + if d.target == outLatex: + result.add addNodes(lastSon(n)) + + of rnImage, rnFigure: renderImage(d, n, result) + of rnCodeBlock: renderCodeBlock(d, n, result) + of rnContainer: renderContainer(d, n, result) + of rnSubstitutionReferences, rnSubstitutionDef: + renderAux(d, n, "|$1|", "|$1|", result) + of rnDirective: + renderAux(d, n, "", "", result) + of rnGeneralRole: + var tmp0 = "" + var tmp1 = "" + renderRstToOut(d, n.sons[0], tmp0) + renderRstToOut(d, n.sons[1], tmp1) + dispA(d.target, result, "$1", "\\span$2{$1}", + [tmp0, tmp1]) + of rnSub: renderAux(d, n, "$1", "\\rstsub{$1}", result) + of rnSup: renderAux(d, n, "$1", "\\rstsup{$1}", result) + of rnEmphasis: renderAux(d, n, "$1", "\\emph{$1}", result) + of rnStrongEmphasis: + renderAux(d, n, "$1", "\\textbf{$1}", result) + of rnTripleEmphasis: + renderAux(d, n, "$1", + "\\textbf{emph{$1}}", result) + of rnInterpretedText: + renderAux(d, n, "$1", "\\emph{$1}", result) + of rnIdx: + renderIndexTerm(d, n, result) + of rnInlineLiteral: + renderAux(d, n, + "$1", + "\\texttt{$1}", result) + of rnSmiley: renderSmiley(d, n, result) + of rnLeaf: result.add(esc(d.target, n.text)) + of rnContents: d.hasToc = true + of rnTitle: + d.meta[metaTitle] = "" + renderRstToOut(d, n.sons[0], d.meta[metaTitle]) + +# ----------------------------------------------------------------------------- + +proc getVarIdx(varnames: openarray[string], id: string): int = + for i in countup(0, high(varnames)): + if cmpIgnoreStyle(varnames[i], id) == 0: + return i + result = -1 + +proc formatNamedVars*(frmt: string, varnames: openarray[string], + varvalues: openarray[string]): string = + var i = 0 + var L = len(frmt) + result = "" + var num = 0 + while i < L: + if frmt[i] == '$': + inc(i) # skip '$' + case frmt[i] + of '#': + add(result, varvalues[num]) + inc(num) + inc(i) + of '$': + add(result, "$") + inc(i) + of '0'..'9': + var j = 0 + while true: + j = (j * 10) + Ord(frmt[i]) - ord('0') + inc(i) + if i > L-1 or frmt[i] notin {'0'..'9'}: break + if j > high(varvalues) + 1: + raise newException(EInvalidValue, "invalid index: " & $j) + num = j + add(result, varvalues[j - 1]) + of 'A'..'Z', 'a'..'z', '\x80'..'\xFF': + var id = "" + while true: + add(id, frmt[i]) + inc(i) + if frmt[i] notin {'A'..'Z', '_', 'a'..'z', '\x80'..'\xFF'}: break + var idx = getVarIdx(varnames, id) + if idx >= 0: + add(result, varvalues[idx]) + else: + raise newException(EInvalidValue, "unknown substitution var: " & id) + of '{': + var id = "" + inc(i) + while frmt[i] != '}': + if frmt[i] == '\0': + raise newException(EInvalidValue, "'}' expected") + add(id, frmt[i]) + inc(i) + inc(i) # skip } + # search for the variable: + var idx = getVarIdx(varnames, id) + if idx >= 0: add(result, varvalues[idx]) + else: + raise newException(EInvalidValue, "unknown substitution var: " & id) + else: + raise newException(EInvalidValue, "unknown substitution: $" & $frmt[i]) + var start = i + while i < L: + if frmt[i] != '$': inc(i) + else: break + if i-1 >= start: add(result, substr(frmt, start, i - 1)) + + +proc defaultConfig*(): PStringTable = + ## creates a default configuration for HTML generation. + result = newStringTable(modeStyleInsensitive) + + template setConfigVar(key, val: expr) = + result[key] = val + + setConfigVar("split.item.toc", "20") + setConfigVar("doc.section", """ +
    +

    $sectionTitle

    +
    +$content +
    +""") + setConfigVar("doc.section.toc", """ +
  • + $sectionTitle +
      + $content +
    +
    • +""") + setConfigVar("doc.item", """ +
    $header
    +
    +$desc +
    +""") + setConfigVar("doc.item.toc", """ +
  • $name
    • +""") + setConfigVar("doc.toc", """ +""") + setConfigVar("doc.body_toc", """ +$tableofcontents +
    +$moduledesc +$content +
    +""") + setConfigVar("doc.body_no_toc", "$moduledesc $content") + setConfigVar("doc.file", "$content") + +# ---------- forum --------------------------------------------------------- + +proc rstToHtml*(s: string, options: TRstParseOptions, + config: PStringTable): string = + ## exported for *nimforum*. + + proc myFindFile(filename: string): string = + # we don't find any files in online mode: + result = "" + + const filen = "input" + var d: TRstGenerator + initRstGenerator(d, outHtml, config, filen, options, myFindFile, nil) + var dummyHasToc = false + var rst = rstParse(s, filen, 0, 1, dummyHasToc, options) + result = "" + renderRstToOut(d, rst, result) \ No newline at end of file -- cgit 1.4.1-2-gfad0