diff options
Diffstat (limited to 'compiler')
| -rwxr-xr-x | compiler/docgen.nim | 320 | ||||
| -rwxr-xr-x | compiler/highlite.nim | 531 | ||||
| -rwxr-xr-x | compiler/lexer.nim | 2 | ||||
| -rwxr-xr-x | compiler/msgs.nim | 2 | ||||
| -rwxr-xr-x | compiler/nimrod.cfg | 2 | ||||
| -rwxr-xr-x | compiler/nimrod.ini | 12 | ||||
| -rwxr-xr-x | compiler/options.nim | 2 | ||||
| -rwxr-xr-x | compiler/rst.nim | 1718 |
8 files changed, 100 insertions, 2489 deletions
diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 2bcb397c3..50abcfc4e 100755 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -13,11 +13,14 @@ import ast, astalgo, strutils, hashes, options, nversion, msgs, os, ropes, idents, - wordrecg, math, syntaxes, renderer, lexer, rst, times, highlite, importer + wordrecg, syntaxes, renderer, lexer, rstast, rst, rstgen, times, highlite, + importer proc CommandDoc*() proc CommandRst2Html*() proc CommandRst2TeX*() + +#proc CommandBuildIndex*() # implementation type @@ -29,9 +32,9 @@ type TMetaEnum = enum metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion TDocumentor {.final.} = object # contains a module's documentation + target: TOutputTarget options: TRstParseOptions filename*: string # filename of the source file; without extension - basedir*: string # base directory (where to put the documentation) modDesc*: PRope # module description id*: int # for generating IDs splitAfter*: int # split too long entries in the TOC @@ -40,13 +43,10 @@ type toc*, section*: TSections indexFile*, theIndex*: PRstNode indexValFilename*: string - indent*, verbatim*: int # for code generation meta*: array[TMetaEnum, PRope] PDoc = ref TDocumentor -var splitter: string = "<wbr />" - proc findIndexNode(n: PRstNode): PRstNode = if n == nil: result = nil @@ -59,10 +59,31 @@ proc findIndexNode(n: PRstNode): PRstNode = result = result.sons[0] else: result = nil - for i in countup(0, rsonsLen(n) - 1): + for i in countup(0, len(n) - 1): result = findIndexNode(n.sons[i]) if result != nil: return +proc compilerMsgHandler(filename: string, line, col: int, + msgKind: rst.TMsgKind, arg: string) {.procvar.} = + # translate msg kind: + var k: msgs.TMsgKind + case msgKind + of meCannotOpenFile: k = errCannotOpenFile + of meExpected: k = errXExpected + of meGridTableNotImplemented: k = errGridTableNotImplemented + of meNewSectionExpected: k = errNewSectionExpected + of meGeneralParseError: k = errGeneralParseError + of meInvalidDirective: k = errInvalidDirectiveX + of mwRedefinitionOfLabel: k = warnRedefinitionOfLabel + of mwUnknownSubstitution: k = warnUnknownSubstitutionX + GlobalError(newLineInfo(filename, line, col), k, arg) + +proc parseRst(text, filename: string, + line, column: int, hasToc: var bool, + rstOptions: TRstParseOptions): PRstNode = + result = rstParse(text, filename, line, column, hasToc, rstOptions, + options.FindFile, compilerMsgHandler) + proc initIndexFile(d: PDoc) = var h: PRstNode @@ -71,7 +92,7 @@ proc initIndexFile(d: PDoc) = gIndexFile = addFileExt(gIndexFile, "txt") d.indexValFilename = changeFileExt(extractFilename(d.filename), HtmlExt) if ExistsFile(gIndexFile): - d.indexFile = rstParse(readFile(gIndexFile), gIndexFile, 0, 1, + d.indexFile = parseRst(readFile(gIndexFile), gIndexFile, 0, 1, dummyHasToc, {roSupportRawDirective}) d.theIndex = findIndexNode(d.indexFile) if (d.theIndex == nil) or (d.theIndex.kind != rnDefList): @@ -81,17 +102,19 @@ proc initIndexFile(d: PDoc) = d.indexFile = newRstNode(rnInner) h = newRstNode(rnOverline) h.level = 1 - addSon(h, newRstNode(rnLeaf, "Index")) - addSon(d.indexFile, h) + add(h, newRstNode(rnLeaf, "Index")) + add(d.indexFile, h) h = newRstNode(rnIndex) - addSon(h, nil) # no argument - addSon(h, nil) # no options + add(h, nil) # no argument + add(h, nil) # no options d.theIndex = newRstNode(rnDefList) - addSon(h, d.theIndex) - addSon(d.indexFile, h) + add(h, d.theIndex) + add(d.indexFile, h) proc newDocumentor(filename: string): PDoc = new(result) + if gCmd != cmdRst2Tex: result.target = outHtml + else: result.target = outLatex result.tocPart = @[] result.filename = filename result.id = 100 @@ -159,72 +182,7 @@ proc ropeFormatNamedVars(frmt: TFormatStr, varnames: openarray[string], if (frmt[i] != '$'): inc(i) else: break if i - 1 >= start: app(result, substr(frmt, start, i - 1)) - -proc addXmlChar(dest: var string, c: Char) = - case c - of '&': add(dest, "&") - of '<': add(dest, "<") - of '>': add(dest, ">") - of '\"': add(dest, """) - else: add(dest, c) - -proc addRtfChar(dest: var string, c: Char) = - case c - of '{': add(dest, "\\{") - of '}': add(dest, "\\}") - of '\\': add(dest, "\\\\") - else: add(dest, c) - -proc addTexChar(dest: var string, c: Char) = - case c - of '_': add(dest, "\\_") - of '{': add(dest, "\\symbol{123}") - of '}': add(dest, "\\symbol{125}") - of '[': add(dest, "\\symbol{91}") - of ']': add(dest, "\\symbol{93}") - of '\\': add(dest, "\\symbol{92}") - of '$': add(dest, "\\$") - of '&': add(dest, "\\&") - of '#': add(dest, "\\#") - of '%': add(dest, "\\%") - of '~': add(dest, "\\symbol{126}") - of '@': add(dest, "\\symbol{64}") - of '^': add(dest, "\\symbol{94}") - of '`': add(dest, "\\symbol{96}") - else: add(dest, c) - -proc escChar(dest: var string, c: Char) = - if gCmd != cmdRst2Tex: addXmlChar(dest, c) - else: addTexChar(dest, c) - -proc nextSplitPoint(s: string, start: int): int = - result = start - while result < len(s) + 0: - case s[result] - of '_': return - of 'a'..'z': - if result + 1 < len(s) + 0: - if s[result + 1] in {'A'..'Z'}: return - else: nil - inc(result) - dec(result) # last valid index - -proc esc(s: string, splitAfter: int = - 1): string = - result = "" - if splitAfter >= 0: - var partLen = 0 - var j = 0 - while j < len(s): - var k = nextSplitPoint(s, j) - if (splitter != " ") or (partLen + k - j + 1 > splitAfter): - partLen = 0 - add(result, splitter) - for i in countup(j, k): escChar(result, s[i]) - inc(partLen, k - j + 1) - j = k + 1 - else: - for i in countup(0, len(s) + 0 - 1): escChar(result, s[i]) - + proc disp(xml, tex: string): string = if gCmd != cmdRst2Tex: result = xml else: result = tex @@ -241,17 +199,17 @@ proc renderRstToOut(d: PDoc, n: PRstNode): PRope proc renderAux(d: PDoc, n: PRstNode, outer: string = "$1"): PRope = result = nil - for i in countup(0, rsonsLen(n) - 1): app(result, renderRstToOut(d, n.sons[i])) + for i in countup(0, len(n) - 1): app(result, renderRstToOut(d, n.sons[i])) result = ropef(outer, [result]) proc setIndexForSourceTerm(d: PDoc, name: PRstNode, id: int) = if d.theIndex == nil: return var h = newRstNode(rnHyperlink) var a = newRstNode(rnLeaf, d.indexValFilename & disp("#", "") & $id) - addSon(h, a) - addSon(h, a) + add(h, a) + add(h, a) a = newRstNode(rnIdx) - addSon(a, name) + add(a, name) setIndexPair(d.theIndex, a, h) proc renderIndexTerm(d: PDoc, n: PRstNode): PRope = @@ -260,14 +218,14 @@ proc renderIndexTerm(d: PDoc, n: PRstNode): PRope = [toRope(d.id), renderAux(d, n)]) var h = newRstNode(rnHyperlink) var a = newRstNode(rnLeaf, d.indexValFilename & disp("#", "") & $d.id) - addSon(h, a) - addSon(h, a) + add(h, a) + add(h, a) setIndexPair(d.theIndex, n, h) proc genComment(d: PDoc, n: PNode): PRope = var dummyHasToc: bool if n.comment != nil and startsWith(n.comment, "##"): - result = renderRstToOut(d, rstParse(n.comment, toFilename(n.info), + result = renderRstToOut(d, parseRst(n.comment, toFilename(n.info), toLineNumber(n.info), toColumn(n.info), dummyHasToc, d.options + {roSkipPounds})) @@ -294,16 +252,16 @@ proc isVisible(n: PNode): bool = elif n.kind == nkPragmaExpr: result = isVisible(n.sons[0]) -proc getName(n: PNode, splitAfter: int = - 1): string = +proc getName(d: PDoc, n: PNode, splitAfter: int = - 1): string = case n.kind - of nkPostfix: result = getName(n.sons[1], splitAfter) - of nkPragmaExpr: result = getName(n.sons[0], splitAfter) - of nkSym: result = esc(n.sym.name.s, splitAfter) - of nkIdent: result = esc(n.ident.s, splitAfter) + of nkPostfix: result = getName(d, n.sons[1], splitAfter) + of nkPragmaExpr: result = getName(d, n.sons[0], splitAfter) + of nkSym: result = esc(d.target, n.sym.name.s, splitAfter) + of nkIdent: result = esc(d.target, n.ident.s, splitAfter) of nkAccQuoted: - result = esc("`") - for i in 0.. <n.len: result.add(getName(n[i], splitAfter)) - result.add esc("`") + result = esc(d.target, "`") + for i in 0.. <n.len: result.add(getName(d, n[i], splitAfter)) + result.add esc(d.target, "`") else: internalError(n.info, "getName()") result = "" @@ -323,7 +281,7 @@ proc getRstName(n: PNode): PRstNode = proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = if not isVisible(nameNode): return - var name = toRope(getName(nameNode)) + var name = toRope(getName(d, nameNode)) var result: PRope = nil var literal = "" var kind = tkEof @@ -338,28 +296,28 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = break of tkComment: dispA(result, "<span class=\"Comment\">$1</span>", "\\spanComment{$1}", - [toRope(esc(literal))]) + [toRope(esc(d.target, literal))]) of tokKeywordLow..tokKeywordHigh: dispA(result, "<span class=\"Keyword\">$1</span>", "\\spanKeyword{$1}", [toRope(literal)]) of tkOpr: dispA(result, "<span class=\"Operator\">$1</span>", "\\spanOperator{$1}", - [toRope(esc(literal))]) + [toRope(esc(d.target, literal))]) of tkStrLit..tkTripleStrLit: dispA(result, "<span class=\"StringLit\">$1</span>", - "\\spanStringLit{$1}", [toRope(esc(literal))]) + "\\spanStringLit{$1}", [toRope(esc(d.target, literal))]) of tkCharLit: dispA(result, "<span class=\"CharLit\">$1</span>", "\\spanCharLit{$1}", - [toRope(esc(literal))]) + [toRope(esc(d.target, literal))]) of tkIntLit..tkInt64Lit: dispA(result, "<span class=\"DecNumber\">$1</span>", - "\\spanDecNumber{$1}", [toRope(esc(literal))]) + "\\spanDecNumber{$1}", [toRope(esc(d.target, literal))]) of tkFloatLit..tkFloat64Lit: dispA(result, "<span class=\"FloatNumber\">$1</span>", - "\\spanFloatNumber{$1}", [toRope(esc(literal))]) + "\\spanFloatNumber{$1}", [toRope(esc(d.target, literal))]) of tkSymbol: dispA(result, "<span class=\"Identifier\">$1</span>", - "\\spanIdentifier{$1}", [toRope(esc(literal))]) + "\\spanIdentifier{$1}", [toRope(esc(d.target, literal))]) of tkInd, tkSad, tkDed, tkSpaces, tkInvalid: app(result, literal) of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi, @@ -368,19 +326,19 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = tkAccent, tkColonColon, tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr: dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}", - [toRope(esc(literal))]) + [toRope(esc(d.target, literal))]) inc(d.id) app(d.section[k], ropeFormatNamedVars(getConfigVar("doc.item"), ["name", "header", "desc", "itemID"], [name, result, comm, toRope(d.id)])) app(d.toc[k], ropeFormatNamedVars(getConfigVar("doc.item.toc"), ["name", "header", "desc", "itemID"], [ - toRope(getName(nameNode, d.splitAfter)), result, comm, toRope(d.id)])) + toRope(getName(d, nameNode, d.splitAfter)), result, comm, toRope(d.id)])) setIndexForSourceTerm(d, getRstName(nameNode), d.id) proc renderHeadline(d: PDoc, n: PRstNode): PRope = result = nil - for i in countup(0, rsonsLen(n) - 1): app(result, renderRstToOut(d, n.sons[i])) + for i in countup(0, len(n) - 1): app(result, renderRstToOut(d, n.sons[i])) var refname = toRope(rstnodeToRefname(n)) if d.hasToc: var length = len(d.tocPart) @@ -400,7 +358,7 @@ proc renderHeadline(d: PDoc, n: PRstNode): PRope = proc renderOverline(d: PDoc, n: PRstNode): PRope = var t: PRope = nil - for i in countup(0, rsonsLen(n) - 1): app(t, renderRstToOut(d, n.sons[i])) + for i in countup(0, len(n) - 1): app(t, renderRstToOut(d, n.sons[i])) result = nil if d.meta[metaTitle] == nil: d.meta[metaTitle] = t @@ -411,123 +369,7 @@ proc renderOverline(d: PDoc, n: PRstNode): PRope = "\\rstov$4{$3}\\label{$2}$n", [toRope(n.level), toRope(rstnodeToRefname(n)), t, toRope($chr(n.level - 1 + ord('A')))]) -proc renderRstToRst(d: PDoc, n: PRstNode): PRope -proc renderRstSons(d: PDoc, n: PRstNode): PRope = - for i in countup(0, rsonsLen(n) - 1): - app(result, renderRstToRst(d, n.sons[i])) - -proc renderRstToRst(d: PDoc, n: PRstNode): PRope = - # this is needed for the index generation; it may also be useful for - # debugging, but most code is already debugged... - const - lvlToChar: array[0..8, char] = ['!', '=', '-', '~', '`', '<', '*', '|', '+'] - result = nil - if n == nil: return - var ind = toRope(repeatChar(d.indent)) - case n.kind - of rnInner: - result = renderRstSons(d, n) - of rnHeadline: - result = renderRstSons(d, n) - var L = ropeLen(result) - result = ropef("$n$1$2$n$1$3", - [ind, result, toRope(repeatChar(L, lvlToChar[n.level]))]) - of rnOverline: - result = renderRstSons(d, n) - var L = ropeLen(result) - result = ropef("$n$1$3$n$1$2$n$1$3", - [ind, result, toRope(repeatChar(L, lvlToChar[n.level]))]) - of rnTransition: - result = ropef("$n$n$1$2$n$n", [ind, toRope(repeatChar(78-d.indent, '-'))]) - of rnParagraph: - result = renderRstSons(d, n) - result = ropef("$n$n$1$2", [ind, result]) - of rnBulletItem: - inc(d.indent, 2) - result = renderRstSons(d, n) - if result != nil: result = ropef("$n$1* $2", [ind, result]) - dec(d.indent, 2) - of rnEnumItem: - inc(d.indent, 4) - result = renderRstSons(d, n) - if result != nil: result = ropef("$n$1(#) $2", [ind, result]) - dec(d.indent, 4) - of rnOptionList, rnFieldList, rnDefList, rnDefItem, rnLineBlock, rnFieldName, - rnFieldBody, rnStandaloneHyperlink, rnBulletList, rnEnumList: - result = renderRstSons(d, n) - of rnDefName: - result = renderRstSons(d, n) - result = ropef("$n$n$1$2", [ind, result]) - of rnDefBody: - inc(d.indent, 2) - result = renderRstSons(d, n) - if n.sons[0].kind != rnBulletList: result = ropef("$n$1 $2", [ind, result]) - dec(d.indent, 2) - of rnField: - result = renderRstToRst(d, n.sons[0]) - var L = max(ropeLen(result) + 3, 30) - inc(d.indent, L) - result = ropef("$n$1:$2:$3$4", [ind, result, toRope( - repeatChar(L - ropeLen(result) - 2)), renderRstToRst(d, n.sons[1])]) - dec(d.indent, L) - of rnLineBlockItem: - result = renderRstSons(d, n) - result = ropef("$n$1| $2", [ind, result]) - of rnBlockQuote: - inc(d.indent, 2) - result = renderRstSons(d, n) - dec(d.indent, 2) - of rnRef: - result = renderRstSons(d, n) - result = ropef("`$1`_", [result]) - of rnHyperlink: - result = ropef("`$1 <$2>`_", - [renderRstToRst(d, n.sons[0]), renderRstToRst(d, n.sons[1])]) - of rnGeneralRole: - result = renderRstToRst(d, n.sons[0]) - result = ropef("`$1`:$2:", [result, renderRstToRst(d, n.sons[1])]) - of rnSub: - result = renderRstSons(d, n) - result = ropef("`$1`:sub:", [result]) - of rnSup: - result = renderRstSons(d, n) - result = ropef("`$1`:sup:", [result]) - of rnIdx: - result = renderRstSons(d, n) - result = ropef("`$1`:idx:", [result]) - of rnEmphasis: - result = renderRstSons(d, n) - result = ropef("*$1*", [result]) - of rnStrongEmphasis: - result = renderRstSons(d, n) - result = ropef("**$1**", [result]) - of rnTripleEmphasis: - result = renderRstSons(d, n) - result = ropef("***$1***", [result]) - of rnInterpretedText: - result = renderRstSons(d, n) - result = ropef("`$1`", [result]) - of rnInlineLiteral: - inc(d.verbatim) - result = renderRstSons(d, n) - result = ropef("``$1``", [result]) - dec(d.verbatim) - of rnSmiley: - result = toRope(n.text) - of rnLeaf: - if (d.verbatim == 0) and (n.text == "\\"): - result = toRope("\\\\") # XXX: escape more special characters! - else: - result = toRope(n.text) - of rnIndex: - inc(d.indent, 3) - if n.sons[2] != nil: result = renderRstSons(d, n.sons[2]) - dec(d.indent, 3) - result = ropef("$n$n$1.. index::$n$2", [ind, result]) - of rnContents: - result = ropef("$n$n$1.. contents::", [ind]) - else: rawMessage(errCannotRenderX, $n.kind) - + proc renderTocEntry(d: PDoc, e: TTocEntry): PRope = result = dispF( "<li><a class=\"reference\" id=\"$1_toc\" href=\"#$1\">$2</a></li>$n", @@ -537,10 +379,10 @@ proc renderTocEntries(d: PDoc, j: var int, lvl: int): PRope = result = nil while j <= high(d.tocPart): var a = abs(d.tocPart[j].n.level) - if (a == lvl): + if a == lvl: app(result, renderTocEntry(d, d.tocPart[j])) inc(j) - elif (a > lvl): + elif a > lvl: app(result, renderTocEntries(d, j, a)) else: break @@ -566,7 +408,7 @@ proc renderImage(d: PDoc, n: PRstNode): PRope = if options != nil: options = dispF("$1", "[$1]", [options]) result = dispF("<img src=\"$1\"$2 />", "\\includegraphics$2{$1}", [toRope(getArgument(n)), options]) - if rsonsLen(n) >= 3: app(result, renderRstToOut(d, n.sons[2])) + if len(n) >= 3: app(result, renderRstToOut(d, n.sons[2])) proc renderSmiley(d: PDoc, n: PRstNode): PRope = result = dispF( @@ -596,13 +438,13 @@ proc renderCodeBlock(d: PDoc, n: PRstNode): PRope = case g.kind of gtEof: break of gtNone, gtWhitespace: - app(result, substr(m.text, g.start + 0, g.length + g.start - 1)) + app(result, substr(m.text, g.start, g.length + g.start - 1)) else: dispA(result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", [ - toRope(esc(substr(m.text, g.start + 0, g.length + g.start - 1))), - toRope(tokenClassToStr[g.kind])]) + toRope(esc(d.target, substr(m.text, g.start, g.length+g.start-1))), + toRope(tokenClassToStr[g.kind])]) deinitGeneralTokenizer(g) - if result != nil: + if result != nil: result = dispF("<pre>$1</pre>", "\\begin{rstpre}$n$1$n\\end{rstpre}$n", [result]) @@ -614,13 +456,13 @@ proc renderContainer(d: PDoc, n: PRstNode): PRope = proc texColumns(n: PRstNode): string = result = "" - for i in countup(1, rsonsLen(n)): add(result, "|X") + for i in countup(1, len(n)): add(result, "|X") proc renderField(d: PDoc, n: PRstNode): PRope = var b = false if gCmd == cmdRst2Tex: var fieldname = addNodes(n.sons[0]) - var fieldval = toRope(esc(strip(addNodes(n.sons[1])))) + var fieldval = toRope(esc(d.target, strip(addNodes(n.sons[1])))) if cmpIgnoreStyle(fieldname, "author") == 0: if d.meta[metaAuthor] == nil: d.meta[metaAuthor] = fieldval @@ -657,7 +499,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode): PRope = of rnDefBody: result = renderAux(d, n, disp("<dd>$1</dd>\n", "$1\n")) of rnFieldList: result = nil - for i in countup(0, rsonsLen(n) - 1): + for i in countup(0, len(n) - 1): app(result, renderRstToOut(d, n.sons[i])) if result != nil: result = dispf( @@ -705,9 +547,9 @@ proc renderRstToOut(d: PDoc, n: PRstNode): PRope = "\\begin{table}\\begin{rsttab}{" & texColumns(n) & "|}$n\\hline$n$1\\end{rsttab}\\end{table}")) of rnTableRow: - if rsonsLen(n) >= 1: + if len(n) >= 1: result = renderRstToOut(d, n.sons[0]) - for i in countup(1, rsonsLen(n) - 1): + for i in countup(1, len(n) - 1): dispa(result, "$1", " & $1", [renderRstToOut(d, n.sons[i])]) result = dispf("<tr>$1</tr>$n", "$1\\\\$n\\hline$n", [result]) else: @@ -770,7 +612,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode): PRope = "<tt class=\"docutils literal\"><span class=\"pre\">$1</span></tt>", "\\texttt{$1}")) of rnSmiley: result = renderSmiley(d, n) - of rnLeaf: result = toRope(esc(n.text)) + of rnLeaf: result = toRope(esc(d.target, n.text)) of rnContents: d.hasToc = true of rnTitle: d.meta[metaTitle] = renderRstToOut(d, n.sons[0]) @@ -863,7 +705,9 @@ proc genOutFile(d: PDoc): PRope = proc generateIndex(d: PDoc) = if d.theIndex != nil: sortIndex(d.theIndex) - writeRope(renderRstToRst(d, d.indexFile), gIndexFile) + var content = newStringOfCap(2_000_000) + renderRstToRst(d.indexFile, content) + writeFile(gIndexFile, content) proc writeOutput(d: PDoc, filename, outExt: string) = var content = genOutFile(d) @@ -886,7 +730,7 @@ proc CommandRstAux(filename, outExt: string) = var filen = addFileExt(filename, "txt") var d = newDocumentor(filen) initIndexFile(d) - var rst = rstParse(readFile(filen), filen, 0, 1, d.hasToc, + var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc, {roSupportRawDirective}) d.modDesc = renderRstToOut(d, rst) writeOutput(d, filename, outExt) diff --git a/compiler/highlite.nim b/compiler/highlite.nim deleted file mode 100755 index ff4b27086..000000000 --- a/compiler/highlite.nim +++ /dev/null @@ -1,531 +0,0 @@ -# -# -# The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# Source highlighter for programming or markup languages. -# Currently only few languages are supported, other languages may be added. -# The interface supports one language nested in another. - -import - hashes, options, msgs, strutils, platform, idents, lexbase, wordrecg, lexer - -type - TTokenClass* = enum - gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber, - gtOctNumber, gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit, - gtLongStringLit, gtCharLit, gtEscapeSequence, # escape sequence like \xff - gtOperator, gtPunctation, gtComment, gtLongComment, gtRegularExpression, - gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler, - gtPreprocessor, gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel, - gtReference, gtOther - TGeneralTokenizer* = object of TObject - kind*: TTokenClass - start*, length*: int - buf: cstring - pos: int - state: TTokenClass - - TSourceLanguage* = enum - langNone, langNimrod, langCpp, langCsharp, langC, langJava - -const - sourceLanguageToStr*: array[TSourceLanguage, string] = ["none", "Nimrod", - "C++", "C#", "C", "Java"] - tokenClassToStr*: array[TTokenClass, string] = ["Eof", "None", "Whitespace", - "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber", - "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit", - "EscapeSequence", "Operator", "Punctation", "Comment", "LongComment", - "RegularExpression", "TagStart", "TagEnd", "Key", "Value", "RawData", - "Assembler", "Preprocessor", "Directive", "Command", "Rule", "Hyperlink", - "Label", "Reference", "Other"] - -proc getSourceLanguage*(name: string): TSourceLanguage -proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: string) -proc deinitGeneralTokenizer*(g: var TGeneralTokenizer) -proc getNextToken*(g: var TGeneralTokenizer, lang: TSourceLanguage) -# implementation - -proc getSourceLanguage(name: string): TSourceLanguage = - for i in countup(succ(low(TSourceLanguage)), high(TSourceLanguage)): - if cmpIgnoreStyle(name, sourceLanguageToStr[i]) == 0: - return i - result = langNone - -proc initGeneralTokenizer(g: var TGeneralTokenizer, buf: string) = - g.buf = cstring(buf) - g.kind = low(TTokenClass) - g.start = 0 - g.length = 0 - g.state = low(TTokenClass) - var pos = 0 # skip initial whitespace: - while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) - g.pos = pos - -proc deinitGeneralTokenizer(g: var TGeneralTokenizer) = - nil - -proc nimGetKeyword(id: string): TTokenClass = - var i = getIdent(id) - if (i.id >= ord(tokKeywordLow) - ord(tkSymbol)) and - (i.id <= ord(tokKeywordHigh) - ord(tkSymbol)): - result = gtKeyword - else: - result = gtIdentifier - -proc nimNumberPostfix(g: var TGeneralTokenizer, position: int): int = - var pos = position - if g.buf[pos] == '\'': - inc(pos) - case g.buf[pos] - of 'f', 'F': - g.kind = gtFloatNumber - inc(pos) - if g.buf[pos] in {'0'..'9'}: inc(pos) - if g.buf[pos] in {'0'..'9'}: inc(pos) - of 'i', 'I': - inc(pos) - if g.buf[pos] in {'0'..'9'}: inc(pos) - if g.buf[pos] in {'0'..'9'}: inc(pos) - else: - nil - result = pos - -proc nimNumber(g: var TGeneralTokenizer, position: int): int = - const decChars = {'0'..'9', '_'} - var pos = position - g.kind = gtDecNumber - while g.buf[pos] in decChars: inc(pos) - if g.buf[pos] == '.': - g.kind = gtFloatNumber - inc(pos) - while g.buf[pos] in decChars: inc(pos) - if g.buf[pos] in {'e', 'E'}: - g.kind = gtFloatNumber - inc(pos) - if g.buf[pos] in {'+', '-'}: inc(pos) - while g.buf[pos] in decChars: inc(pos) - result = nimNumberPostfix(g, pos) - -proc nimNextToken(g: var TGeneralTokenizer) = - const - hexChars = {'0'..'9', 'A'..'F', 'a'..'f', '_'} - octChars = {'0'..'7', '_'} - binChars = {'0'..'1', '_'} - var pos = g.pos - g.start = g.pos - if g.state == gtStringLit: - g.kind = gtStringLit - while true: - case g.buf[pos] - of '\\': - g.kind = gtEscapeSequence - inc(pos) - case g.buf[pos] - of 'x', 'X': - inc(pos) - if g.buf[pos] in hexChars: inc(pos) - if g.buf[pos] in hexChars: inc(pos) - of '0'..'9': - while g.buf[pos] in {'0'..'9'}: inc(pos) - of '\0': - g.state = gtNone - else: inc(pos) - break - of '\0', '\x0D', '\x0A': - g.state = gtNone - break - of '\"': - inc(pos) - g.state = gtNone - break - else: inc(pos) - else: - case g.buf[pos] - of ' ', '\x09'..'\x0D': - g.kind = gtWhitespace - while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) - of '#': - g.kind = gtComment - while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos) - of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF': - var id = "" - while g.buf[pos] in lexer.SymChars + {'_'}: - add(id, g.buf[pos]) - inc(pos) - if (g.buf[pos] == '\"'): - if (g.buf[pos + 1] == '\"') and (g.buf[pos + 2] == '\"'): - inc(pos, 3) - g.kind = gtLongStringLit - while true: - case g.buf[pos] - of '\0': - break - of '\"': - inc(pos) - if g.buf[pos] == '\"' and g.buf[pos+1] == '\"' and - g.buf[pos+2] != '\"': - inc(pos, 2) - break - else: inc(pos) - else: - g.kind = gtRawData - inc(pos) - while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): - if g.buf[pos] == '"' and g.buf[pos+1] != '"': break - inc(pos) - if g.buf[pos] == '\"': inc(pos) - else: - g.kind = nimGetKeyword(id) - of '0': - inc(pos) - case g.buf[pos] - of 'b', 'B': - inc(pos) - while g.buf[pos] in binChars: inc(pos) - pos = nimNumberPostfix(g, pos) - of 'x', 'X': - inc(pos) - while g.buf[pos] in hexChars: inc(pos) - pos = nimNumberPostfix(g, pos) - of 'o', 'O': - inc(pos) - while g.buf[pos] in octChars: inc(pos) - pos = nimNumberPostfix(g, pos) - else: pos = nimNumber(g, pos) - of '1'..'9': - pos = nimNumber(g, pos) - of '\'': - inc(pos) - g.kind = gtCharLit - while true: - case g.buf[pos] - of '\0', '\x0D', '\x0A': - break - of '\'': - inc(pos) - break - of '\\': - inc(pos, 2) - else: inc(pos) - of '\"': - inc(pos) - if (g.buf[pos] == '\"') and (g.buf[pos + 1] == '\"'): - inc(pos, 2) - g.kind = gtLongStringLit - while true: - case g.buf[pos] - of '\0': - break - of '\"': - inc(pos) - if g.buf[pos] == '\"' and g.buf[pos+1] == '\"' and - g.buf[pos+2] != '\"': - inc(pos, 2) - break - else: inc(pos) - else: - g.kind = gtStringLit - while true: - case g.buf[pos] - of '\0', '\x0D', '\x0A': - break - of '\"': - inc(pos) - break - of '\\': - g.state = g.kind - break - else: inc(pos) - of '(', ')', '[', ']', '{', '}', '`', ':', ',', ';': - inc(pos) - g.kind = gtPunctation - of '\0': - g.kind = gtEof - else: - if g.buf[pos] in lexer.OpChars: - g.kind = gtOperator - while g.buf[pos] in lexer.OpChars: inc(pos) - else: - inc(pos) - g.kind = gtNone - g.length = pos - g.pos - if (g.kind != gtEof) and (g.length <= 0): - InternalError("nimNextToken: " & $(g.buf)) - g.pos = pos - -proc generalNumber(g: var TGeneralTokenizer, position: int): int = - const decChars = {'0'..'9'} - var pos = position - g.kind = gtDecNumber - while g.buf[pos] in decChars: inc(pos) - if g.buf[pos] == '.': - g.kind = gtFloatNumber - inc(pos) - while g.buf[pos] in decChars: inc(pos) - if g.buf[pos] in {'e', 'E'}: - g.kind = gtFloatNumber - inc(pos) - if g.buf[pos] in {'+', '-'}: inc(pos) - while g.buf[pos] in decChars: inc(pos) - result = pos - -proc generalStrLit(g: var TGeneralTokenizer, position: int): int = - const - decChars = {'0'..'9'} - hexChars = {'0'..'9', 'A'..'F', 'a'..'f'} - var pos = position - g.kind = gtStringLit - var c = g.buf[pos] - inc(pos) # skip " or ' - while true: - case g.buf[pos] - of '\0': - break - of '\\': - inc(pos) - case g.buf[pos] - of '\0': - break - of '0'..'9': - while g.buf[pos] in decChars: inc(pos) - of 'x', 'X': - inc(pos) - if g.buf[pos] in hexChars: inc(pos) - if g.buf[pos] in hexChars: inc(pos) - else: inc(pos, 2) - else: - if g.buf[pos] == c: - inc(pos) - break - else: - inc(pos) - result = pos - -proc isKeyword(x: openarray[string], y: string): int = - var a = 0 - var b = len(x) - 1 - while a <= b: - var mid = (a + b) div 2 - var c = cmp(x[mid], y) - if c < 0: - a = mid + 1 - elif c > 0: - b = mid - 1 - else: - return mid - result = - 1 - -proc isKeywordIgnoreCase(x: openarray[string], y: string): int = - var a = 0 - var b = len(x) - 1 - while a <= b: - var mid = (a + b) div 2 - var c = cmpIgnoreCase(x[mid], y) - if c < 0: - a = mid + 1 - elif c > 0: - b = mid - 1 - else: - return mid - result = - 1 - -type - TTokenizerFlag = enum - hasPreprocessor, hasNestedComments - TTokenizerFlags = set[TTokenizerFlag] - -proc clikeNextToken(g: var TGeneralTokenizer, keywords: openarray[string], - flags: TTokenizerFlags) = - const - hexChars = {'0'..'9', 'A'..'F', 'a'..'f'} - octChars = {'0'..'7'} - binChars = {'0'..'1'} - symChars = {'A'..'Z', 'a'..'z', '0'..'9', '_', '\x80'..'\xFF'} - var pos = g.pos - g.start = g.pos - if g.state == gtStringLit: - g.kind = gtStringLit - while true: - case g.buf[pos] - of '\\': - g.kind = gtEscapeSequence - inc(pos) - case g.buf[pos] - of 'x', 'X': - inc(pos) - if g.buf[pos] in hexChars: inc(pos) - if g.buf[pos] in hexChars: inc(pos) - of '0'..'9': - while g.buf[pos] in {'0'..'9'}: inc(pos) - of '\0': - g.state = gtNone - else: inc(pos) - break - of '\0', '\x0D', '\x0A': - g.state = gtNone - break - of '\"': - inc(pos) - g.state = gtNone - break - else: inc(pos) - else: - case g.buf[pos] - of ' ', '\x09'..'\x0D': - g.kind = gtWhitespace - while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) - of '/': - inc(pos) - if g.buf[pos] == '/': - g.kind = gtComment - while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos) - elif g.buf[pos] == '*': - g.kind = gtLongComment - var nested = 0 - inc(pos) - while true: - case g.buf[pos] - of '*': - inc(pos) - if g.buf[pos] == '/': - inc(pos) - if nested == 0: break - of '/': - inc(pos) - if g.buf[pos] == '*': - inc(pos) - if hasNestedComments in flags: inc(nested) - of '\0': - break - else: inc(pos) - of '#': - inc(pos) - if hasPreprocessor in flags: - g.kind = gtPreprocessor - while g.buf[pos] in {' ', Tabulator}: inc(pos) - while g.buf[pos] in symChars: inc(pos) - else: - g.kind = gtOperator - of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF': - var id = "" - while g.buf[pos] in SymChars: - add(id, g.buf[pos]) - inc(pos) - if isKeyword(keywords, id) >= 0: g.kind = gtKeyword - else: g.kind = gtIdentifier - of '0': - inc(pos) - case g.buf[pos] - of 'b', 'B': - inc(pos) - while g.buf[pos] in binChars: inc(pos) - if g.buf[pos] in {'A'..'Z', 'a'..'z'}: inc(pos) - of 'x', 'X': - inc(pos) - while g.buf[pos] in hexChars: inc(pos) - if g.buf[pos] in {'A'..'Z', 'a'..'z'}: inc(pos) - of '0'..'7': - inc(pos) - while g.buf[pos] in octChars: inc(pos) - if g.buf[pos] in {'A'..'Z', 'a'..'z'}: inc(pos) - else: - pos = generalNumber(g, pos) - if g.buf[pos] in {'A'..'Z', 'a'..'z'}: inc(pos) - of '1'..'9': - pos = generalNumber(g, pos) - if g.buf[pos] in {'A'..'Z', 'a'..'z'}: inc(pos) - of '\'': - pos = generalStrLit(g, pos) - g.kind = gtCharLit - of '\"': - inc(pos) - g.kind = gtStringLit - while true: - case g.buf[pos] - of '\0': - break - of '\"': - inc(pos) - break - of '\\': - g.state = g.kind - break - else: inc(pos) - of '(', ')', '[', ']', '{', '}', ':', ',', ';', '.': - inc(pos) - g.kind = gtPunctation - of '\0': - g.kind = gtEof - else: - if g.buf[pos] in lexer.OpChars: - g.kind = gtOperator - while g.buf[pos] in lexer.OpChars: inc(pos) - else: - inc(pos) - g.kind = gtNone - g.length = pos - g.pos - if (g.kind != gtEof) and (g.length <= 0): InternalError("clikeNextToken") - g.pos = pos - -proc cNextToken(g: var TGeneralTokenizer) = - const - keywords: array[0..36, string] = ["_Bool", "_Complex", "_Imaginary", "auto", - "break", "case", "char", "const", "continue", "default", "do", "double", - "else", "enum", "extern", "float", "for", "goto", "if", "inline", "int", - "long", "register", "restrict", "return", "short", "signed", "sizeof", - "static", "struct", "switch", "typedef", "union", "unsigned", "void", - "volatile", "while"] - clikeNextToken(g, keywords, {hasPreprocessor}) - -proc cppNextToken(g: var TGeneralTokenizer) = - const - keywords: array[0..47, string] = ["asm", "auto", "break", "case", "catch", - "char", "class", "const", "continue", "default", "delete", "do", "double", - "else", "enum", "extern", "float", "for", "friend", "goto", "if", - "inline", "int", "long", "new", "operator", "private", "protected", - "public", "register", "return", "short", "signed", "sizeof", "static", - "struct", "switch", "template", "this", "throw", "try", "typedef", - "union", "unsigned", "virtual", "void", "volatile", "while"] - clikeNextToken(g, keywords, {hasPreprocessor}) - -proc csharpNextToken(g: var TGeneralTokenizer) = - const - keywords: array[0..76, string] = ["abstract", "as", "base", "bool", "break", - "byte", "case", "catch", "char", "checked", "class", "const", "continue", - "decimal", "default", "delegate", "do", "double", "else", "enum", "event", - "explicit", "extern", "false", "finally", "fixed", "float", "for", - "foreach", "goto", "if", "implicit", "in", "int", "interface", "internal", - "is", "lock", "long", "namespace", "new", "null", "object", "operator", - "out", "override", "params", "private", "protected", "public", "readonly", - "ref", "return", "sbyte", "sealed", "short", "sizeof", "stackalloc", - "static", "string", "struct", "switch", "this", "throw", "true", "try", - "typeof", "uint", "ulong", "unchecked", "unsafe", "ushort", "using", - "virtual", "void", "volatile", "while"] - clikeNextToken(g, keywords, {hasPreprocessor}) - -proc javaNextToken(g: var TGeneralTokenizer) = - const - keywords: array[0..52, string] = ["abstract", "assert", "boolean", "break", - "byte", "case", "catch", "char", "class", "const", "continue", "default", - "do", "double", "else", "enum", "extends", "false", "final", "finally", - "float", "for", "goto", "if", "implements", "import", "instanceof", "int", - "interface", "long", "native", "new", "null", "package", "private", - "protected", "public", "return", "short", "static", "strictfp", "super", - "switch", "synchronized", "this", "throw", "throws", "transient", "true", - "try", "void", "volatile", "while"] - clikeNextToken(g, keywords, {}) - -proc getNextToken(g: var TGeneralTokenizer, lang: TSourceLanguage) = - case lang - of langNimrod: nimNextToken(g) - of langCpp: cppNextToken(g) - of langCsharp: csharpNextToken(g) - of langC: cNextToken(g) - of langJava: javaNextToken(g) - else: InternalError("getNextToken") - diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 496f7d2f1..a17871e3a 100755 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -27,6 +27,8 @@ const OpChars*: TCharSet = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.', '|', '=', '%', '&', '$', '@', '~', ':', '\x80'..'\xFF'} +# don't forget to update the 'highlite' module if these charsets should change + type TTokType* = enum tkInvalid, tkEof, # order is important here! diff --git a/compiler/msgs.nim b/compiler/msgs.nim index a0058ebe3..17cfeae4d 100755 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -663,7 +663,7 @@ proc InternalError*(errMsg: string) = rawMessage(errInternal, errMsg) template AssertNotNil*(e: expr): expr = - if(e == nil): InternalError($InstantiationInfo()) + if e == nil: InternalError($InstantiationInfo()) e template InternalAssert*(e: bool): stmt = diff --git a/compiler/nimrod.cfg b/compiler/nimrod.cfg index 7d8d4d94f..f47143527 100755 --- a/compiler/nimrod.cfg +++ b/compiler/nimrod.cfg @@ -4,6 +4,8 @@ path="llvm" path="$projectPath/.." +path="$nimrod/packages/docutils" + @if llvm_gcc or gcc: # GCC, LLVM and Visual C++ have a problem to optimize some modules. # This is really strange. diff --git a/compiler/nimrod.ini b/compiler/nimrod.ini index 9fd6f0ce6..da87d957d 100755 --- a/compiler/nimrod.ini +++ b/compiler/nimrod.ini @@ -33,6 +33,7 @@ Files: "doc/*.pdf" Files: "doc/*.ini" Start: "doc/overview.html" + [Other] Files: "readme.txt;install.txt;contributors.txt" Files: "configure;makefile" @@ -56,6 +57,9 @@ Files: "compiler/*.nim" Files: "build/empty.txt" Files: "bin/empty.txt" +Files: "packages/docutils/*.nim" + + [Lib] Files: "lib/nimbase.h;lib/cycle.h" Files: "lib/*.nim" @@ -72,6 +76,7 @@ Files: "lib/wrappers/cairo/*.nim" Files: "lib/wrappers/gtk/*.nim" Files: "lib/wrappers/lua/*.nim" Files: "lib/wrappers/opengl/*.nim" +Files: "lib/wrappers/readline/*.nim" Files: "lib/wrappers/sdl/*.nim" Files: "lib/wrappers/x11/*.nim" Files: "lib/wrappers/zip/*.nim" @@ -81,6 +86,7 @@ Files: "lib/windows/*.nim" Files: "lib/posix/*.nim" Files: "lib/ecmas/*.nim" + [Other] Files: "examples/*.nim" Files: "examples/gtk/*.nim" @@ -99,6 +105,7 @@ Files: "examples/*.txt" Files: "examples/*.cfg" Files: "examples/*.tmpl" + [Windows] Files: "bin/nimrod.exe" Files: "bin/c2nim.exe" @@ -112,21 +119,26 @@ Files: "start.bat" BinPath: r"bin;dist\mingw\bin;dist" InnoSetup: "Yes" + [UnixBin] Files: "bin/nimrod" + [Unix] InstallScript: "yes" UninstallScript: "yes" + [InnoSetup] path = r"c:\programme\inno setup 5\iscc.exe" flags = "/Q" + [C_Compiler] path = r"" flags = "-w" + [deb] buildDepends: "gcc (>= 4:4.3.2)" pkgDepends: "gcc (>= 4:4.3.2)" diff --git a/compiler/options.nim b/compiler/options.nim index 3a2352c7f..9d7b41180 100755 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -199,7 +199,7 @@ proc rawFindFile(f: string): string = return result.canonicalizePath result = "" -proc FindFile*(f: string): string = +proc FindFile*(f: string): string {.procvar.} = result = rawFindFile(f) if len(result) == 0: result = rawFindFile(toLower(f)) diff --git a/compiler/rst.nim b/compiler/rst.nim deleted file mode 100755 index 5cb9c3b6a..000000000 --- a/compiler/rst.nim +++ /dev/null @@ -1,1718 +0,0 @@ -# -# -# The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# This module implements a *reStructuredText* parser. A large -# subset is provided. - -import - os, msgs, strutils, hashes, options - -type - TRstNodeKind* = enum - rnInner, # an inner node or a root - rnHeadline, # a headline - rnOverline, # an over- and underlined headline - rnTransition, # a transition (the ------------- <hr> thingie) - rnParagraph, # a paragraph - rnBulletList, # a bullet list - rnBulletItem, # a bullet item - rnEnumList, # an enumerated list - rnEnumItem, # an enumerated item - rnDefList, # a definition list - rnDefItem, # an item of a definition list consisting of ... - rnDefName, # ... a name part ... - rnDefBody, # ... and a body part ... - rnFieldList, # a field list - rnField, # a field item - rnFieldName, # consisting of a field name ... - rnFieldBody, # ... and a field body - rnOptionList, rnOptionListItem, rnOptionGroup, rnOption, rnOptionString, - rnOptionArgument, rnDescription, rnLiteralBlock, rnQuotedLiteralBlock, - rnLineBlock, # the | thingie - rnLineBlockItem, # sons of the | thing - rnBlockQuote, # text just indented - rnTable, rnGridTable, rnTableRow, rnTableHeaderCell, rnTableDataCell, - rnLabel, # used for footnotes and other things - rnFootnote, # a footnote - rnCitation, # similar to footnote - rnStandaloneHyperlink, rnHyperlink, rnRef, rnDirective, # a directive - rnDirArg, rnRaw, rnTitle, rnContents, rnImage, rnFigure, rnCodeBlock, - rnRawHtml, rnRawLatex, - rnContainer, # ``container`` directive - rnIndex, # index directve: - # .. index:: - # key - # * `file#id <file#id>`_ - # * `file#id <file#id>'_ - rnSubstitutionDef, # a definition of a substitution - rnGeneralRole, # Inline markup: - rnSub, rnSup, rnIdx, - rnEmphasis, # "*" - rnStrongEmphasis, # "**" - rnTripleEmphasis, # "***" - rnInterpretedText, # "`" - rnInlineLiteral, # "``" - rnSubstitutionReferences, # "|" - rnSmiley, # some smiley - rnLeaf # a leaf; the node's text field contains the - # leaf val - -type - TRstParseOption* = enum ## options for the RST parser - roSkipPounds, ## skip ``#`` at line beginning (documentation - ## embedded in Nimrod comments) - roSupportSmilies, ## make the RST parser support smilies like ``:)`` - roSupportRawDirective ## support the ``raw`` directive (don't support - ## it for sandboxing) - - TRstParseOptions* = set[TRstParseOption] - - PRSTNode* = ref TRstNode - TRstNodeSeq* = seq[PRstNode] - TRSTNode*{.acyclic, final.} = object - kind*: TRstNodeKind - text*: string # valid for leafs in the AST; and the title of - # the document or the section - level*: int # valid for some node kinds - sons*: TRstNodeSeq # the node's sons - - -proc rstParse*(text, filename: string, - line, column: int, hasToc: var bool, - options: TRstParseOptions): PRstNode -proc rsonsLen*(n: PRstNode): int -proc newRstNode*(kind: TRstNodeKind): PRstNode -proc newRstNode*(kind: TRstNodeKind, s: string): PRstNode -proc addSon*(father, son: PRstNode) -proc rstnodeToRefname*(n: PRstNode): string -proc addNodes*(n: PRstNode): string -proc getFieldValue*(n: PRstNode, fieldname: string): string -proc getArgument*(n: PRstNode): string - # index handling: -proc setIndexPair*(index, key, val: PRstNode) -proc sortIndex*(a: PRstNode) -proc clearIndex*(index: PRstNode, filename: string) -# implementation -# ----------------------------- scanner part -------------------------------- - -const - SymChars: TCharSet = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF'} - SmileyStartChars: TCharSet = {':', ';', '8'} - Smilies = { - ":D": "icon_e_biggrin", - ":-D": "icon_e_biggrin", - ":)": "icon_e_smile", - ":-)": "icon_e_smile", - ";)": "icon_e_wink", - ";-)": "icon_e_wink", - ":(": "icon_e_sad", - ":-(": "icon_e_sad", - ":o": "icon_e_surprised", - ":-o": "icon_e_surprised", - ":shock:": "icon_eek", - ":?": "icon_e_confused", - ":-?": "icon_e_confused", - ":-/": "icon_e_confused", - - "8-)": "icon_cool", - - ":lol:": "icon_lol", - ":x": "icon_mad", - ":-x": "icon_mad", - ":P": "icon_razz", - ":-P": "icon_razz", - ":oops:": "icon_redface", - ":cry:": "icon_cry", - ":evil:": "icon_evil", - ":twisted:": "icon_twisted", - ":roll:": "icon_rolleyes", - ":!:": "icon_exclaim", - - ":?:": "icon_question", - ":idea:": "icon_idea", - ":arrow:": "icon_arrow", - ":|": "icon_neutral", - ":-|": "icon_neutral", - ":mrgreen:": "icon_mrgreen", - ":geek:": "icon_e_geek", - ":ugeek:": "icon_e_ugeek" - } - -type - TTokType = enum - tkEof, tkIndent, tkWhite, tkWord, tkAdornment, tkPunct, tkOther - TToken{.final.} = object # a RST token - kind*: TTokType # the type of the token - ival*: int # the indentation or parsed integer value - symbol*: string # the parsed symbol as string - line*, col*: int # line and column of the token - - TTokenSeq = seq[TToken] - TLexer = object of TObject - buf*: cstring - bufpos*: int - line*, col*, baseIndent*: int - skipPounds*: bool - - -proc getThing(L: var TLexer, tok: var TToken, s: TCharSet) = - tok.kind = tkWord - tok.line = L.line - tok.col = L.col - var pos = L.bufpos - while True: - add(tok.symbol, L.buf[pos]) - inc(pos) - if L.buf[pos] notin s: break - inc(L.col, pos - L.bufpos) - L.bufpos = pos - -proc getAdornment(L: var TLexer, tok: var TToken) = - tok.kind = tkAdornment - tok.line = L.line - tok.col = L.col - var pos = L.bufpos - var c = L.buf[pos] - while True: - add(tok.symbol, L.buf[pos]) - inc(pos) - if L.buf[pos] != c: break - inc(L.col, pos - L.bufpos) - L.bufpos = pos - -proc getIndentAux(L: var TLexer, start: int): int = - var pos = start - var buf = L.buf - # skip the newline (but include it in the token!) - if buf[pos] == '\x0D': - if buf[pos + 1] == '\x0A': inc(pos, 2) - else: inc(pos) - elif buf[pos] == '\x0A': - inc(pos) - if L.skipPounds: - if buf[pos] == '#': inc(pos) - if buf[pos] == '#': inc(pos) - while True: - case buf[pos] - of ' ', '\x0B', '\x0C': - inc(pos) - inc(result) - of '\x09': - inc(pos) - result = result - (result mod 8) + 8 - else: - break # EndOfFile also leaves the loop - if buf[pos] == '\0': - result = 0 - elif (buf[pos] == '\x0A') or (buf[pos] == '\x0D'): - # look at the next line for proper indentation: - result = getIndentAux(L, pos) - L.bufpos = pos # no need to set back buf - -proc getIndent(L: var TLexer, tok: var TToken) = - inc(L.line) - tok.line = L.line - tok.col = 0 - tok.kind = tkIndent # skip the newline (but include it in the token!) - tok.ival = getIndentAux(L, L.bufpos) - L.col = tok.ival - tok.ival = max(tok.ival - L.baseIndent, 0) - tok.symbol = "\n" & repeatChar(tok.ival) - -proc rawGetTok(L: var TLexer, tok: var TToken) = - tok.symbol = "" - tok.ival = 0 - var c = L.buf[L.bufpos] - case c - of 'a'..'z', 'A'..'Z', '\x80'..'\xFF', '0'..'9': - getThing(L, tok, SymChars) - of ' ', '\x09', '\x0B', '\x0C': - getThing(L, tok, {' ', '\x09'}) - tok.kind = tkWhite - if L.buf[L.bufpos] in {'\x0D', '\x0A'}: - rawGetTok(L, tok) # ignore spaces before \n - of '\x0D', '\x0A': - getIndent(L, tok) - of '!', '\"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', - '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', - '|', '}', '~': - getAdornment(L, tok) - if len(tok.symbol) <= 3: tok.kind = tkPunct - else: - tok.line = L.line - tok.col = L.col - if c == '\0': - tok.kind = tkEof - else: - tok.kind = tkOther - add(tok.symbol, c) - inc(L.bufpos) - inc(L.col) - tok.col = max(tok.col - L.baseIndent, 0) - -proc getTokens(buffer: string, skipPounds: bool, tokens: var TTokenSeq) = - var L: TLexer - var length = len(tokens) - L.buf = cstring(buffer) - L.line = 1 # skip UTF-8 BOM - if (L.buf[0] == '\xEF') and (L.buf[1] == '\xBB') and (L.buf[2] == '\xBF'): - inc(L.bufpos, 3) - L.skipPounds = skipPounds - if skipPounds: - if L.buf[L.bufpos] == '#': inc(L.bufpos) - if L.buf[L.bufpos] == '#': inc(L.bufpos) - L.baseIndent = 0 - while L.buf[L.bufpos] == ' ': - inc(L.bufpos) - inc(L.baseIndent) - while true: - inc(length) - setlen(tokens, length) - rawGetTok(L, tokens[length - 1]) - if tokens[length - 1].kind == tkEof: break - if tokens[0].kind == tkWhite: - # BUGFIX - tokens[0].ival = len(tokens[0].symbol) - tokens[0].kind = tkIndent - -proc addSon(father, son: PRstNode) = - add(father.sons, son) - -proc addSonIfNotNil(father, son: PRstNode) = - if son != nil: addSon(father, son) - -proc rsonsLen(n: PRstNode): int = - result = len(n.sons) - -proc newRstNode(kind: TRstNodeKind): PRstNode = - new(result) - result.sons = @[] - result.kind = kind - -proc newRstNode(kind: TRstNodeKind, s: string): PRstNode = - result = newRstNode(kind) - result.text = s - -proc lastSon*(n: PRstNode): PRstNode = - result = n.sons[len(n.sons)-1] - -type - TLevelMap = array[Char, int] - TSubstitution{.final.} = object - key*: string - value*: PRstNode - - TSharedState {.final.} = object - options: TRstParseOptions # parsing options - uLevel*, oLevel*: int # counters for the section levels - subs*: seq[TSubstitution] # substitutions - refs*: seq[TSubstitution] # references - underlineToLevel*: TLevelMap # Saves for each possible title adornment - # character its level in the - # current document. - # This is for single underline adornments. - overlineToLevel*: TLevelMap # Saves for each possible title adornment - # character its level in the current - # document. - # This is for over-underline adornments. - - PSharedState = ref TSharedState - TRstParser = object of TObject - idx*: int - tok*: TTokenSeq - s*: PSharedState - indentStack*: seq[int] - filename*: string - line*, col*: int - hasToc*: bool - - -proc newSharedState(options: TRstParseOptions): PSharedState = - new(result) - result.subs = @[] - result.refs = @[] - result.options = options - -proc tokInfo(p: TRstParser, tok: TToken): TLineInfo = - result = newLineInfo(p.filename, p.line + tok.line, p.col + tok.col) - -proc rstMessage(p: TRstParser, msgKind: TMsgKind, arg: string) = - GlobalError(tokInfo(p, p.tok[p.idx]), msgKind, arg) - -proc rstMessage(p: TRstParser, msgKind: TMsgKind) = - GlobalError(tokInfo(p, p.tok[p.idx]), msgKind, p.tok[p.idx].symbol) - -proc currInd(p: TRstParser): int = - result = p.indentStack[high(p.indentStack)] - -proc pushInd(p: var TRstParser, ind: int) = - add(p.indentStack, ind) - -proc popInd(p: var TRstParser) = - if len(p.indentStack) > 1: setlen(p.indentStack, len(p.indentStack) - 1) - -proc initParser(p: var TRstParser, sharedState: PSharedState) = - p.indentStack = @[0] - p.tok = @[] - p.idx = 0 - p.filename = "" - p.hasToc = false - p.col = 0 - p.line = 1 - p.s = sharedState - -proc addNodesAux(n: PRstNode, result: var string) = - if n.kind == rnLeaf: - add(result, n.text) - else: - for i in countup(0, rsonsLen(n) - 1): addNodesAux(n.sons[i], result) - -proc addNodes(n: PRstNode): string = - result = "" - addNodesAux(n, result) - -proc rstnodeToRefnameAux(n: PRstNode, r: var string, b: var bool) = - if n.kind == rnLeaf: - for i in countup(0, len(n.text) - 1): - case n.text[i] - of '0'..'9': - if b: - add(r, '-') - b = false - if len(r) == 0: add(r, 'Z') - add(r, n.text[i]) - of 'a'..'z': - if b: - add(r, '-') - b = false - add(r, n.text[i]) - of 'A'..'Z': - if b: - add(r, '-') - b = false - add(r, chr(ord(n.text[i]) - ord('A') + ord('a'))) - else: - if (len(r) > 0): b = true - else: - for i in countup(0, rsonsLen(n) - 1): rstnodeToRefnameAux(n.sons[i], r, b) - -proc rstnodeToRefname(n: PRstNode): string = - result = "" - var b = false - rstnodeToRefnameAux(n, result, b) - -proc findSub(p: var TRstParser, n: PRstNode): int = - var key = addNodes(n) - # the spec says: if no exact match, try one without case distinction: - for i in countup(0, high(p.s.subs)): - if key == p.s.subs[i].key: - return i - for i in countup(0, high(p.s.subs)): - if cmpIgnoreStyle(key, p.s.subs[i].key) == 0: - return i - result = - 1 - -proc setSub(p: var TRstParser, key: string, value: PRstNode) = - var length = len(p.s.subs) - for i in countup(0, length - 1): - if key == p.s.subs[i].key: - p.s.subs[i].value = value - return - setlen(p.s.subs, length + 1) - p.s.subs[length].key = key - p.s.subs[length].value = value - -proc setRef(p: var TRstParser, key: string, value: PRstNode) = - var length = len(p.s.refs) - for i in countup(0, length - 1): - if key == p.s.refs[i].key: - if p.s.refs[i].value.addNodes != value.addNodes: - rstMessage(p, warnRedefinitionOfLabel, key) - - p.s.refs[i].value = value - return - setlen(p.s.refs, length + 1) - p.s.refs[length].key = key - p.s.refs[length].value = value - -proc findRef(p: var TRstParser, key: string): PRstNode = - for i in countup(0, high(p.s.refs)): - if key == p.s.refs[i].key: - return p.s.refs[i].value - -proc cmpNodes(a, b: PRstNode): int = - assert(a.kind == rnDefItem) - assert(b.kind == rnDefItem) - var x = a.sons[0] - var y = b.sons[0] - result = cmpIgnoreStyle(addNodes(x), addNodes(y)) - -proc sortIndex(a: PRstNode) = - # we use shellsort here; fast and simple - assert(a.kind == rnDefList) - var N = rsonsLen(a) - var h = 1 - while true: - h = 3 * h + 1 - if h > N: break - while true: - h = h div 3 - for i in countup(h, N - 1): - var v = a.sons[i] - var j = i - while cmpNodes(a.sons[j - h], v) >= 0: - a.sons[j] = a.sons[j - h] - j = j - h - if j < h: break - a.sons[j] = v - if h == 1: break - -proc eqRstNodes(a, b: PRstNode): bool = - if a.kind != b.kind: return - if a.kind == rnLeaf: - result = a.text == b.text - else: - if rsonsLen(a) != rsonsLen(b): return - for i in countup(0, rsonsLen(a) - 1): - if not eqRstNodes(a.sons[i], b.sons[i]): return - result = true - -proc matchesHyperlink(h: PRstNode, filename: string): bool = - if h.kind == rnInner: # this may happen in broken indexes! - assert(rsonsLen(h) == 1) - result = matchesHyperlink(h.sons[0], filename) - elif h.kind == rnHyperlink: - var s = addNodes(h.sons[1]) - if startsWith(s, filename) and (s[len(filename)] == '#'): result = true - else: result = false - else: - result = false - -proc clearIndex(index: PRstNode, filename: string) = - var - lastItem: int - assert(index.kind == rnDefList) - for i in countup(0, rsonsLen(index) - 1): - assert(index.sons[i].sons[1].kind == rnDefBody) - var val = index.sons[i].sons[1].sons[0] - if val.kind == rnInner: val = val.sons[0] - if val.kind == rnBulletList: - var items = rsonsLen(val) - lastItem = - 1 # save the last valid item index - for j in countup(0, rsonsLen(val) - 1): - if val.sons[j] == nil: - dec(items) - elif matchesHyperlink(val.sons[j].sons[0], filename): - val.sons[j] = nil - dec(items) - else: - lastItem = j - if items == 1: - index.sons[i].sons[1].sons[0] = val.sons[lastItem].sons[0] - elif items == 0: - index.sons[i] = nil - elif matchesHyperlink(val, filename): - index.sons[i] = nil - var k = 0 - for i in countup(0, rsonsLen(index) - 1): - if index.sons[i] != nil: - if k != i: index.sons[k] = index.sons[i] - inc(k) - setlen(index.sons, k) - -proc setIndexPair(index, key, val: PRstNode) = - var e, a, b: PRstNode - assert(index.kind == rnDefList) - assert(key.kind != rnDefName) - a = newRstNode(rnDefName) - addSon(a, key) - for i in countup(0, rsonsLen(index) - 1): - if eqRstNodes(index.sons[i].sons[0], a): - assert(index.sons[i].sons[1].kind == rnDefBody) - e = index.sons[i].sons[1].sons[0] - if e.kind != rnBulletList: - e = newRstNode(rnBulletList) - b = newRstNode(rnBulletItem) - addSon(b, index.sons[i].sons[1].sons[0]) - addSon(e, b) - index.sons[i].sons[1].sons[0] = e - b = newRstNode(rnBulletItem) - addSon(b, val) - addSon(e, b) - return # key already exists - e = newRstNode(rnDefItem) - assert(val.kind != rnDefBody) - b = newRstNode(rnDefBody) - addSon(b, val) - addSon(e, a) - addSon(e, b) - addSon(index, e) - -proc newLeaf(p: var TRstParser): PRstNode = - result = newRstNode(rnLeaf, p.tok[p.idx].symbol) - -proc getReferenceName(p: var TRstParser, endStr: string): PRstNode = - var res = newRstNode(rnInner) - while true: - case p.tok[p.idx].kind - of tkWord, tkOther, tkWhite: - addSon(res, newLeaf(p)) - of tkPunct: - if p.tok[p.idx].symbol == endStr: - inc(p.idx) - break - else: - addSon(res, newLeaf(p)) - else: - rstMessage(p, errXexpected, endStr) - break - inc(p.idx) - result = res - -proc untilEol(p: var TRstParser): PRstNode = - result = newRstNode(rnInner) - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - addSon(result, newLeaf(p)) - inc(p.idx) - -proc expect(p: var TRstParser, tok: string) = - if p.tok[p.idx].symbol == tok: inc(p.idx) - else: rstMessage(p, errXexpected, tok) - -proc isInlineMarkupEnd(p: TRstParser, markup: string): bool = - result = p.tok[p.idx].symbol == markup - if not result: - return # Rule 3: - result = not (p.tok[p.idx - 1].kind in {tkIndent, tkWhite}) - if not result: - return # Rule 4: - result = (p.tok[p.idx + 1].kind in {tkIndent, tkWhite, tkEof}) or - (p.tok[p.idx + 1].symbol[0] in - {'\'', '\"', ')', ']', '}', '>', '-', '/', '\\', ':', '.', ',', ';', '!', - '?', '_'}) - if not result: - return # Rule 7: - if p.idx > 0: - if (markup != "``") and (p.tok[p.idx - 1].symbol == "\\"): - result = false - -proc isInlineMarkupStart(p: TRstParser, markup: string): bool = - var d: Char - result = p.tok[p.idx].symbol == markup - if not result: - return # Rule 1: - result = (p.idx == 0) or (p.tok[p.idx - 1].kind in {tkIndent, tkWhite}) or - (p.tok[p.idx - 1].symbol[0] in - {'\'', '\"', '(', '[', '{', '<', '-', '/', ':', '_'}) - if not result: - return # Rule 2: - result = not (p.tok[p.idx + 1].kind in {tkIndent, tkWhite, tkEof}) - if not result: - return # Rule 5 & 7: - if p.idx > 0: - if p.tok[p.idx - 1].symbol == "\\": - result = false - else: - var c = p.tok[p.idx - 1].symbol[0] - case c - of '\'', '\"': d = c - of '(': d = ')' - of '[': d = ']' - of '{': d = '}' - of '<': d = '>' - else: d = '\0' - if d != '\0': result = p.tok[p.idx + 1].symbol[0] != d - -proc match(p: TRstParser, start: int, expr: string): bool = - # regular expressions are: - # special char exact match - # 'w' tkWord - # ' ' tkWhite - # 'a' tkAdornment - # 'i' tkIndent - # 'p' tkPunct - # 'T' always true - # 'E' whitespace, indent or eof - # 'e' tkWord or '#' (for enumeration lists) - var i = 0 - var j = start - var last = len(expr) - 1 - while i <= last: - case expr[i] - of 'w': result = p.tok[j].kind == tkWord - of ' ': result = p.tok[j].kind == tkWhite - of 'i': result = p.tok[j].kind == tkIndent - of 'p': result = p.tok[j].kind == tkPunct - of 'a': result = p.tok[j].kind == tkAdornment - of 'o': result = p.tok[j].kind == tkOther - of 'T': result = true - of 'E': result = p.tok[j].kind in {tkEof, tkWhite, tkIndent} - of 'e': - result = (p.tok[j].kind == tkWord) or (p.tok[j].symbol == "#") - if result: - case p.tok[j].symbol[0] - of 'a'..'z', 'A'..'Z': result = len(p.tok[j].symbol) == 1 - of '0'..'9': result = allCharsInSet(p.tok[j].symbol, {'0'..'9'}) - else: nil - else: - var c = expr[i] - var length = 0 - while (i <= last) and (expr[i] == c): - inc(i) - inc(length) - dec(i) - result = (p.tok[j].kind in {tkPunct, tkAdornment}) and - (len(p.tok[j].symbol) == length) and (p.tok[j].symbol[0] == c) - if not result: return - inc(j) - inc(i) - result = true - -proc fixupEmbeddedRef(n, a, b: PRstNode) = - var sep = - 1 - for i in countdown(rsonsLen(n) - 2, 0): - if n.sons[i].text == "<": - sep = i - break - var incr = if (sep > 0) and (n.sons[sep - 1].text[0] == ' '): 2 else: 1 - for i in countup(0, sep - incr): addSon(a, n.sons[i]) - for i in countup(sep + 1, rsonsLen(n) - 2): addSon(b, n.sons[i]) - -proc parsePostfix(p: var TRstParser, n: PRstNode): PRstNode = - result = n - if isInlineMarkupEnd(p, "_"): - inc(p.idx) - if (p.tok[p.idx - 2].symbol == "`") and (p.tok[p.idx - 3].symbol == ">"): - var a = newRstNode(rnInner) - var b = newRstNode(rnInner) - fixupEmbeddedRef(n, a, b) - if rsonsLen(a) == 0: - result = newRstNode(rnStandaloneHyperlink) - addSon(result, b) - else: - result = newRstNode(rnHyperlink) - addSon(result, a) - addSon(result, b) - setRef(p, rstnodeToRefname(a), b) - elif n.kind == rnInterpretedText: - n.kind = rnRef - else: - result = newRstNode(rnRef) - addSon(result, n) - elif match(p, p.idx, ":w:"): - # a role: - if p.tok[p.idx + 1].symbol == "idx": - n.kind = rnIdx - elif p.tok[p.idx + 1].symbol == "literal": - n.kind = rnInlineLiteral - elif p.tok[p.idx + 1].symbol == "strong": - n.kind = rnStrongEmphasis - elif p.tok[p.idx + 1].symbol == "emphasis": - n.kind = rnEmphasis - elif (p.tok[p.idx + 1].symbol == "sub") or - (p.tok[p.idx + 1].symbol == "subscript"): - n.kind = rnSub - elif (p.tok[p.idx + 1].symbol == "sup") or - (p.tok[p.idx + 1].symbol == "supscript"): - n.kind = rnSup - else: - result = newRstNode(rnGeneralRole) - n.kind = rnInner - addSon(result, n) - addSon(result, newRstNode(rnLeaf, p.tok[p.idx + 1].symbol)) - inc(p.idx, 3) - -proc matchVerbatim(p: TRstParser, start: int, expr: string): int = - result = start - var j = 0 - while j < expr.len and continuesWith(expr, p.tok[result].symbol, j): - inc j, p.tok[result].symbol.len - inc result - if j < expr.len: result = 0 - -proc parseSmiley(p: var TRstParser): PRstNode = - if p.tok[p.idx].symbol[0] notin SmileyStartChars: return - for key, val in items(smilies): - let m = matchVerbatim(p, p.idx, key) - if m > 0: - p.idx = m - result = newRstNode(rnSmiley) - result.text = val - return - -proc isURL(p: TRstParser, i: int): bool = - result = (p.tok[i+1].symbol == ":") and (p.tok[i+2].symbol == "//") and - (p.tok[i+3].kind == tkWord) and (p.tok[i+4].symbol == ".") - -proc parseURL(p: var TRstParser, father: PRstNode) = - #if p.tok[p.idx].symbol[strStart] == '<': - if isURL(p, p.idx): - var n = newRstNode(rnStandaloneHyperlink) - while true: - case p.tok[p.idx].kind - of tkWord, tkAdornment, tkOther: nil - of tkPunct: - if p.tok[p.idx+1].kind notin {tkWord, tkAdornment, tkOther, tkPunct}: - break - else: break - addSon(n, newLeaf(p)) - inc(p.idx) - addSon(father, n) - else: - var n = newLeaf(p) - inc(p.idx) - if p.tok[p.idx].symbol == "_": n = parsePostfix(p, n) - addSon(father, n) - -proc parseBackslash(p: var TRstParser, father: PRstNode) = - assert(p.tok[p.idx].kind == tkPunct) - if p.tok[p.idx].symbol == "\\\\": - addSon(father, newRstNode(rnLeaf, "\\")) - inc(p.idx) - elif p.tok[p.idx].symbol == "\\": - # XXX: Unicode? - inc(p.idx) - if p.tok[p.idx].kind != tkWhite: addSon(father, newLeaf(p)) - inc(p.idx) - else: - addSon(father, newLeaf(p)) - inc(p.idx) - -when false: - proc parseAdhoc(p: var TRstParser, father: PRstNode, verbatim: bool) = - if not verbatim and isURL(p, p.idx): - var n = newRstNode(rnStandaloneHyperlink) - while true: - case p.tok[p.idx].kind - of tkWord, tkAdornment, tkOther: nil - of tkPunct: - if p.tok[p.idx+1].kind notin {tkWord, tkAdornment, tkOther, tkPunct}: - break - else: break - addSon(n, newLeaf(p)) - inc(p.idx) - addSon(father, n) - elif not verbatim and roSupportSmilies in p.shared.options: - let n = parseSmiley(p) - if s != nil: - addSon(father, n) - else: - var n = newLeaf(p) - inc(p.idx) - if p.tok[p.idx].symbol == "_": n = parsePostfix(p, n) - addSon(father, n) - -proc parseUntil(p: var TRstParser, father: PRstNode, postfix: string, - interpretBackslash: bool) = - while true: - case p.tok[p.idx].kind - of tkPunct: - if isInlineMarkupEnd(p, postfix): - inc(p.idx) - break - elif interpretBackslash: - parseBackslash(p, father) - else: - addSon(father, newLeaf(p)) - inc(p.idx) - of tkAdornment, tkWord, tkOther: - addSon(father, newLeaf(p)) - inc(p.idx) - of tkIndent: - addSon(father, newRstNode(rnLeaf, " ")) - inc(p.idx) - if p.tok[p.idx].kind == tkIndent: - rstMessage(p, errXExpected, postfix) - break - of tkWhite: - addSon(father, newRstNode(rnLeaf, " ")) - inc(p.idx) - else: rstMessage(p, errXExpected, postfix) - -proc parseInline(p: var TRstParser, father: PRstNode) = - case p.tok[p.idx].kind - of tkPunct: - if isInlineMarkupStart(p, "***"): - inc(p.idx) - var n = newRstNode(rnTripleEmphasis) - parseUntil(p, n, "***", true) - addSon(father, n) - elif isInlineMarkupStart(p, "**"): - inc(p.idx) - var n = newRstNode(rnStrongEmphasis) - parseUntil(p, n, "**", true) - addSon(father, n) - elif isInlineMarkupStart(p, "*"): - inc(p.idx) - var n = newRstNode(rnEmphasis) - parseUntil(p, n, "*", true) - addSon(father, n) - elif isInlineMarkupStart(p, "``"): - inc(p.idx) - var n = newRstNode(rnInlineLiteral) - parseUntil(p, n, "``", false) - addSon(father, n) - elif isInlineMarkupStart(p, "`"): - inc(p.idx) - var n = newRstNode(rnInterpretedText) - parseUntil(p, n, "`", true) - n = parsePostfix(p, n) - addSon(father, n) - elif isInlineMarkupStart(p, "|"): - inc(p.idx) - var n = newRstNode(rnSubstitutionReferences) - parseUntil(p, n, "|", false) - addSon(father, n) - else: - if roSupportSmilies in p.s.options: - let n = parseSmiley(p) - if n != nil: - addSon(father, n) - return - parseBackslash(p, father) - of tkWord: - if roSupportSmilies in p.s.options: - let n = parseSmiley(p) - if n != nil: - addSon(father, n) - return - parseURL(p, father) - of tkAdornment, tkOther, tkWhite: - if roSupportSmilies in p.s.options: - let n = parseSmiley(p) - if n != nil: - addSon(father, n) - return - addSon(father, newLeaf(p)) - inc(p.idx) - else: nil - -proc getDirective(p: var TRstParser): string = - if p.tok[p.idx].kind == tkWhite and p.tok[p.idx+1].kind == tkWord: - var j = p.idx - inc(p.idx) - result = p.tok[p.idx].symbol - inc(p.idx) - while p.tok[p.idx].kind in {tkWord, tkPunct, tkAdornment, tkOther}: - if p.tok[p.idx].symbol == "::": break - add(result, p.tok[p.idx].symbol) - inc(p.idx) - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - if p.tok[p.idx].symbol == "::": - inc(p.idx) - if (p.tok[p.idx].kind == tkWhite): inc(p.idx) - else: - p.idx = j # set back - result = "" # error - else: - result = "" - -proc parseComment(p: var TRstParser): PRstNode = - case p.tok[p.idx].kind - of tkIndent, tkEof: - if p.tok[p.idx + 1].kind == tkIndent: - inc(p.idx) # empty comment - else: - var indent = p.tok[p.idx].ival - while True: - case p.tok[p.idx].kind - of tkEof: - break - of tkIndent: - if (p.tok[p.idx].ival < indent): break - else: - nil - inc(p.idx) - else: - while p.tok[p.idx].kind notin {tkIndent, tkEof}: inc(p.idx) - result = nil - -type - TDirKind = enum # must be ordered alphabetically! - dkNone, dkAuthor, dkAuthors, dkCodeBlock, dkContainer, dkContents, - dkFigure, dkImage, dkInclude, dkIndex, dkRaw, dkTitle - -const - DirIds: array[0..11, string] = ["", "author", "authors", "code-block", - "container", "contents", "figure", "image", "include", "index", "raw", - "title"] - -proc getDirKind(s: string): TDirKind = - var i: int - i = binaryStrSearch(DirIds, s) - if i >= 0: result = TDirKind(i) - else: result = dkNone - -proc parseLine(p: var TRstParser, father: PRstNode) = - while True: - case p.tok[p.idx].kind - of tkWhite, tkWord, tkOther, tkPunct: parseInline(p, father) - else: break - -proc parseSection(p: var TRstParser, result: PRstNode) -proc parseField(p: var TRstParser): PRstNode = - result = newRstNode(rnField) - var col = p.tok[p.idx].col - inc(p.idx) # skip : - var fieldname = newRstNode(rnFieldname) - parseUntil(p, fieldname, ":", false) - var fieldbody = newRstNode(rnFieldbody) - if p.tok[p.idx].kind != tkIndent: parseLine(p, fieldbody) - if p.tok[p.idx].kind == tkIndent: - var indent = p.tok[p.idx].ival - if indent > col: - pushInd(p, indent) - parseSection(p, fieldbody) - popInd(p) - addSon(result, fieldname) - addSon(result, fieldbody) - -proc parseFields(p: var TRstParser): PRstNode = - result = nil - var atStart = p.idx == 0 and p.tok[0].symbol == ":" - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx + 1].symbol == ":") or - atStart: - var col = if atStart: p.tok[p.idx].col else: p.tok[p.idx].ival - result = newRstNode(rnFieldList) - if not atStart: inc(p.idx) - while true: - addSon(result, parseField(p)) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - (p.tok[p.idx + 1].symbol == ":"): - inc(p.idx) - else: - break - -proc getFieldValue(n: PRstNode, fieldname: string): string = - result = "" - if n.sons[1] == nil: return - if (n.sons[1].kind != rnFieldList): - #InternalError("getFieldValue (2): " & $n.sons[1].kind) - # We don't like internal errors here anymore as that would break the forum! - return - for i in countup(0, rsonsLen(n.sons[1]) - 1): - var f = n.sons[1].sons[i] - if cmpIgnoreStyle(addNodes(f.sons[0]), fieldname) == 0: - result = addNodes(f.sons[1]) - if result == "": result = "\x01\x01" # indicates that the field exists - return - -proc getArgument(n: PRstNode): string = - if n.sons[0] == nil: result = "" - else: result = addNodes(n.sons[0]) - -proc parseDotDot(p: var TRstParser): PRstNode -proc parseLiteralBlock(p: var TRstParser): PRstNode = - result = newRstNode(rnLiteralBlock) - var n = newRstNode(rnLeaf, "") - if p.tok[p.idx].kind == tkIndent: - var indent = p.tok[p.idx].ival - inc(p.idx) - while True: - case p.tok[p.idx].kind - of tkEof: - break - of tkIndent: - if (p.tok[p.idx].ival < indent): - break - else: - add(n.text, "\n") - add(n.text, repeatChar(p.tok[p.idx].ival - indent)) - inc(p.idx) - else: - add(n.text, p.tok[p.idx].symbol) - inc(p.idx) - else: - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - add(n.text, p.tok[p.idx].symbol) - inc(p.idx) - addSon(result, n) - -proc getLevel(map: var TLevelMap, lvl: var int, c: Char): int = - if map[c] == 0: - inc(lvl) - map[c] = lvl - result = map[c] - -proc tokenAfterNewline(p: TRstParser): int = - result = p.idx - while true: - case p.tok[result].kind - of tkEof: - break - of tkIndent: - inc(result) - break - else: inc(result) - -proc isLineBlock(p: TRstParser): bool = - var j = tokenAfterNewline(p) - result = (p.tok[p.idx].col == p.tok[j].col) and (p.tok[j].symbol == "|") or - (p.tok[j].col > p.tok[p.idx].col) - -proc predNL(p: TRstParser): bool = - result = true - if p.idx > 0: - result = p.tok[p.idx-1].kind == tkIndent and - p.tok[p.idx-1].ival == currInd(p) - -proc isDefList(p: TRstParser): bool = - var j = tokenAfterNewline(p) - result = (p.tok[p.idx].col < p.tok[j].col) and - (p.tok[j].kind in {tkWord, tkOther, tkPunct}) and - (p.tok[j - 2].symbol != "::") - -proc isOptionList(p: TRstParser): bool = - result = match(p, p.idx, "-w") or match(p, p.idx, "--w") or - match(p, p.idx, "/w") or match(p, p.idx, "//w") - -proc whichSection(p: TRstParser): TRstNodeKind = - case p.tok[p.idx].kind - of tkAdornment: - if match(p, p.idx + 1, "ii"): result = rnTransition - elif match(p, p.idx + 1, " a"): result = rnTable - elif match(p, p.idx + 1, "i"): result = rnOverline - else: result = rnLeaf - of tkPunct: - if match(p, tokenAfterNewLine(p), "ai"): - result = rnHeadline - elif p.tok[p.idx].symbol == "::": - result = rnLiteralBlock - elif predNL(p) and - ((p.tok[p.idx].symbol == "+") or (p.tok[p.idx].symbol == "*") or - (p.tok[p.idx].symbol == "-")) and (p.tok[p.idx + 1].kind == tkWhite): - result = rnBulletList - elif (p.tok[p.idx].symbol == "|") and isLineBlock(p): - result = rnLineBlock - elif (p.tok[p.idx].symbol == "..") and predNL(p): - result = rnDirective - elif match(p, p.idx, ":w:") and predNL(p): - # (p.tok[p.idx].symbol == ":") - result = rnFieldList - elif match(p, p.idx, "(e) "): - result = rnEnumList - elif match(p, p.idx, "+a+"): - result = rnGridTable - rstMessage(p, errGridTableNotImplemented) - elif isDefList(p): - result = rnDefList - elif isOptionList(p): - result = rnOptionList - else: - result = rnParagraph - of tkWord, tkOther, tkWhite: - if match(p, tokenAfterNewLine(p), "ai"): result = rnHeadline - elif isDefList(p): result = rnDefList - elif match(p, p.idx, "e) ") or match(p, p.idx, "e. "): result = rnEnumList - else: result = rnParagraph - else: result = rnLeaf - -proc parseLineBlock(p: var TRstParser): PRstNode = - result = nil - if p.tok[p.idx + 1].kind == tkWhite: - var col = p.tok[p.idx].col - result = newRstNode(rnLineBlock) - pushInd(p, p.tok[p.idx + 2].col) - inc(p.idx, 2) - while true: - var item = newRstNode(rnLineBlockItem) - parseSection(p, item) - addSon(result, item) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - (p.tok[p.idx + 1].symbol == "|") and - (p.tok[p.idx + 2].kind == tkWhite): - inc(p.idx, 3) - else: - break - popInd(p) - -proc parseParagraph(p: var TRstParser, result: PRstNode) = - while True: - case p.tok[p.idx].kind - of tkIndent: - if p.tok[p.idx + 1].kind == tkIndent: - inc(p.idx) - break - elif (p.tok[p.idx].ival == currInd(p)): - inc(p.idx) - case whichSection(p) - of rnParagraph, rnLeaf, rnHeadline, rnOverline, rnDirective: - addSon(result, newRstNode(rnLeaf, " ")) - of rnLineBlock: - addSonIfNotNil(result, parseLineBlock(p)) - else: break - else: - break - of tkPunct: - if (p.tok[p.idx].symbol == "::") and - (p.tok[p.idx + 1].kind == tkIndent) and - (currInd(p) < p.tok[p.idx + 1].ival): - addSon(result, newRstNode(rnLeaf, ":")) - inc(p.idx) # skip '::' - addSon(result, parseLiteralBlock(p)) - break - else: - parseInline(p, result) - of tkWhite, tkWord, tkAdornment, tkOther: - parseInline(p, result) - else: break - -proc parseParagraphWrapper(p: var TRstParser): PRstNode = - result = newRstNode(rnParagraph) - parseParagraph(p, result) - -proc parseHeadline(p: var TRstParser): PRstNode = - result = newRstNode(rnHeadline) - parseLine(p, result) - assert(p.tok[p.idx].kind == tkIndent) - assert(p.tok[p.idx + 1].kind == tkAdornment) - var c = p.tok[p.idx + 1].symbol[0] - inc(p.idx, 2) - result.level = getLevel(p.s.underlineToLevel, p.s.uLevel, c) - -type - TIntSeq = seq[int] - -proc tokEnd(p: TRstParser): int = - result = p.tok[p.idx].col + len(p.tok[p.idx].symbol) - 1 - -proc getColumns(p: var TRstParser, cols: var TIntSeq) = - var L = 0 - while true: - inc(L) - setlen(cols, L) - cols[L - 1] = tokEnd(p) - assert(p.tok[p.idx].kind == tkAdornment) - inc(p.idx) - if p.tok[p.idx].kind != tkWhite: break - inc(p.idx) - if p.tok[p.idx].kind != tkAdornment: break - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - # last column has no limit: - cols[L - 1] = 32000 - -proc parseDoc(p: var TRstParser): PRstNode - -proc parseSimpleTable(p: var TRstParser): PRstNode = - var - cols: TIntSeq - row: seq[string] - i, last, line: int - c: Char - q: TRstParser - a, b: PRstNode - result = newRstNode(rnTable) - cols = @[] - row = @[] - a = nil - c = p.tok[p.idx].symbol[0] - while true: - if p.tok[p.idx].kind == tkAdornment: - last = tokenAfterNewline(p) - if p.tok[last].kind in {tkEof, tkIndent}: - # skip last adornment line: - p.idx = last - break - getColumns(p, cols) - setlen(row, len(cols)) - if a != nil: - for j in 0..rsonsLen(a)-1: a.sons[j].kind = rnTableHeaderCell - if p.tok[p.idx].kind == tkEof: break - for j in countup(0, high(row)): row[j] = "" - # the following while loop iterates over the lines a single cell may span: - line = p.tok[p.idx].line - while true: - i = 0 - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - if (tokEnd(p) <= cols[i]): - add(row[i], p.tok[p.idx].symbol) - inc(p.idx) - else: - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - inc(i) - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - if tokEnd(p) <= cols[0]: break - if p.tok[p.idx].kind in {tkEof, tkAdornment}: break - for j in countup(1, high(row)): add(row[j], '\x0A') - a = newRstNode(rnTableRow) - for j in countup(0, high(row)): - initParser(q, p.s) - q.col = cols[j] - q.line = line - 1 - q.filename = p.filename - getTokens(row[j], false, q.tok) - b = newRstNode(rnTableDataCell) - addSon(b, parseDoc(q)) - addSon(a, b) - addSon(result, a) - -proc parseTransition(p: var TRstParser): PRstNode = - result = newRstNode(rnTransition) - inc(p.idx) - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - -proc parseOverline(p: var TRstParser): PRstNode = - var c = p.tok[p.idx].symbol[0] - inc(p.idx, 2) - result = newRstNode(rnOverline) - while true: - parseLine(p, result) - if p.tok[p.idx].kind == tkIndent: - inc(p.idx) - if p.tok[p.idx - 1].ival > currInd(p): - addSon(result, newRstNode(rnLeaf, " ")) - else: - break - else: - break - result.level = getLevel(p.s.overlineToLevel, p.s.oLevel, c) - if p.tok[p.idx].kind == tkAdornment: - inc(p.idx) # XXX: check? - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - -proc parseBulletList(p: var TRstParser): PRstNode = - result = nil - if p.tok[p.idx + 1].kind == tkWhite: - var bullet = p.tok[p.idx].symbol - var col = p.tok[p.idx].col - result = newRstNode(rnBulletList) - pushInd(p, p.tok[p.idx + 2].col) - inc(p.idx, 2) - while true: - var item = newRstNode(rnBulletItem) - parseSection(p, item) - addSon(result, item) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - (p.tok[p.idx + 1].symbol == bullet) and - (p.tok[p.idx + 2].kind == tkWhite): - inc(p.idx, 3) - else: - break - popInd(p) - -proc parseOptionList(p: var TRstParser): PRstNode = - result = newRstNode(rnOptionList) - while true: - if isOptionList(p): - var a = newRstNode(rnOptionGroup) - var b = newRstNode(rnDescription) - var c = newRstNode(rnOptionListItem) - if match(p, p.idx, "//w"): inc(p.idx) - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - if (p.tok[p.idx].kind == tkWhite) and (len(p.tok[p.idx].symbol) > 1): - inc(p.idx) - break - addSon(a, newLeaf(p)) - inc(p.idx) - var j = tokenAfterNewline(p) - if (j > 0) and (p.tok[j - 1].kind == tkIndent) and - (p.tok[j - 1].ival > currInd(p)): - pushInd(p, p.tok[j - 1].ival) - parseSection(p, b) - popInd(p) - else: - parseLine(p, b) - if (p.tok[p.idx].kind == tkIndent): inc(p.idx) - addSon(c, a) - addSon(c, b) - addSon(result, c) - else: - break - -proc parseDefinitionList(p: var TRstParser): PRstNode = - result = nil - var j = tokenAfterNewLine(p) - 1 - if (j >= 1) and (p.tok[j].kind == tkIndent) and - (p.tok[j].ival > currInd(p)) and (p.tok[j - 1].symbol != "::"): - var col = p.tok[p.idx].col - result = newRstNode(rnDefList) - while true: - j = p.idx - var a = newRstNode(rnDefName) - parseLine(p, a) - if (p.tok[p.idx].kind == tkIndent) and - (p.tok[p.idx].ival > currInd(p)) and - (p.tok[p.idx + 1].symbol != "::") and - not (p.tok[p.idx + 1].kind in {tkIndent, tkEof}): - pushInd(p, p.tok[p.idx].ival) - var b = newRstNode(rnDefBody) - parseSection(p, b) - var c = newRstNode(rnDefItem) - addSon(c, a) - addSon(c, b) - addSon(result, c) - popInd(p) - else: - p.idx = j - break - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col): - inc(p.idx) - j = tokenAfterNewLine(p) - 1 - if j >= 1 and p.tok[j].kind == tkIndent and p.tok[j].ival > col and - p.tok[j-1].symbol != "::" and p.tok[j+1].kind != tkIndent: - nil - else: - break - if rsonsLen(result) == 0: result = nil - -proc parseEnumList(p: var TRstParser): PRstNode = - const - wildcards: array[0..2, string] = ["(e) ", "e) ", "e. "] - wildpos: array[0..2, int] = [1, 0, 0] - result = nil - var w = 0 - while w <= 2: - if match(p, p.idx, wildcards[w]): break - inc(w) - if w <= 2: - var col = p.tok[p.idx].col - result = newRstNode(rnEnumList) - inc(p.idx, wildpos[w] + 3) - var j = tokenAfterNewLine(p) - if (p.tok[j].col == p.tok[p.idx].col) or match(p, j, wildcards[w]): - pushInd(p, p.tok[p.idx].col) - while true: - var item = newRstNode(rnEnumItem) - parseSection(p, item) - addSon(result, item) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - match(p, p.idx + 1, wildcards[w]): - inc(p.idx, wildpos[w] + 4) - else: - break - popInd(p) - else: - dec(p.idx, wildpos[w] + 3) - result = nil - -proc sonKind(father: PRstNode, i: int): TRstNodeKind = - result = rnLeaf - if i < rsonsLen(father): result = father.sons[i].kind - -proc parseSection(p: var TRstParser, result: PRstNode) = - while true: - var leave = false - assert(p.idx >= 0) - while p.tok[p.idx].kind == tkIndent: - if currInd(p) == p.tok[p.idx].ival: - inc(p.idx) - elif p.tok[p.idx].ival > currInd(p): - pushInd(p, p.tok[p.idx].ival) - var a = newRstNode(rnBlockQuote) - parseSection(p, a) - addSon(result, a) - popInd(p) - else: - leave = true - break - if leave: break - if p.tok[p.idx].kind == tkEof: break - var a: PRstNode = nil - var k = whichSection(p) - case k - of rnLiteralBlock: - inc(p.idx) # skip '::' - a = parseLiteralBlock(p) - of rnBulletList: a = parseBulletList(p) - of rnLineblock: a = parseLineBlock(p) - of rnDirective: a = parseDotDot(p) - of rnEnumList: a = parseEnumList(p) - of rnLeaf: rstMessage(p, errNewSectionExpected) - of rnParagraph: nil - of rnDefList: a = parseDefinitionList(p) - of rnFieldList: - if p.idx > 0: dec(p.idx) - a = parseFields(p) - of rnTransition: a = parseTransition(p) - of rnHeadline: a = parseHeadline(p) - of rnOverline: a = parseOverline(p) - of rnTable: a = parseSimpleTable(p) - of rnOptionList: a = parseOptionList(p) - else: - #InternalError("rst.parseSection()") - nil - if a == nil and k != rnDirective: - a = newRstNode(rnParagraph) - parseParagraph(p, a) - addSonIfNotNil(result, a) - if sonKind(result, 0) == rnParagraph and sonKind(result, 1) != rnParagraph: - result.sons[0].kind = rnInner - -proc parseSectionWrapper(p: var TRstParser): PRstNode = - result = newRstNode(rnInner) - parseSection(p, result) - while (result.kind == rnInner) and (rsonsLen(result) == 1): - result = result.sons[0] - -proc parseDoc(p: var TRstParser): PRstNode = - result = parseSectionWrapper(p) - if p.tok[p.idx].kind != tkEof: rstMessage(p, errGeneralParseError) - -type - TDirFlag = enum - hasArg, hasOptions, argIsFile, argIsWord - TDirFlags = set[TDirFlag] - TSectionParser = proc (p: var TRstParser): PRstNode - -proc parseDirective(p: var TRstParser, flags: TDirFlags): PRstNode = - result = newRstNode(rnDirective) - var args: PRstNode = nil - var options: PRstNode = nil - if hasArg in flags: - args = newRstNode(rnDirArg) - if argIsFile in flags: - while True: - case p.tok[p.idx].kind - of tkWord, tkOther, tkPunct, tkAdornment: - addSon(args, newLeaf(p)) - inc(p.idx) - else: break - elif argIsWord in flags: - while p.tok[p.idx].kind == tkWhite: inc(p.idx) - if p.tok[p.idx].kind == tkWord: - addSon(args, newLeaf(p)) - inc(p.idx) - else: - args = nil - else: - parseLine(p, args) - addSon(result, args) - if hasOptions in flags: - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival >= 3) and - (p.tok[p.idx + 1].symbol == ":"): - options = parseFields(p) - addSon(result, options) - -proc indFollows(p: TRstParser): bool = - result = p.tok[p.idx].kind == tkIndent and p.tok[p.idx].ival > currInd(p) - -proc parseDirective(p: var TRstParser, flags: TDirFlags, - contentParser: TSectionParser): PRstNode = - result = parseDirective(p, flags) - if not isNil(contentParser) and indFollows(p): - pushInd(p, p.tok[p.idx].ival) - var content = contentParser(p) - popInd(p) - addSon(result, content) - else: - addSon(result, nil) - -proc parseDirBody(p: var TRstParser, contentParser: TSectionParser): PRstNode = - if indFollows(p): - pushInd(p, p.tok[p.idx].ival) - result = contentParser(p) - popInd(p) - -proc dirInclude(p: var TRstParser): PRstNode = - # - #The following options are recognized: - # - #start-after : text to find in the external data file - # Only the content after the first occurrence of the specified text will - # be included. - #end-before : text to find in the external data file - # Only the content before the first occurrence of the specified text - # (but after any after text) will be included. - #literal : flag (empty) - # The entire included text is inserted into the document as a single - # literal block (useful for program listings). - #encoding : name of text encoding - # The text encoding of the external data file. Defaults to the document's - # encoding (if specified). - # - result = nil - var n = parseDirective(p, {hasArg, argIsFile, hasOptions}, nil) - var filename = strip(addNodes(n.sons[0])) - var path = findFile(filename) - if path == "": - rstMessage(p, errCannotOpenFile, filename) - else: - # XXX: error handling; recursive file inclusion! - if getFieldValue(n, "literal") != "": - result = newRstNode(rnLiteralBlock) - addSon(result, newRstNode(rnLeaf, readFile(path))) - else: - var q: TRstParser - initParser(q, p.s) - q.filename = filename - getTokens(readFile(path), false, q.tok) - # workaround a GCC bug; more like the interior pointer bug? - #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0: - # InternalError("Too many binary zeros in include file") - result = parseDoc(q) - -proc dirCodeBlock(p: var TRstParser): PRstNode = - result = parseDirective(p, {hasArg, hasOptions}, parseLiteralBlock) - var filename = strip(getFieldValue(result, "file")) - if filename != "": - var path = findFile(filename) - if path == "": rstMessage(p, errCannotOpenFile, filename) - var n = newRstNode(rnLiteralBlock) - addSon(n, newRstNode(rnLeaf, readFile(path))) - result.sons[2] = n - result.kind = rnCodeBlock - -proc dirContainer(p: var TRstParser): PRstNode = - result = parseDirective(p, {hasArg}, parseSectionWrapper) - assert(result.kind == rnDirective) - assert(rsonsLen(result) == 3) - result.kind = rnContainer - -proc dirImage(p: var TRstParser): PRstNode = - result = parseDirective(p, {hasOptions, hasArg, argIsFile}, nil) - result.kind = rnImage - -proc dirFigure(p: var TRstParser): PRstNode = - result = parseDirective(p, {hasOptions, hasArg, argIsFile}, - parseSectionWrapper) - result.kind = rnFigure - -proc dirTitle(p: var TRstParser): PRstNode = - result = parseDirective(p, {hasArg}, nil) - result.kind = rnTitle - -proc dirContents(p: var TRstParser): PRstNode = - result = parseDirective(p, {hasArg}, nil) - result.kind = rnContents - -proc dirIndex(p: var TRstParser): PRstNode = - result = parseDirective(p, {}, parseSectionWrapper) - result.kind = rnIndex - -proc dirRawAux(p: var TRstParser, result: var PRstNode, kind: TRstNodeKind, - contentParser: TSectionParser) = - var filename = getFieldValue(result, "file") - if filename.len > 0: - var path = findFile(filename) - if path.len == 0: - rstMessage(p, errCannotOpenFile, filename) - else: - var f = readFile(path) - result = newRstNode(kind) - addSon(result, newRstNode(rnLeaf, f)) - else: - result.kind = kind - addSon(result, parseDirBody(p, contentParser)) - -proc dirRaw(p: var TRstParser): PRstNode = - # - #The following options are recognized: - # - #file : string (newlines removed) - # The local filesystem path of a raw data file to be included. - # - # html - # latex - result = parseDirective(p, {hasOptions, hasArg, argIsWord}) - if result.sons[0] != nil: - if cmpIgnoreCase(result.sons[0].sons[0].text, "html") == 0: - dirRawAux(p, result, rnRawHtml, parseLiteralBlock) - elif cmpIgnoreCase(result.sons[0].sons[0].text, "latex") == 0: - dirRawAux(p, result, rnRawLatex, parseLiteralBlock) - else: - rstMessage(p, errInvalidDirectiveX, result.sons[0].text) - else: - dirRawAux(p, result, rnRaw, parseSectionWrapper) - -proc parseDotDot(p: var TRstParser): PRstNode = - result = nil - var col = p.tok[p.idx].col - inc(p.idx) - var d = getDirective(p) - if d != "": - pushInd(p, col) - case getDirKind(d) - of dkInclude: result = dirInclude(p) - of dkImage: result = dirImage(p) - of dkFigure: result = dirFigure(p) - of dkTitle: result = dirTitle(p) - of dkContainer: result = dirContainer(p) - of dkContents: result = dirContents(p) - of dkRaw: - if roSupportRawDirective in p.s.options: - result = dirRaw(p) - else: - rstMessage(p, errInvalidDirectiveX, d) - of dkCodeblock: result = dirCodeBlock(p) - of dkIndex: result = dirIndex(p) - else: rstMessage(p, errInvalidDirectiveX, d) - popInd(p) - elif match(p, p.idx, " _"): - # hyperlink target: - inc(p.idx, 2) - var a = getReferenceName(p, ":") - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - var b = untilEol(p) - setRef(p, rstnodeToRefname(a), b) - elif match(p, p.idx, " |"): - # substitution definitions: - inc(p.idx, 2) - var a = getReferenceName(p, "|") - var b: PRstNode - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - if cmpIgnoreStyle(p.tok[p.idx].symbol, "replace") == 0: - inc(p.idx) - expect(p, "::") - b = untilEol(p) - elif cmpIgnoreStyle(p.tok[p.idx].symbol, "image") == 0: - inc(p.idx) - b = dirImage(p) - else: - rstMessage(p, errInvalidDirectiveX, p.tok[p.idx].symbol) - setSub(p, addNodes(a), b) - elif match(p, p.idx, " ["): - # footnotes, citations - inc(p.idx, 2) - var a = getReferenceName(p, "]") - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - var b = untilEol(p) - setRef(p, rstnodeToRefname(a), b) - else: - result = parseComment(p) - -proc resolveSubs(p: var TRstParser, n: PRstNode): PRstNode = - result = n - if n == nil: return - case n.kind - of rnSubstitutionReferences: - var x = findSub(p, n) - if x >= 0: - result = p.s.subs[x].value - else: - var key = addNodes(n) - var e = getEnv(key) - if e != "": result = newRstNode(rnLeaf, e) - else: rstMessage(p, warnUnknownSubstitutionX, key) - of rnRef: - var y = findRef(p, rstnodeToRefname(n)) - if y != nil: - result = newRstNode(rnHyperlink) - n.kind = rnInner - addSon(result, n) - addSon(result, y) - of rnLeaf: - nil - of rnContents: - p.hasToc = true - else: - for i in countup(0, rsonsLen(n) - 1): n.sons[i] = resolveSubs(p, n.sons[i]) - -proc rstParse(text, filename: string, - line, column: int, hasToc: var bool, - options: TRstParseOptions): PRstNode = - var p: TRstParser - if isNil(text): rawMessage(errCannotOpenFile, filename) - initParser(p, newSharedState(options)) - p.filename = filename - p.line = line - p.col = column - getTokens(text, roSkipPounds in options, p.tok) - result = resolveSubs(p, parseDoc(p)) - hasToc = p.hasToc |