diff options
Diffstat (limited to 'lib/packages/docutils/rst.nim')
| -rw-r--r-- | lib/packages/docutils/rst.nim | 263 |
1 files changed, 236 insertions, 27 deletions
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index e9f76a26f..b3b1fb9c0 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -22,7 +22,7 @@ import os, strutils, rstast, dochelpers, std/enumutils, algorithm, lists, sequtils, - std/private/miscdollars, tables, strscans + std/private/miscdollars, tables, strscans, rstidx from highlite import SourceLanguage, getSourceLanguage when defined(nimPreviewSlimSystem): @@ -40,7 +40,7 @@ type roNimFile ## set for Nim files where default interpreted ## text role should be :nim: roSandboxDisabled ## this option enables certain options - ## (e.g. raw, include) + ## (e.g. raw, include, importdoc) ## which are disabled by default as they can ## enable users to read arbitrary data and ## perform XSS if the parser is used in a web @@ -73,11 +73,17 @@ type mwUnsupportedLanguage = "language '$1' not supported", mwUnsupportedField = "field '$1' not supported", mwRstStyle = "RST style: $1", + mwUnusedImportdoc = "importdoc for '$1' is not used", meSandboxedDirective = "disabled directive: '$1'", MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind, arg: string) {.closure, gcsafe.} ## what to do in case of an error FindFileHandler* = proc (filename: string): string {.closure, gcsafe.} + FindRefFileHandler* = + proc (targetRelPath: string): + tuple[targetPath: string, linkRelPath: string] {.closure, gcsafe.} + ## returns where .html or .idx file should be found by its relative path; + ## `linkRelPath` is a prefix to be added before a link anchor from such file proc rstnodeToRefname*(n: PRstNode): string proc addNodes*(n: PRstNode): string @@ -333,7 +339,8 @@ type arInternalRst, ## For automatically generated RST anchors (from ## headings, footnotes, inline internal targets): ## case-insensitive, 1-space-significant (by RST spec) - arNim ## For anchors generated by ``docgen.rst``: Nim-style case + arExternalRst, ## For external .nim doc comments or .rst/.md + arNim ## For anchors generated by ``docgen.nim``: Nim-style case ## sensitivity, etc. (see `proc normalizeNimName`_ for details) arHyperlink, ## For links with manually set anchors in ## form `text <pagename.html#anchor>`_ @@ -349,11 +356,15 @@ type of arInternalRst: anchorType: RstAnchorKind target: PRstNode + of arExternalRst: + anchorTypeExt: RstAnchorKind + refnameExt: string of arNim: tooltip: string # displayed tooltip for Nim-generated anchors langSym: LangSymbol refname: string # A reference name that will be inserted directly # into HTML/Latex. + external: bool AnchorSubstTable = Table[string, seq[AnchorSubst]] # use `seq` to account for duplicate anchors FootnoteType = enum @@ -372,6 +383,12 @@ type RstFileTable* = object filenameToIdx*: Table[string, FileIndex] idxToFilename*: seq[string] + ImportdocInfo = object + used: bool # was this import used? + fromInfo: TLineInfo # place of `.. importdoc::` directive + idxPath: string # full path to ``.idx`` file + linkRelPath: string # prefix before target anchor + title: string # document title obtained from ``.idx`` RstSharedState = object options*: RstParseOptions # parsing options hLevels: LevelMap # hierarchy of heading styles @@ -393,12 +410,17 @@ type footnotes: seq[FootnoteSubst] # correspondence b/w footnote label, # number, order of occurrence msgHandler: MsgHandler # How to handle errors. - findFile: FindFileHandler # How to find files. + findFile: FindFileHandler # How to find files for include. + findRefFile: FindRefFileHandler + # How to find files imported by importdoc. filenames*: RstFileTable # map file name <-> FileIndex (for storing # file names for warnings after 1st stage) currFileIdx*: FileIndex # current index in `filenames` tocPart*: seq[PRstNode] # all the headings of a document hasToc*: bool + idxImports*: Table[string, ImportdocInfo] + # map `importdoc`ed filename -> it's info + nimFileImported*: bool # Was any ``.nim`` module `importdoc`ed ? PRstSharedState* = ref RstSharedState ManualAnchor = object @@ -452,6 +474,9 @@ proc defaultFindFile*(filename: string): string = if fileExists(filename): result = filename else: result = "" +proc defaultFindRefFile*(filename: string): (string, string) = + (filename, "") + proc defaultRole(options: RstParseOptions): string = if roNimFile in options: "nim" else: "literal" @@ -492,12 +517,16 @@ proc getFilename(filenames: RstFileTable, fid: FileIndex): string = $fid.int, $(filenames.len - 1)]) result = filenames.idxToFilename[fid.int] +proc getFilename(s: PRstSharedState, subst: AnchorSubst): string = + getFilename(s.filenames, subst.info.fileIndex) + proc currFilename(s: PRstSharedState): string = getFilename(s.filenames, s.currFileIdx) proc newRstSharedState*(options: RstParseOptions, filename: string, findFile: FindFileHandler, + findRefFile: FindRefFileHandler, msgHandler: MsgHandler, hasToc: bool): PRstSharedState = let r = defaultRole(options) @@ -507,6 +536,9 @@ proc newRstSharedState*(options: RstParseOptions, options: options, msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler, findFile: if not isNil(findFile): findFile else: defaultFindFile, + findRefFile: + if not isNil(findRefFile): findRefFile + else: defaultFindRefFile, hasToc: hasToc ) setCurrFilename(result, filename) @@ -525,6 +557,14 @@ proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg) +proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string; + line, col: int) = + s.msgHandler(s.currFilename, line, col, msgKind, arg) + +proc rstMessage(s: PRstSharedState, filename: string, msgKind: MsgKind, + arg: string) = + s.msgHandler(filename, LineRstInit, ColRstInit, msgKind, arg) + proc rstMessage*(filenames: RstFileTable, f: MsgHandler, info: TLineInfo, msgKind: MsgKind, arg: string) = ## Print warnings using `info`, i.e. in 2nd-pass warnings for @@ -756,6 +796,14 @@ proc internalRefPriority(k: RstAnchorKind): int = of footnoteAnchor: result = 4 of headlineAnchor: result = 3 +proc `$`(subst: AnchorSubst): string = # for debug + let s = + case subst.kind + of arInternalRst: "type=" & $subst.anchorType + of arExternalRst: "type=" & $subst.anchorTypeExt + of arNim: "langsym=" & $subst.langSym + result = "(kind=$1, priority=$2, $3)" % [$subst.kind, $subst.priority, s] + proc addAnchorRst(p: var RstParser, name: string, target: PRstNode, anchorType: RstAnchorKind) = ## Associates node `target` (which has field `anchor`) with an @@ -771,31 +819,49 @@ proc addAnchorRst(p: var RstParser, name: string, target: PRstNode, info: prevLineInfo(p), anchorType: anchorType)) p.curAnchors.setLen 0 -proc addAnchorNim*(s: var PRstSharedState, refn: string, tooltip: string, +proc addAnchorExtRst(s: var PRstSharedState, key: string, refn: string, + anchorType: RstAnchorKind, info: TLineInfo) = + let name = key.toLowerAscii + let prio = internalRefPriority(anchorType) + s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arExternalRst, refnameExt: refn, priority: prio, + info: info, + anchorTypeExt: anchorType)) + +proc addAnchorNim*(s: var PRstSharedState, external: bool, refn: string, tooltip: string, langSym: LangSymbol, priority: int, info: TLineInfo) = ## Adds an anchor `refn`, which follows ## the rule `arNim` (i.e. a symbol in ``*.nim`` file) s.anchors.mgetOrPut(langSym.name, newSeq[AnchorSubst]()).add( - AnchorSubst(kind: arNim, refname: refn, langSym: langSym, + AnchorSubst(kind: arNim, external: external, refname: refn, langSym: langSym, tooltip: tooltip, priority: priority, info: info)) proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, info: TLineInfo): seq[AnchorSubst] = - let langSym = toLangSymbol(signature) + var langSym: LangSymbol + try: + langSym = toLangSymbol(signature) + except ValueError: # parsing failed, not a Nim symbol + return let substitutions = s.anchors.getOrDefault(langSym.name, newSeq[AnchorSubst]()) if substitutions.len == 0: return - # map symKind (like "proc") -> found symbols/groups: - var found: Table[string, seq[AnchorSubst]] - for s in substitutions: - if s.kind == arNim: - if match(s.langSym, langSym): - found.mgetOrPut(s.langSym.symKind, newSeq[AnchorSubst]()).add s - for symKind, sList in found: + # logic to select only groups instead of concrete symbols + # with overloads, note that the same symbol can be defined + # in multiple modules and `importdoc`ed: + type GroupKey = tuple[symKind: string, origModule: string] + # map (symKind, file) (like "proc", "os.nim") -> found symbols/groups: + var found: Table[GroupKey, seq[AnchorSubst]] + for subst in substitutions: + if subst.kind == arNim: + if match(subst.langSym, langSym): + let key: GroupKey = (subst.langSym.symKind, getFilename(s, subst)) + found.mgetOrPut(key, newSeq[AnchorSubst]()).add subst + for key, sList in found: if sList.len == 1: result.add sList[0] else: # > 1, there are overloads, potential ambiguity in this `symKind` @@ -812,14 +878,16 @@ proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, result.add s foundGroup = true break - doAssert foundGroup, "docgen has not generated the group" + doAssert(foundGroup, + "docgen has not generated the group for $1 (file $2)" % [ + langSym.name, getFilename(s, sList[0]) ]) proc findMainAnchorRst(s: PRstSharedState, linkText: string, info: TLineInfo): seq[AnchorSubst] = let name = linkText.toLowerAscii let substitutions = s.anchors.getOrDefault(name, newSeq[AnchorSubst]()) for s in substitutions: - if s.kind == arInternalRst: + if s.kind in {arInternalRst, arExternalRst}: result.add s proc addFootnoteNumManual(p: var RstParser, num: int) = @@ -3251,6 +3319,15 @@ proc dirRaw(p: var RstParser): PRstNode = else: dirRawAux(p, result, rnRaw, parseSectionWrapper) +proc dirImportdoc(p: var RstParser): PRstNode = + result = parseDirective(p, rnDirective, {}, parseLiteralBlock) + assert result.sons[2].kind == rnLiteralBlock + assert result.sons[2].sons[0].kind == rnLeaf + let filenames: seq[string] = split(result.sons[2].sons[0].text, seps = {','}) + proc rmSpaces(s: string): string = s.split.join("") + for origFilename in filenames: + p.s.idxImports[origFilename.rmSpaces] = ImportdocInfo(fromInfo: lineInfo(p)) + proc selectDir(p: var RstParser, d: string): PRstNode = result = nil let tok = p.tok[p.idx-2] # report on directive in ".. directive::" @@ -3271,6 +3348,7 @@ proc selectDir(p: var RstParser, d: string): PRstNode = of "hint": result = dirAdmonition(p, d) of "image": result = dirImage(p) of "important": result = dirAdmonition(p, d) + of "importdoc": result = dirImportdoc(p) of "include": result = dirInclude(p) of "index": result = dirIndex(p) of "note": result = dirAdmonition(p, d) @@ -3401,11 +3479,90 @@ proc rstParsePass1*(fragment: string, getTokens(fragment, p.tok) result = parseDoc(p) -proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) = +proc extractLinkEnd(x: string): string = + ## From links like `path/to/file.html#/%` extract `file.html#/%`. + let i = find(x, '#') + let last = + if i >= 0: i + else: x.len - 1 + let j = rfind(x, '/', start=0, last=last) + if j >= 0: + result = x[j+1 .. ^1] + else: + result = x + +proc loadIdxFile(s: var PRstSharedState, origFilename: string) = + doAssert roSandboxDisabled in s.options + var info: TLineInfo + info.fileIndex = addFilename(s, origFilename) + var (dir, basename, ext) = origFilename.splitFile + if ext notin [".md", ".rst", ".nim", ""]: + rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo, + meCannotOpenFile, origFilename & ": unknown extension") + let idxFilename = dir / basename & ".idx" + let (idxPath, linkRelPath) = s.findRefFile(idxFilename) + s.idxImports[origFilename].linkRelPath = linkRelPath + var + fileEntries: seq[IndexEntry] + title: IndexEntry + try: + (fileEntries, title) = parseIdxFile(idxPath) + except IOError: + rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo, + meCannotOpenFile, idxPath) + except ValueError as e: + s.msgHandler(idxPath, LineRstInit, ColRstInit, meInvalidField, e.msg) + + var isMarkup = false # for sanity check to avoid mixing .md <-> .nim + for entry in fileEntries: + # Though target .idx already has inside it the path to HTML relative + # project's root, we won't rely on it and use `linkRelPath` instead. + let refn = extractLinkEnd(entry.link) + # select either markup (rst/md) or Nim cases: + if entry.kind in {ieMarkupTitle, ieNimTitle}: + s.idxImports[origFilename].title = entry.keyword + case entry.kind + of ieIdxRole, ieHeading, ieMarkupTitle: + if ext == ".nim" and entry.kind == ieMarkupTitle: + rstMessage(s, idxPath, meInvalidField, + $ieMarkupTitle & " in supposedly .nim-derived file") + if entry.kind == ieMarkupTitle: + isMarkup = true + info.line = entry.line.uint16 + addAnchorExtRst(s, key = entry.keyword, refn = refn, + anchorType = headlineAnchor, info=info) + of ieNim, ieNimGroup, ieNimTitle: + if ext in [".md", ".rst"] or isMarkup: + rstMessage(s, idxPath, meInvalidField, + $entry.kind & " in supposedly markup-derived file") + s.nimFileImported = true + var langSym: LangSymbol + if entry.kind in {ieNim, ieNimTitle}: + var q: RstParser + initParser(q, s) + info.line = entry.line.uint16 + setLen(q.tok, 0) + q.idx = 0 + getTokens(entry.linkTitle, q.tok) + var sons = newSeq[PRstNode](q.tok.len) + for i in 0 ..< q.tok.len: sons[i] = newLeaf(q.tok[i].symbol) + let linkTitle = newRstNode(rnInner, sons) + langSym = linkTitle.toLangSymbol + else: # entry.kind == ieNimGroup + langSym = langSymbolGroup(kind=entry.linkTitle, name=entry.keyword) + addAnchorNim(s, external = true, refn = refn, tooltip = entry.linkDesc, + langSym = langSym, priority = -4, # lowest + info=info) + doAssert s.idxImports[origFilename].title != "" + +proc preparePass2*(s: var PRstSharedState, mainNode: PRstNode, importdoc = true) = ## Records titles in node `mainNode` and orders footnotes. countTitles(s, mainNode) fixHeadlines(s) orderFootnotes(s) + if importdoc: + for origFilename in s.idxImports.keys: + loadIdxFile(s, origFilename) proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = # Associate this link alias with its target and change node kind to @@ -3423,6 +3580,9 @@ proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = tooltip: string target: PRstNode info: TLineInfo + externFilename: string + # when external anchor: origin filename where anchor was defined + isTitle: bool proc cmp(x, y: LinkDef): int = result = cmp(x.priority, y.priority) if result == 0: @@ -3435,26 +3595,67 @@ proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = target: y.value, info: y.info, tooltip: "(" & $y.kind & ")") let substRst = findMainAnchorRst(s, alias.addNodes, n.info) + template getExternFilename(subst: AnchorSubst): string = + if subst.kind == arExternalRst or + (subst.kind == arNim and subst.external): + getFilename(s, subst) + else: "" for subst in substRst: - foundLinks.add LinkDef(ar: arInternalRst, priority: subst.priority, - target: newLeaf(subst.target.anchor), + var refname, fullRefname: string + if subst.kind == arInternalRst: + refname = subst.target.anchor + fullRefname = refname + else: # arExternalRst + refname = subst.refnameExt + fullRefname = s.idxImports[getFilename(s, subst)].linkRelPath & + "/" & refname + let anchorType = + if subst.kind == arInternalRst: subst.anchorType + else: subst.anchorTypeExt # arExternalRst + foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority, + target: newLeaf(fullRefname), info: subst.info, - tooltip: "(" & $subst.anchorType & ")") + externFilename: getExternFilename(subst), + isTitle: isDocumentationTitle(refname), + tooltip: "(" & $anchorType & ")") # find anchors automatically generated from Nim symbols - if roNimFile in s.options: + if roNimFile in s.options or s.nimFileImported: let substNim = findMainAnchorNim(s, signature=alias, n.info) for subst in substNim: - foundLinks.add LinkDef(ar: arNim, priority: subst.priority, - target: newLeaf(subst.refname), + let fullRefname = + if subst.external: + s.idxImports[getFilename(s, subst)].linkRelPath & + "/" & subst.refname + else: subst.refname + foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority, + target: newLeaf(fullRefname), + externFilename: getExternFilename(subst), + isTitle: isDocumentationTitle(subst.refname), info: subst.info, tooltip: subst.tooltip) foundLinks.sort(cmp = cmp, order = Descending) let aliasStr = addNodes(alias) if foundLinks.len >= 1: - let kind = if foundLinks[0].ar == arHyperlink: rnHyperlink - elif foundLinks[0].ar == arNim: rnNimdocRef + if foundLinks[0].externFilename != "": + s.idxImports[foundLinks[0].externFilename].used = true + let kind = if foundLinks[0].ar in {arHyperlink, arExternalRst}: rnHyperlink + elif foundLinks[0].ar == arNim: + if foundLinks[0].externFilename == "": rnNimdocRef + else: rnHyperlink else: rnInternalRef result = newRstNode(kind) - result.sons = @[newRstNode(rnInner, desc.sons), foundLinks[0].target] + let documentName = # filename without ext for `.nim`, title for `.md` + if foundLinks[0].ar == arNim: + changeFileExt(foundLinks[0].externFilename.extractFilename, "") + elif foundLinks[0].externFilename != "": + s.idxImports[foundLinks[0].externFilename].title + else: foundLinks[0].externFilename.extractFilename + let linkText = + if foundLinks[0].externFilename != "": + if foundLinks[0].isTitle: newLeaf(addNodes(desc)) + else: newLeaf(documentName & ": " & addNodes(desc)) + else: + newRstNode(rnInner, desc.sons) + result.sons = @[linkText, foundLinks[0].target] if kind == rnNimdocRef: result.tooltip = foundLinks[0].tooltip if foundLinks.len > 1: # report ambiguous link var targets = newSeq[string]() @@ -3568,20 +3769,28 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = inc i result.sons = newSons +proc completePass2*(s: PRstSharedState) = + for (filename, importdocInfo) in s.idxImports.pairs: + if not importdocInfo.used: + rstMessage(s.filenames, s.msgHandler, importdocInfo.fromInfo, + mwUnusedImportdoc, filename) + proc rstParse*(text, filename: string, line, column: int, options: RstParseOptions, findFile: FindFileHandler = nil, + findRefFile: FindRefFileHandler = nil, msgHandler: MsgHandler = nil): tuple[node: PRstNode, filenames: RstFileTable, hasToc: bool] = ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`, ## note that 2nd tuple element should be fed to `initRstGenerator` ## argument `filenames` (it is being filled here at least with `filename` ## and possibly with other files from RST ``.. include::`` statement). - var sharedState = newRstSharedState(options, filename, findFile, + var sharedState = newRstSharedState(options, filename, findFile, findRefFile, msgHandler, hasToc=false) let unresolved = rstParsePass1(text, line, column, sharedState) preparePass2(sharedState, unresolved) result.node = resolveSubs(sharedState, unresolved) + completePass2(sharedState) result.filenames = sharedState.filenames result.hasToc = sharedState.hasToc |