#
#
# The Nim Compiler
# (c) Copyright 2012 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
# This is the documentation generator. Cross-references are generated
# by knowing how the anchors are going to be named.
import
ast, strutils, strtabs, algorithm, sequtils, options, msgs, os, idents,
wordrecg, syntaxes, renderer, lexer,
packages/docutils/[rst, rstgen, dochelpers],
json, xmltree, trees, types,
typesrenderer, astalgo, lineinfos, intsets,
pathutils, tables, nimpaths, renderverbatim, osproc
import packages/docutils/rstast except FileIndex, TLineInfo
from uri import encodeUrl
from std/private/globs import nativeToUnixPath
from nodejs import findNodeJs
const
exportSection = skField
docCmdSkip = "skip"
DocColOffset = "## ".len # assuming that a space was added after ##
type
ItemFragment = object ## A fragment from each item will be eventually
## constructed by converting `rst` fields to strings.
case isRst: bool
of true:
rst: PRstNode
of false: ## contains ready markup e.g. from runnableExamples
str: string
ItemPre = seq[ItemFragment] ## A pre-processed item.
Item = object ## Any item in documentation, e.g. symbol
## entry. Configuration variable ``doc.item``
## is used for its HTML rendering.
descRst: ItemPre ## Description of the item (may contain
## runnableExamples).
substitutions: seq[string] ## Variable names in `doc.item`...
sortName: string ## The string used for sorting in output
info: rstast.TLineInfo ## place where symbol was defined (for messages)
anchor: string ## e.g. HTML anchor
name: string ## short name of the symbol, not unique
## (includes backticks ` if present)
detailedName: string ## longer name like `proc search(x: int): int`
ModSection = object ## Section like Procs, Types, etc.
secItems: Table[string, seq[Item]]
## Map basic name -> pre-processed items.
finalMarkup: string ## The items, after RST pass 2 and rendering.
ModSections = array[TSymKind, ModSection]
TocItem = object ## HTML TOC item
content: string
sortName: string
TocSectionsFinal = array[TSymKind, string]
ExampleGroup = ref object
## a group of runnableExamples with same rdoccmd
rdoccmd: string ## from 1st arg in `runnableExamples(rdoccmd): body`
docCmd: string ## from user config, e.g. --doccmd:-d:foo
code: string ## contains imports; each import contains `body`
index: int ## group index
JsonItem = object # pre-processed item: `rst` should be finalized
json: JsonNode
rst: PRstNode
rstField: string
TDocumentor = object of rstgen.RstGenerator
modDescPre: ItemPre # module description, not finalized
modDescFinal: string # module description, after RST pass 2 and rendering
module: PSym
modDeprecationMsg: string
section: ModSections # entries of ``.nim`` file (for `proc`s, etc)
tocSimple: array[TSymKind, seq[TocItem]]
# TOC entries for non-overloadable symbols (e.g. types, constants)...
tocTable: array[TSymKind, Table[string, seq[TocItem]]]
# ...otherwise (e.g. procs)
toc2: TocSectionsFinal # TOC `content`, which is probably wrapped
# in `doc.section.toc2`
toc: TocSectionsFinal # final TOC (wrapped in `doc.section.toc`)
indexValFilename: string
analytics: string # Google Analytics javascript, "" if doesn't exist
seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML.
jEntriesPre: seq[JsonItem] # pre-processed RST + JSON content
jEntriesFinal: JsonNode # final JSON after RST pass 2 and rendering
types: TStrTable
sharedState: PRstSharedState
isPureRst: bool
conf*: ConfigRef
cache*: IdentCache
exampleCounter: int
emitted: IntSet # we need to track which symbols have been emitted
# already. See bug #3655
thisDir*: AbsoluteDir
exampleGroups: OrderedTable[string, ExampleGroup]
wroteSupportFiles*: bool
nimToRstFid: Table[lineinfos.FileIndex, rstast.FileIndex]
## map Nim FileIndex -> RST one, it's needed because we keep them separate
PDoc* = ref TDocumentor ## Alias to type less.
proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst)
proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str)
proc addRstFileIndex(d: PDoc, info: lineinfos.TLineInfo): rstast.FileIndex =
let invalid = rstast.FileIndex(-1)
result = d.nimToRstFid.getOrDefault(info.fileIndex, default = invalid)
if result == invalid:
let fname = toFullPath(d.conf, info)
result = addFilename(d.sharedState, fname)
d.nimToRstFid[info.fileIndex] = result
proc cmpDecimalsIgnoreCase(a, b: string): int =
## For sorting with correct handling of cases like 'uint8' and 'uint16'.
## Also handles leading zeros well (however note that leading zeros are
## significant when lengths of numbers mismatch, e.g. 'bar08' > 'bar8' !).
runnableExamples:
doAssert cmpDecimalsIgnoreCase("uint8", "uint16") < 0
doAssert cmpDecimalsIgnoreCase("val00032", "val16suffix") > 0
doAssert cmpDecimalsIgnoreCase("val16suffix", "val16") > 0
doAssert cmpDecimalsIgnoreCase("val_08_32", "val_08_8") > 0
doAssert cmpDecimalsIgnoreCase("val_07_32", "val_08_8") < 0
doAssert cmpDecimalsIgnoreCase("ab8", "ab08") < 0
doAssert cmpDecimalsIgnoreCase("ab8de", "ab08c") < 0 # sanity check
let aLen = a.len
let bLen = b.len
var
iA = 0
iB = 0
while iA < aLen and iB < bLen:
if isDigit(a[iA]) and isDigit(b[iB]):
var
limitA = iA # index after the last (least significant) digit
limitB = iB
while limitA < aLen and isDigit(a[limitA]): inc limitA
while limitB < bLen and isDigit(b[limitB]): inc limitB
var pos = max(limitA-iA, limitB-iA)
while pos > 0:
if limitA-pos < iA: # digit in `a` is 0 effectively
result = ord('0') - ord(b[limitB-pos])
elif limitB-pos < iB: # digit in `b` is 0 effectively
result = ord(a[limitA-pos]) - ord('0')
else:
result = ord(a[limitA-pos]) - ord(b[limitB-pos])
if result != 0: return
dec pos
result = (limitA - iA) - (limitB - iB) # consider 'bar08' > 'bar8'
if result != 0: return
iA = limitA
iB = limitB
else:
result = ord(toLowerAscii(a[iA])) - ord(toLowerAscii(b[iB]))
if result != 0: return
inc iA
inc iB
result = (aLen - iA) - (bLen - iB)
proc prettyString(a: object): string =
# xxx pending std/prettyprint refs https://github.com/nim-lang/RFCs/issues/203#issuecomment-602534906
for k, v in fieldPairs(a):
result.add k & ": " & $v & "\n"
proc presentationPath*(conf: ConfigRef, file: AbsoluteFile): RelativeFile =
## returns a relative file that will be appended to outDir
let file2 = $file
template bail() =
result = relativeTo(file, conf.projectPath)
proc nimbleDir(): AbsoluteDir =
getNimbleFile(conf, file2).parentDir.AbsoluteDir
case conf.docRoot:
of docRootDefault:
result = getRelativePathFromConfigPath(conf, file)
let dir = nimbleDir()
if not dir.isEmpty:
let result2 = relativeTo(file, dir)
if not result2.isEmpty and (result.isEmpty or result2.string.len < result.string.len):
result = result2
if result.isEmpty: bail()
of "@pkg":
let dir = nimbleDir()
if dir.isEmpty: bail()
else: result = relativeTo(file, dir)
of "@path":
result = getRelativePathFromConfigPath(conf, file)
if result.isEmpty: bail()
elif conf.docRoot.len > 0:
# we're (currently) requiring `isAbsolute` to avoid confusion when passing
# a relative path (would it be relative with regard to $PWD or to projectfile)
conf.globalAssert conf.docRoot.isAbsolute, arg=conf.docRoot
conf.globalAssert conf.docRoot.dirExists, arg=conf.docRoot
# needed because `canonicalizePath` called on `file`
result = file.relativeTo conf.docRoot.expandFilename.AbsoluteDir
else:
bail()
if isAbsolute(result.string):
result = file.string.splitPath()[1].RelativeFile
result = result.string.replace("..", dotdotMangle).RelativeFile
doAssert not result.isEmpty
doAssert not isAbsolute(result.string)
proc whichType(d: PDoc; n: PNode): PSym =
if n.kind == nkSym:
if d.types.strTableContains(n.sym):
result = n.sym
else:
for i in 0..<n.safeLen:
let x = whichType(d, n[i])
if x != nil: return x
proc attachToType(d: PDoc; p: PSym): PSym =
let params = p.ast[paramsPos]
template check(i) =
result = whichType(d, params[i])
if result != nil: return result
# first check the first parameter, then the return type,
# then the other parameter:
if params.len > 1: check(1)
if params.len > 0: check(0)
for i in 2..<params.len: check(i)
template declareClosures =
proc compilerMsgHandler(filename: string, line, col: int,
msgKind: rst.MsgKind, arg: string) {.gcsafe.} =
# translate msg kind:
var k: TMsgKind
case msgKind
of meCannotOpenFile: k = errCannotOpenFile
of meExpected: k = errXExpected
of meGridTableNotImplemented: k = errRstGridTableNotImplemented
of meMarkdownIllformedTable: k = errRstMarkdownIllformedTable
of meNewSectionExpected: k = errRstNewSectionExpected
of meGeneralParseError: k = errRstGeneralParseError
of meInvalidDirective: k = errRstInvalidDirectiveX
of meInvalidField: k = errRstInvalidField
of meFootnoteMismatch: k = errRstFootnoteMismatch
of meSandboxedDirective: k = errRstSandboxedDirective
of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel
of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX
of mwAmbiguousLink: k = warnRstAmbiguousLink
of mwBrokenLink: k = warnRstBrokenLink
of mwUnsupportedLanguage: k = warnRstLanguageXNotSupported
of mwUnsupportedField: k = warnRstFieldXNotSupported
of mwRstStyle: k = warnRstStyle
{.gcsafe.}:
globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg)
proc docgenFindFile(s: string): string {.gcsafe.} =
result = options.findFile(conf, s).string
if result.len == 0:
result = getCurrentDir() / s
if not fileExists(result): result = ""
proc parseRst(text: string,
line, column: int,
conf: ConfigRef, sharedState: PRstSharedState): PRstNode =
declareClosures()
result = rstParsePass1(text, line, column, sharedState)
proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
ext: string, guessTarget: bool): AbsoluteFile =
if optWholeProject in conf.globalOptions or guessTarget:
let d = conf.outDir
createDir(d)
result = d / changeFileExt(filename, ext)
elif not conf.outFile.isEmpty:
result = absOutFile(conf)
else:
result = getOutFile(conf, filename, ext)
proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex}
proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
outExt: string = HtmlExt, module: PSym = nil,
isPureRst = false): PDoc =
declareClosures()
new(result)
result.module = module
result.conf = conf
result.cache = cache
result.outDir = conf.outDir.string
result.isPureRst = isPureRst
var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown, roSandboxDisabled}
if not isPureRst: options.incl roNimFile
result.sharedState = newRstSharedState(
options, filename.string,
docgenFindFile, compilerMsgHandler)
initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
conf.configVars, filename.string,
docgenFindFile, compilerMsgHandler)
if conf.configVars.hasKey("doc.googleAnalytics"):
result.analytics = """
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
ga('create', '$1', 'auto');
ga('send', 'pageview');
</script>
""" % [conf.configVars.getOrDefault"doc.googleAnalytics"]
else:
result.analytics = ""
result.seenSymbols = newStringTable(modeCaseInsensitive)
result.id = 100
result.jEntriesFinal = newJArray()
initStrTable result.types
result.onTestSnippet =
proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) =
if conf.docCmd == docCmdSkip: return
inc(gen.id)
var d = (ptr TDocumentor)(addr gen)
var outp: AbsoluteFile
if filename.len == 0:
let nameOnly = splitFile(d.filename).name
# "snippets" needed, refs bug #17183
outp = getNimcacheDir(conf) / "snippets".RelativeDir / RelativeDir(nameOnly) /
RelativeFile(nameOnly & "_snippet_" & $d.id & ".nim")
elif isAbsolute(filename):
outp = AbsoluteFile(filename)
else:
# Nim's convention: every path is relative to the file it was written in:
let nameOnly = splitFile(d.filename).name
outp = AbsoluteDir(nameOnly) / RelativeFile(filename)
# Make sure the destination directory exists
createDir(outp.splitFile.dir)
# Include the current file if we're parsing a nim file
let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")]
writeFile(outp, importStmt & content)
proc interpSnippetCmd(cmd: string): string =
# backward compatibility hacks; interpolation commands should explicitly use `$`
if cmd.startsWith "nim ": result = "$nim " & cmd[4..^1]
else: result = cmd
# factor with D20210224T221756
result = result.replace("$1", "$options") % [
"nim", os.getAppFilename().quoteShell,
"libpath", quoteShell(d.conf.libpath),
"docCmd", d.conf.docCmd,
"backend", $d.conf.backend,
"options", outp.quoteShell,
# xxx `quoteShell` seems buggy if user passes options = "-d:foo somefile.nim"
]
let cmd = cmd.interpSnippetCmd
rawMessage(conf, hintExecuting, cmd)
let (output, gotten) = execCmdEx(cmd)
if gotten != status:
rawMessage(conf, errGenerated, "snippet failed: cmd: '$1' status: $2 expected: $3 output: $4" % [cmd, $gotten, $status, output])
result.emitted = initIntSet()
result.destFile = getOutFile2(conf, presentationPath(conf, filename), outExt, false).string
result.thisDir = result.destFile.AbsoluteFile.splitFile.dir
template dispA(conf: ConfigRef; dest: var string, xml, tex: string,
args: openArray[string]) =
if not conf.isLatexCmd: dest.addf(xml, args)
else: dest.addf(tex, args)
proc getVarIdx(varnames: openArray[string], id: string): int =
for i in 0..high(varnames):
if cmpIgnoreStyle(varnames[i], id) == 0:
return i
result = -1
proc genComment(d: PDoc, n: PNode): PRstNode =
if n.comment.len > 0:
d.sharedState.currFileIdx = addRstFileIndex(d, n.info)
result = parseRst(n.comment,
toLinenumber(n.info),
toColumn(n.info) + DocColOffset,
d.conf, d.sharedState)
proc genRecCommentAux(d: PDoc, n: PNode): PRstNode =
if n == nil: return nil
result = genComment(d, n)
if result == nil:
if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef,
nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkHiddenStdConv}:
# notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}:
for i in 0..<n.len:
result = genRecCommentAux(d, n[i])
if result != nil: return
else:
n.comment = ""
proc genRecComment(d: PDoc, n: PNode): PRstNode =
if n == nil: return nil
result = genComment(d, n)
if result == nil:
if n.kind in {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef,
nkMacroDef, nkTemplateDef, nkConverterDef}:
result = genRecCommentAux(d, n[bodyPos])
else:
result = genRecCommentAux(d, n)
proc getPlainDocstring(n: PNode): string =
## Gets the plain text docstring of a node non destructively.
##
## You need to call this before genRecComment, whose side effects are removal
## of comments from the tree. The proc will recursively scan and return all
## the concatenated ``##`` comments of the node.
if n == nil: result = ""
elif startsWith(n.comment, "##"):
result = n.comment
else:
for i in 0..<n.safeLen:
result = getPlainDocstring(n[i])
if result.len > 0: return
proc belongsToPackage(conf: ConfigRef; module: PSym): bool =
result = module.kind == skModule and module.getnimblePkgId == conf.mainPackageId
proc externalDep(d: PDoc; module: PSym): string =
if optWholeProject in d.conf.globalOptions or d.conf.docRoot.len > 0:
let full = AbsoluteFile toFullPath(d.conf, FileIndex module.position)
let tmp = getOutFile2(d.conf, presentationPath(d.conf, full), HtmlExt, sfMainModule notin module.flags)
result = relativeTo(tmp, d.thisDir, '/').string
else:
result = extractFilename toFullPath(d.conf, FileIndex module.position)
proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var string;
renderFlags: TRenderFlags = {};
procLink: string) =
var r: TSrcGen
var literal = ""
initTokRender(r, n, renderFlags)
var kind = tkEof
var tokenPos = 0
var procTokenPos = 0
template escLit(): untyped = esc(d.target, literal)
while true:
getNextTok(r, kind, literal)
inc tokenPos
case kind
of tkEof:
break
of tkComment:
dispA(d.conf, result, "<span class=\"Comment\">$1</span>", "\\spanComment{$1}",
[escLit])
of tokKeywordLow..tokKeywordHigh:
if kind in {tkProc, tkMethod, tkIterator, tkMacro, tkTemplate, tkFunc, tkConverter}:
procTokenPos = tokenPos
dispA(d.conf, result, "<span class=\"Keyword\">$1</span>", "\\spanKeyword{$1}",
[literal])
of tkOpr:
dispA(d.conf, result, "<span class=\"Operator\">$1</span>", "\\spanOperator{$1}",
[escLit])
of tkStrLit..tkTripleStrLit, tkCustomLit:
dispA(d.conf, result, "<span class=\"StringLit\">$1</span>",
"\\spanStringLit{$1}", [escLit])
of tkCharLit:
dispA(d.conf, result, "<span class=\"CharLit\">$1</span>", "\\spanCharLit{$1}",
[escLit])
of tkIntLit..tkUInt64Lit:
dispA(d.conf, result, "<span class=\"DecNumber\">$1</span>",
"\\spanDecNumber{$1}", [escLit])
of tkFloatLit..tkFloat128Lit:
dispA(d.conf, result, "<span class=\"FloatNumber\">$1</span>",
"\\spanFloatNumber{$1}", [escLit])
of tkSymbol:
let s = getTokSym(r)
# -2 because of the whitespace in between:
if procTokenPos == tokenPos-2 and procLink != "":
dispA(d.conf, result, "<a href=\"#$2\"><span class=\"Identifier\">$1</span></a>",
"\\spanIdentifier{$1}", [escLit, procLink])
elif s != nil and s.kind in {skType, skVar, skLet, skConst} and
sfExported in s.flags and s.owner != nil and
belongsToPackage(d.conf, s.owner) and d.target == outHtml:
let external = externalDep(d, s.owner)
result.addf "<a href=\"$1#$2\"><span class=\"Identifier\">$3</span></a>",
[changeFileExt(external, "html"), literal,
escLit]
else:
dispA(d.conf, result, "<span class=\"Identifier\">$1</span>",
"\\spanIdentifier{$1}", [escLit])
of tkSpaces, tkInvalid:
result.add(literal)
of tkHideableStart:
template fun(s) = dispA(d.conf, result, s, "\\spanOther{$1}", [escLit])
if renderRunnableExamples in renderFlags: fun "$1"
else:
# 1st span is required for the JS to work properly
fun """
<span>
<span class="Other pragmadots">...</span>
</span>
<span class="pragmawrap">""".replace("\n", "") # Must remove newlines because wrapped in a <pre>
of tkHideableEnd:
template fun(s) = dispA(d.conf, result, s, "\\spanOther{$1}", [escLit])
if renderRunnableExamples in renderFlags: fun "$1"
else: fun "</span>"
of tkCurlyDotLe: dispA(d.conf, result, "$1", "\\spanOther{$1}", [escLit])
of tkCurlyDotRi: dispA(d.conf, result, "$1", "\\spanOther{$1}", [escLit])
of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi,
tkBracketDotLe, tkBracketDotRi, tkParDotLe,
tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot,
tkAccent, tkColonColon,
tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr,
tkBracketLeColon:
dispA(d.conf, result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}",
[escLit])
proc exampleOutputDir(d: PDoc): AbsoluteDir = d.conf.getNimcacheDir / RelativeDir"runnableExamples"
proc runAllExamples(d: PDoc) =
# This used to be: `let backend = if isDefined(d.conf, "js"): "js"` (etc), however
# using `-d:js` (etc) cannot work properly, e.g. would fail with `importjs`
# since semantics are affected by `config.backend`, not by isDefined(d.conf, "js")
let outputDir = d.exampleOutputDir
for _, group in d.exampleGroups:
if group.docCmd == docCmdSkip: continue
let outp = outputDir / RelativeFile("$1_group$2_examples.nim" % [d.filename.splitFile.name, $group.index])
group.code = "# autogenerated by docgen\n# source: $1\n# rdoccmd: $2\n$3" % [d.filename, group.rdoccmd, group.code]
writeFile(outp, group.code)
# most useful semantics is that `docCmd` comes after `rdoccmd`, so that we can (temporarily) override
# via command line
# D20210224T221756:here
let cmd = "$nim $backend -r --lib:$libpath --warning:UnusedImport:off --path:$path --nimcache:$nimcache $rdoccmd $docCmd $file" % [
"nim", quoteShell(os.getAppFilename()),
"backend", $d.conf.backend,
"path", quoteShell(d.conf.projectPath),
"libpath", quoteShell(d.conf.libpath),
"nimcache", quoteShell(outputDir),
"file", quoteShell(outp),
"rdoccmd", group.rdoccmd,
"docCmd", group.docCmd,
]
if d.conf.backend == backendJs and findNodeJs() == "":
discard "ignore JS runnableExample"
elif os.execShellCmd(cmd) != 0:
d.conf.quitOrRaise "[runnableExamples] failed: generated file: '$1' group: '$2' cmd: $3" % [outp.string, group[].prettyString, cmd]
else:
# keep generated source file `outp` to allow inspection.
rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
# removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove
proc quoted(a: string): string = result.addQuoted(a)
proc toInstantiationInfo(conf: ConfigRef, info: TLineInfo): auto =
# xxx expose in compiler/lineinfos.nim
(conf.toMsgFilename(info), info.line.int, info.col.int + ColOffset)
proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, code: string] =
## returns `rdoccmd` and source code for this runnableExamples
var rdoccmd = ""
if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid")
if n.len == 3:
let n1 = n[1]
# xxx this should be evaluated during sempass
if n1.kind notin nkStrKinds: globalError(d.conf, n1.info, "string litteral expected")
rdoccmd = n1.strVal
let useRenderModule = false
let loc = d.conf.toFileLineCol(n.info)
let code = extractRunnableExamplesSource(d.conf, n)
let codeIndent = extractRunnableExamplesSource(d.conf, n, indent = 2)
if d.conf.errorCounter > 0:
return (rdoccmd, code)
let comment = "autogenerated by docgen\nloc: $1\nrdoccmd: $2" % [loc, rdoccmd]
let outputDir = d.exampleOutputDir
createDir(outputDir)
inc d.exampleCounter
let outp = outputDir / RelativeFile("$#_examples_$#.nim" % [d.filename.extractFilename.changeFileExt"", $d.exampleCounter])
if useRenderModule:
var docComment = newTree(nkCommentStmt)
docComment.comment = comment
var runnableExamples = newTree(nkStmtList,
docComment,
newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
runnableExamples.info = n.info
for a in n.lastSon: runnableExamples.add a
# buggy, refs bug #17292
# still worth fixing as it can affect other code relying on `renderModule`,
# so we keep this code path here for now, which could still be useful in some
# other situations.
renderModule(runnableExamples, outp.string, conf = d.conf)
else:
var code2 = code
if code.len > 0 and "codeReordering" notin code:
# hacky but simplest solution, until we devise a way to make `{.line.}`
# work without introducing a scope
code2 = """
{.line: $#.}:
$#
""" % [$toInstantiationInfo(d.conf, n.info), codeIndent]
code2 = """
#[
$#
]#
import $#
$#
""" % [comment, d.filename.quoted, code2]
writeFile(outp.string, code2)
if rdoccmd notin d.exampleGroups:
d.exampleGroups[rdoccmd] = ExampleGroup(rdoccmd: rdoccmd, docCmd: d.conf.docCmd, index: d.exampleGroups.len)
d.exampleGroups[rdoccmd].code.add "import $1\n" % outp.string.quoted
var codeShown: string
if topLevel: # refs https://github.com/nim-lang/RFCs/issues/352
let title = canonicalImport(d.conf, AbsoluteFile d.filename)
codeShown = "import $#\n$#" % [title, code]
else:
codeShown = code
result = (rdoccmd, codeShown)
when false:
proc extractImports(n: PNode; result: PNode) =
if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
result.add copyTree(n)
n.kind = nkEmpty
return
for i in 0..<n.safeLen: extractImports(n[i], result)
let imports = newTree(nkStmtList)
var savedLastSon = copyTree n.lastSon
extractImports(savedLastSon, imports)
for imp in imports: runnableExamples.add imp
runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon)
type RunnableState = enum
rsStart
rsComment
rsRunnable
rsDone
proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var ItemPre,
state: RunnableState, topLevel: bool):
RunnableState =
##[
Simple state machine to tell whether we render runnableExamples and doc comments.
This is to ensure that we can interleave runnableExamples and doc comments freely;
the logic is easy to change but currently a doc comment following another doc comment
will not render, to avoid rendering in following case:
proc fn* =
runnableExamples: discard
## d1
runnableExamples: discard
## d2
## internal explanation # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
discard # some code
]##
case n.kind
of nkCommentStmt:
if state in {rsStart, rsRunnable}:
dest.add genRecComment(d, n)
return rsComment
of nkCallKinds:
if isRunnableExamples(n[0]) and
n.len >= 2 and n.lastSon.kind == nkStmtList:
if state in {rsStart, rsComment, rsRunnable}:
let (rdoccmd, code) = prepareExample(d, n, topLevel)
var msg = "Example:"
if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
var s: string
dispA(d.conf, s, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
"\n\n\\textbf{$1}\n", [msg])
dest.add s
inc d.listingCounter
let id = $d.listingCounter
dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
var dest2 = ""
renderNimCode(dest2, code, d.target)
dest.add dest2
dest.add(d.config.getOrDefault"doc.listing_end" % id)
return rsRunnable
else:
localError(d.conf, n.info, errUser, "runnableExamples must appear before the first non-comment statement")
else: discard
return rsDone
# change this to `rsStart` if you want to keep generating doc comments
# and runnableExamples that occur after some code in routine
proc getRoutineBody(n: PNode): PNode =
##[
nim transforms these quite differently:
proc someType*(): int =
## foo
result = 3
=>
result =
## foo
3;
proc someType*(): int =
## foo
3
=>
## foo
result = 3;
so we normalize the results to get to the statement list containing the
(0 or more) doc comments and runnableExamples.
]##
result = n[bodyPos]
# This won't be transformed: result.id = 10. Namely result[0].kind != nkSym.
if result.kind == nkAsgn and result[0].kind == nkSym and
n.len > bodyPos+1 and n[bodyPos+1].kind == nkSym:
doAssert result.len == 2
result = result[1]
proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var ItemPre) =
var n = n
var state = rsStart
template fn(n2, topLevel) =
state = getAllRunnableExamplesImpl(d, n2, dest, state, topLevel)
dest.add genComment(d, n)
case n.kind
of routineDefs:
n = n.getRoutineBody
case n.kind
of nkCommentStmt, nkCallKinds: fn(n, topLevel = false)
else:
for i in 0..<n.safeLen:
fn(n[i], topLevel = false)
if state == rsDone: discard # check all sons
else: fn(n, topLevel = true)
proc isVisible(d: PDoc; n: PNode): bool =
result = false
if n.kind == nkPostfix:
if n.len == 2 and n[0].kind == nkIdent:
var v = n[0].ident
result = v.id == ord(wStar) or v.id == ord(wMinus)
elif n.kind == nkSym:
# we cannot generate code for forwarded symbols here as we have no
# exception tracking information here. Instead we copy over the comment
# from the proc header.
if optDocInternal in d.conf.globalOptions:
result = {sfFromGeneric, sfForward}*n.sym.flags == {}
else:
result = {sfExported, sfFromGeneric, sfForward}*n.sym.flags == {sfExported}
if result and containsOrIncl(d.emitted, n.sym.id):
result = false
elif n.kind == nkPragmaExpr:
result = isVisible(d, n[0])
proc getName(d: PDoc, n: PNode, splitAfter = -1): string =
case n.kind
of nkPostfix: result = getName(d, n[1], splitAfter)
of nkPragmaExpr: result = getName(d, n[0], splitAfter)
of nkSym: result = esc(d.target, n.sym.renderDefinitionName, splitAfter)
of nkIdent: result = esc(d.target, n.ident.s, splitAfter)
of nkAccQuoted:
result = esc(d.target, "`")
for i in 0..<n.len: result.add(getName(d, n[i], splitAfter))
result.add esc(d.target, "`")
of nkOpenSymChoice, nkClosedSymChoice:
result = getName(d, n[0], splitAfter)
else:
result = ""
proc getNameIdent(cache: IdentCache; n: PNode): PIdent =
case n.kind
of nkPostfix: result = getNameIdent(cache, n[1])
of nkPragmaExpr: result = getNameIdent(cache, n[0])
of nkSym: result = n.sym.name
of nkIdent: result = n.ident
of nkAccQuoted:
var r = ""
for i in 0..<n.len: r.add(getNameIdent(cache, n[i]).s)
result = getIdent(cache, r)
of nkOpenSymChoice, nkClosedSymChoice:
result = getNameIdent(cache, n[0])
else:
result = nil
proc getRstName(n: PNode): PRstNode =
case n.kind
of nkPostfix: result = getRstName(n[1])
of nkPragmaExpr: result = getRstName(n[0])
of nkSym: result = newRstLeaf(n.sym.renderDefinitionName)
of nkIdent: result = newRstLeaf(n.ident.s)
of nkAccQuoted:
result = getRstName(n[0])
for i in 1..<n.len: result.text.add(getRstName(n[i]).text)
of nkOpenSymChoice, nkClosedSymChoice:
result = getRstName(n[0])
else:
result = nil
proc newUniquePlainSymbol(d: PDoc, original: string): string =
## Returns a new unique plain symbol made up from the original.
##
## When a collision is found in the seenSymbols table, new numerical variants
## with underscore + number will be generated.
if not d.seenSymbols.hasKey(original):
result = original
d.seenSymbols[original] = ""
return
# Iterate over possible numeric variants of the original name.
var count = 2
while true:
result = original & "_" & $count
if not d.seenSymbols.hasKey(result):
d.seenSymbols[result] = ""
break
count += 1
proc complexName(k: TSymKind, n: PNode, baseName: string): string =
## Builds a complex unique href name for the node.
##
## Pass as ``baseName`` the plain symbol obtained from the nodeName. The
## format of the returned symbol will be ``baseName(.callable type)?,(param
## type)?(,param type)*``. The callable type part will be added only if the
## node is not a proc, as those are the common ones. The suffix will be a dot
## and a single letter representing the type of the callable. The parameter
## types will be added with a preceding dash. Return types won't be added.
##
## If you modify the output of this proc, please update the anchor generation
## section of ``doc/docgen.rst``.
result = baseName
case k
of skProc, skFunc: discard
of skMacro: result.add(".m")
of skMethod: result.add(".e")
of skIterator: result.add(".i")
of skTemplate: result.add(".t")
of skConverter: result.add(".c")
else: discard
if n.safeLen > paramsPos and n[paramsPos].kind == nkFormalParams:
let params = renderParamTypes(n[paramsPos])
if params.len > 0:
result.add(defaultParamSeparator)
result.add(params)
proc docstringSummary(rstText: string): string =
## Returns just the first line or a brief chunk of text from a rst string.
##
## Most docstrings will contain a one liner summary, so stripping at the
## first newline is usually fine. If after that the content is still too big,
## it is stripped at the first comma, colon or dot, usual English sentence
## separators.
##
## No guarantees are made on the size of the output, but it should be small.
## Also, we hope to not break the rst, but maybe we do. If there is any
## trimming done, an ellipsis unicode char is added.
const maxDocstringChars = 100
assert(rstText.len < 2 or (rstText[0] == '#' and rstText[1] == '#'))
result = rstText.substr(2).strip
var pos = result.find('\L')
if pos > 0:
result.setLen(pos - 1)
result.add("…")
if pos < maxDocstringChars:
return
# Try to keep trimming at other natural boundaries.
pos = result.find({'.', ',', ':'})
let last = result.len - 1
if pos > 0 and pos < last:
result.setLen(pos - 1)
result.add("…")
proc genDeprecationMsg(d: PDoc, n: PNode): string =
## Given a nkPragma wDeprecated node output a well-formatted section
if n == nil: return
case n.safeLen:
of 0: # Deprecated w/o any message
result = getConfigVar(d.conf, "doc.deprecationmsg") % [
"label" , "Deprecated", "message", ""]
of 2: # Deprecated w/ a message
if n[1].kind in {nkStrLit..nkTripleStrLit}:
result = getConfigVar(d.conf, "doc.deprecationmsg") % [
"label", "Deprecated:", "message", xmltree.escape(n[1].strVal)]
else:
doAssert false
type DocFlags = enum
kDefault
kForceExport
proc genSeeSrc(d: PDoc, path: string, line: int): string =
let docItemSeeSrc = getConfigVar(d.conf, "doc.item.seesrc")
if docItemSeeSrc.len > 0:
let path = relativeTo(AbsoluteFile path, AbsoluteDir getCurrentDir(), '/')
when false:
let cwd = canonicalizePath(d.conf, getCurrentDir())
var path = path
if path.startsWith(cwd):
path = path[cwd.len+1..^1].replace('\\', '/')
let gitUrl = getConfigVar(d.conf, "git.url")
if gitUrl.len > 0:
let defaultBranch =
if NimPatch mod 2 == 1: "devel"
else: "version-$1-$2" % [$NimMajor, $NimMinor]
let commit = getConfigVar(d.conf, "git.commit", defaultBranch)
let develBranch = getConfigVar(d.conf, "git.devel", "devel")
dispA(d.conf, result, "$1", "", [docItemSeeSrc % [
"path", path.string, "line", $line, "url", gitUrl,
"commit", commit, "devel", develBranch]])
proc symbolPriority(k: TSymKind): int =
result = case k
of skMacro: -3
of skTemplate: -2
of skIterator: -1
else: 0 # including skProc which have higher priority
# documentation itself has even higher priority 1
proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol =
## Converts symbol info (names/types/parameters) in `n` into format
## `LangSymbol` convenient for ``rst.nim``/``dochelpers.nim``.
result.name = baseName.nimIdentNormalize
result.symKind = k.toHumanStr
if k in routineKinds:
var
paramTypes: seq[string]
renderParamTypes(paramTypes, n[paramsPos], toNormalize=true)
let paramNames = renderParamNames(n[paramsPos], toNormalize=true)
# In some rare cases (system.typeof) parameter type is not set for default:
doAssert paramTypes.len <= paramNames.len
for i in 0 ..< paramNames.len:
if i < paramTypes.len:
result.parameters.add (paramNames[i], paramTypes[i])
else:
result.parameters.add (paramNames[i], "")
result.parametersProvided = true
result.outType = renderOutType(n[paramsPos], toNormalize=true)
if k in {skProc, skFunc, skType, skIterator}:
# Obtain `result.generics`
# Use `n[miscPos]` since n[genericParamsPos] does not contain constraints
var genNode: PNode = nil
if k == skType:
genNode = n[1] # FIXME: what is index 1?
else:
if n[miscPos].kind != nkEmpty:
genNode = n[miscPos][1] # FIXME: what is index 1?
if genNode != nil:
var literal = ""
var r: TSrcGen
initTokRender(r, genNode, {renderNoBody, renderNoComments,
renderNoPragmas, renderNoProcDefs})
var kind = tkEof
while true:
getNextTok(r, kind, literal)
if kind == tkEof:
break
if kind != tkSpaces:
result.generics.add(literal.nimIdentNormalize)
if k == skType:
case n[2].kind
of nkEnumTy: result.symTypeKind = "enum"
of nkObjectTy: result.symTypeKind = "object"
of nkTupleTy: result.symTypeKind = "tuple"
else: discard
proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
if (docFlags != kForceExport) and not isVisible(d, nameNode): return
let
name = getName(d, nameNode)
var plainDocstring = getPlainDocstring(n) # call here before genRecComment!
var result = ""
var literal, plainName = ""
var kind = tkEof
var comm: ItemPre
if n.kind in routineDefs:
getAllRunnableExamples(d, n, comm)
else:
comm.add genRecComment(d, n)
var r: TSrcGen
# Obtain the plain rendered string for hyperlink titles.
initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
renderNoPragmas, renderNoProcDefs})
while true:
getNextTok(r, kind, literal)
if kind == tkEof:
break
plainName.add(literal)
var pragmaNode = getDeclPragma(n)
if pragmaNode != nil: pragmaNode = findPragma(pragmaNode, wDeprecated)
inc(d.id)
let
plainNameEsc = esc(d.target, plainName.strip)
detailedName = k.toHumanStr & " " & (
if k in routineKinds: plainName else: name)
uniqueName = if k in routineKinds: plainNameEsc else: name
sortName = if k in routineKinds: plainName.strip else: name
cleanPlainSymbol = renderPlainSymbolName(nameNode)
complexSymbol = complexName(k, n, cleanPlainSymbol)
plainSymbolEnc = encodeUrl(cleanPlainSymbol, usePlus = false)
symbolOrId = d.newUniquePlainSymbol(complexSymbol)
symbolOrIdEnc = encodeUrl(symbolOrId, usePlus = false)
deprecationMsg = genDeprecationMsg(d, pragmaNode)
rstLangSymbol = toLangSymbol(k, n, cleanPlainSymbol)
# we generate anchors automatically for subsequent use in doc comments
let lineinfo = rstast.TLineInfo(
line: nameNode.info.line, col: nameNode.info.col,
fileIndex: addRstFileIndex(d, nameNode.info))
addAnchorNim(d.sharedState, refn = symbolOrId, tooltip = detailedName,
rstLangSymbol, priority = symbolPriority(k), info = lineinfo)
nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,
renderDocComments, renderSyms}, symbolOrIdEnc)
let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
d.section[k].secItems.mgetOrPut(cleanPlainSymbol, newSeq[Item]()).add Item(
descRst: comm,
sortName: sortName,
info: lineinfo,
anchor: symbolOrId,
detailedName: detailedName,
name: name,
substitutions: @[
"uniqueName", uniqueName,
"header", result, "itemID", $d.id,
"header_plain", plainNameEsc, "itemSym", cleanPlainSymbol,
"itemSymEnc", plainSymbolEnc,
"itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc,
"deprecationMsg", deprecationMsg])
let external = d.destFile.AbsoluteFile.relativeTo(d.conf.outDir, '/').changeFileExt(HtmlExt).string
var attype = ""
if k in routineKinds and nameNode.kind == nkSym:
let att = attachToType(d, nameNode.sym)
if att != nil:
attype = esc(d.target, att.name.s)
elif k == skType and nameNode.kind == nkSym and nameNode.sym.typ.kind in {tyEnum, tyBool}:
let etyp = nameNode.sym.typ
for e in etyp.n:
if e.sym.kind != skEnumField: continue
let plain = renderPlainSymbolName(e)
let symbolOrId = d.newUniquePlainSymbol(plain)
setIndexTerm(d[], external, symbolOrId, plain, nameNode.sym.name.s & '.' & plain,
xmltree.escape(getPlainDocstring(e).docstringSummary))
d.tocSimple[k].add TocItem(
sortName: sortName,
content: getConfigVar(d.conf, "doc.item.toc") % [
"name", name, "header_plain", plainNameEsc,
"itemSymOrIDEnc", symbolOrIdEnc])
d.tocTable[k].mgetOrPut(cleanPlainSymbol, newSeq[TocItem]()).add TocItem(
sortName: sortName,
content: getConfigVar(d.conf, "doc.item.tocTable") % [
"name", name, "header_plain", plainNameEsc,
"itemSymOrID", symbolOrId.replace(",", ",<wbr>"),
"itemSymOrIDEnc", symbolOrIdEnc])
# Ironically for types the complexSymbol is *cleaner* than the plainName
# because it doesn't include object fields or documentation comments. So we
# use the plain one for callable elements, and the complex for the rest.
var linkTitle = changeFileExt(extractFilename(d.filename), "") & ": "
if n.kind in routineDefs: linkTitle.add(xmltree.escape(plainName.strip))
else: linkTitle.add(xmltree.escape(complexSymbol.strip))
setIndexTerm(d[], external, symbolOrId, name, linkTitle,
xmltree.escape(plainDocstring.docstringSummary))
if k == skType and nameNode.kind == nkSym:
d.types.strTableAdd nameNode.sym
proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem =
if not isVisible(d, nameNode): return
var
name = getName(d, nameNode)
comm = genRecComment(d, n)
r: TSrcGen
initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments})
result.json = %{ "name": %name, "type": %($k), "line": %n.info.line.int,
"col": %n.info.col}
if comm != nil:
result.rst = comm
result.rstField = "description"
if r.buf.len > 0:
result.json["code"] = %r.buf
if k in routineKinds:
result.json["signature"] = newJObject()
if n[paramsPos][0].kind != nkEmpty:
result.json["signature"]["return"] = %($n[paramsPos][0])
if n[paramsPos].len > 1:
result.json["signature"]["arguments"] = newJArray()
for paramIdx in 1 ..< n[paramsPos].len:
for identIdx in 0 ..< n[paramsPos][paramIdx].len - 2:
let
paramName = $n[paramsPos][paramIdx][identIdx]
paramType = $n[paramsPos][paramIdx][^2]
if n[paramsPos][paramIdx][^1].kind != nkEmpty:
let paramDefault = $n[paramsPos][paramIdx][^1]
result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault}
else:
result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType}
if n[pragmasPos].kind != nkEmpty:
result.json["signature"]["pragmas"] = newJArray()
for pragma in n[pragmasPos]:
result.json["signature"]["pragmas"].add %($pragma)
if n[genericParamsPos].kind != nkEmpty:
result.json["signature"]["genericParams"] = newJArray()
for genericParam in n[genericParamsPos]:
var param = %{"name": %($genericParam)}
if genericParam.sym.typ.sons.len > 0:
param["types"] = newJArray()
for kind in genericParam.sym.typ.sons:
param["types"].add %($kind)
result.json["signature"]["genericParams"].add param
proc checkForFalse(n: PNode): bool =
result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0
proc traceDeps(d: PDoc, it: PNode) =
const k = skModule
if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
let sep = it[0]
let dir = it[1]
let a = newNodeI(nkInfix, it.info)
a.add sep
a.add dir
a.add sep # dummy entry, replaced in the loop
for x in it[2]:
a[2] = x
traceDeps(d, a)
elif it.kind == nkSym and belongsToPackage(d.conf, it.sym):
let external = externalDep(d, it.sym)
if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
dispA(d.conf, d.section[k].finalMarkup,
"<a class=\"reference external\" href=\"$2\">$1</a>",
"$1", [esc(d.target, external.prettyLink),
changeFileExt(external, "html")])
proc exportSym(d: PDoc; s: PSym) =
const k = exportSection
if s.kind == skModule and belongsToPackage(d.conf, s):
let external = externalDep(d, s)
if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
dispA(d.conf, d.section[k].finalMarkup,
"<a class=\"reference external\" href=\"$2\">$1</a>",
"$1", [esc(d.target, external.prettyLink),
changeFileExt(external, "html")])
elif s.kind != skModule and s.owner != nil:
let module = originatingModule(s)
if belongsToPackage(d.conf, module):
let
complexSymbol = complexName(s.kind, s.ast, s.name.s)
symbolOrId = d.newUniquePlainSymbol(complexSymbol)
external = externalDep(d, module)
if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
# XXX proper anchor generation here
dispA(d.conf, d.section[k].finalMarkup,
"<a href=\"$2#$3\"><span class=\"Identifier\">$1</span></a>",
"$1", [esc(d.target, s.name.s),
changeFileExt(external, "html"),
symbolOrId])
proc documentNewEffect(cache: IdentCache; n: PNode): PNode =
let s = n[namePos].sym
if tfReturnsNew in s.typ.flags:
result = newIdentNode(getIdent(cache, "new"), n.info)
proc documentEffect(cache: IdentCache; n, x: PNode, effectType: TSpecialWord, idx: int): PNode =
let spec = effectSpec(x, effectType)
if isNil(spec):
let s = n[namePos].sym
let actual = s.typ.n[0]
if actual.len != effectListLen: return
let real = actual[idx]
if real == nil: return
let realLen = real.len
# warning: hack ahead:
var effects = newNodeI(nkBracket, n.info, realLen)
for i in 0..<realLen:
var t = typeToString(real[i].typ)
if t.startsWith("ref "): t = substr(t, 4)
effects[i] = newIdentNode(getIdent(cache, t), n.info)
# set the type so that the following analysis doesn't screw up:
effects[i].typ = real[i].typ
result = newTreeI(nkExprColonExpr, n.info,
newIdentNode(getIdent(cache, $effectType), n.info), effects)
proc documentWriteEffect(cache: IdentCache; n: PNode; flag: TSymFlag; pragmaName: string): PNode =
let s = n[namePos].sym
let params = s.typ.n
var effects = newNodeI(nkBracket, n.info)
for i in 1..<params.len:
if params[i].kind == nkSym and flag in params[i].sym.flags:
effects.add params[i]
if effects.len > 0:
result = newTreeI(nkExprColonExpr, n.info,
newIdentNode(getIdent(cache, pragmaName), n.info), effects)
proc documentRaises*(cache: IdentCache; n: PNode) =
if n[namePos].kind != nkSym: return
let pragmas = n[pragmasPos]
let p1 = documentEffect(cache, n, pragmas, wRaises, exceptionEffects)
let p2 = documentEffect(cache, n, pragmas, wTags, tagEffects)
let p3 = documentWriteEffect(cache, n, sfWrittenTo, "writes")
let p4 = documentNewEffect(cache, n)
let p5 = documentWriteEffect(cache, n, sfEscapes, "escapes")
if p1 != nil or p2 != nil or p3 != nil or p4 != nil or p5 != nil:
if pragmas.kind == nkEmpty:
n[pragmasPos] = newNodeI(nkPragma, n.info)
if p1 != nil: n[pragmasPos].add p1
if p2 != nil: n[pragmasPos].add p2
if p3 != nil: n[pragmasPos].add p3
if p4 != nil: n[pragmasPos].add p4
if p5 != nil: n[pragmasPos].add p5
proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
## Goes through nim nodes recursively and collects doc comments.
## Main function for `doc`:option: command,
## which is implemented in ``docgen2.nim``.
template genItemAux(skind) =
genItem(d, n, n[namePos], skind, docFlags)
case n.kind
of nkPragma:
let pragmaNode = findPragma(n, wDeprecated)
d.modDeprecationMsg.add(genDeprecationMsg(d, pragmaNode))
of nkCommentStmt: d.modDescPre.add(genComment(d, n))
of nkProcDef, nkFuncDef:
when useEffectSystem: documentRaises(d.cache, n)
genItemAux(skProc)
of nkMethodDef:
when useEffectSystem: documentRaises(d.cache, n)
genItemAux(skMethod)
of nkIteratorDef:
when useEffectSystem: documentRaises(d.cache, n)
genItemAux(skIterator)
of nkMacroDef: genItemAux(skMacro)
of nkTemplateDef: genItemAux(skTemplate)
of nkConverterDef:
when useEffectSystem: documentRaises(d.cache, n)
genItemAux(skConverter)
of nkTypeSection, nkVarSection, nkLetSection, nkConstSection:
for i in 0..<n.len:
if n[i].kind != nkCommentStmt:
# order is always 'type var let const':
genItem(d, n[i], n[i][0],
succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags)
of nkStmtList:
for i in 0..<n.len: generateDoc(d, n[i], orig)
of nkWhenStmt:
# generate documentation for the first branch only:
if not checkForFalse(n[0][0]):
generateDoc(d, lastSon(n[0]), orig)
of nkImportStmt:
for it in n: traceDeps(d, it)
of nkExportStmt:
for it in n:
if it.kind == nkSym:
if d.module != nil and d.module == it.sym.owner:
generateDoc(d, it.sym.ast, orig, kForceExport)
elif it.sym.ast != nil:
exportSym(d, it.sym)
of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept"
of nkFromStmt, nkImportExceptStmt: traceDeps(d, n[0])
of nkCallKinds:
var comm: ItemPre
getAllRunnableExamples(d, n, comm)
if comm.len != 0: d.modDescPre.add(comm)
else: discard
proc overloadGroupName(s: string, k: TSymKind): string =
## Turns a name like `f` into anchor `f-procs-all`
#s & " " & k.toHumanStr & "s all"
s & "-" & k.toHumanStr & "s-all"
proc finishGenerateDoc*(d: var PDoc) =
## Perform 2nd RST pass for resolution of links/footnotes/headings...
# copy file map `filenames` to ``rstgen.nim`` for its warnings
d.filenames = d.sharedState.filenames
# Main title/subtitle are allowed only in the first RST fragment of document
var firstRst = PRstNode(nil)
for fragment in d.modDescPre:
if fragment.isRst:
firstRst = fragment.rst
break
preparePass2(d.sharedState, firstRst)
# add anchors to overload groups before RST resolution
for k in TSymKind:
if k in routineKinds:
for plainName, overloadChoices in d.section[k].secItems:
if overloadChoices.len > 1:
let refn = overloadGroupName(plainName, k)
let tooltip = "$1 ($2 overloads)" % [
k.toHumanStr & " " & plainName, $overloadChoices.len]
addAnchorNim(d.sharedState, refn, tooltip,
LangSymbol(symKind: k.toHumanStr,
name: nimIdentBackticksNormalize(plainName),
isGroup: true),
priority = symbolPriority(k),
# select index `0` just to have any meaningful warning:
info = overloadChoices[0].info)
# Finalize fragments of ``.nim`` or ``.rst`` file
proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) =
for f in fragments:
case f.isRst:
of true:
var resolved = resolveSubs(d.sharedState, f.rst)
renderRstToOut(d[], resolved, result)
of false: result &= f.str
proc cmp(x, y: Item): int = cmpDecimalsIgnoreCase(x.sortName, y.sortName)
for k in TSymKind:
# add symbols to section for each `k`, while optionally wrapping
# overloadable items with the same basic name by ``doc.item2``
let overloadableNames = toSeq(keys(d.section[k].secItems))
for plainName in overloadableNames.sorted(cmpDecimalsIgnoreCase):
var overloadChoices = d.section[k].secItems[plainName]
overloadChoices.sort(cmp)
var nameContent = ""
for item in overloadChoices:
var itemDesc: string
renderItemPre(d, item.descRst, itemDesc)
nameContent.add(
getConfigVar(d.conf, "doc.item") % (
item.substitutions & @[
"desc", itemDesc,
"name", item.name,
"itemSymOrID", item.anchor]))
if k in routineKinds:
let plainNameEsc1 = esc(d.target, plainName.strip)
let plainNameEsc2 = esc(d.target, plainName.strip, escMode=emUrl)
d.section[k].finalMarkup.add(
getConfigVar(d.conf, "doc.item2") % (
@["header_plain", plainNameEsc1,
"overloadGroupName", overloadGroupName(plainNameEsc2, k),
"content", nameContent]))
else:
d.section[k].finalMarkup.add(nameContent)
d.section[k].secItems.clear
renderItemPre(d, d.modDescPre, d.modDescFinal)
d.modDescPre.setLen 0
d.hasToc = d.hasToc or d.sharedState.hasToc
# Finalize fragments of ``.json`` file
for i, entry in d.jEntriesPre:
if entry.rst != nil:
let resolved = resolveSubs(d.sharedState, entry.rst)
var str: string
renderRstToOut(d[], resolved, str)
entry.json[entry.rstField] = %str
d.jEntriesFinal.add entry.json
d.jEntriesPre[i].rst = nil
proc add(d: PDoc; j: JsonItem) =
if j.json != nil or j.rst != nil: d.jEntriesPre.add j
proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
case n.kind
of nkCommentStmt:
if includeComments:
d.add JsonItem(rst: genComment(d, n), rstField: "comment",
json: %Table[string, string]())
else:
d.modDescPre.add(genComment(d, n))
of nkProcDef, nkFuncDef:
when useEffectSystem: documentRaises(d.cache, n)
d.add genJsonItem(d, n, n[namePos], skProc)
of nkMethodDef:
when useEffectSystem: documentRaises(d.cache, n)
d.add genJsonItem(d, n, n[namePos], skMethod)
of nkIteratorDef:
when useEffectSystem: documentRaises(d.cache, n)
d.add genJsonItem(d, n, n[namePos], skIterator)
of nkMacroDef:
d.add genJsonItem(d, n, n[namePos], skMacro)
of nkTemplateDef:
d.add genJsonItem(d, n, n[namePos], skTemplate)
of nkConverterDef:
when useEffectSystem: documentRaises(d.cache, n)
d.add genJsonItem(d, n, n[namePos], skConverter)
of nkTypeSection, nkVarSection, nkLetSection, nkConstSection:
for i in 0..<n.len:
if n[i].kind != nkCommentStmt:
# order is always 'type var let const':
d.add genJsonItem(d, n[i], n[i][0],
succ(skType, ord(n.kind)-ord(nkTypeSection)))
of nkStmtList:
for i in 0..<n.len:
generateJson(d, n[i], includeComments)
of nkWhenStmt:
# generate documentation for the first branch only:
if not checkForFalse(n[0][0]):
generateJson(d, lastSon(n[0]), includeComments)
else: discard
proc genTagsItem(d: PDoc, n, nameNode: PNode, k: TSymKind): string =
result = getName(d, nameNode) & "\n"
proc generateTags*(d: PDoc, n: PNode, r: var string) =
case n.kind
of nkCommentStmt:
if startsWith(n.comment, "##"):
let stripped = n.comment.substr(2).strip
r.add stripped
of nkProcDef:
when useEffectSystem: documentRaises(d.cache, n)
r.add genTagsItem(d, n, n[namePos], skProc)
of nkFuncDef:
when useEffectSystem: documentRaises(d.cache, n)
r.add genTagsItem(d, n, n[namePos], skFunc)
of nkMethodDef:
when useEffectSystem: documentRaises(d.cache, n)
r.add genTagsItem(d, n, n[namePos], skMethod)
of nkIteratorDef:
when useEffectSystem: documentRaises(d.cache, n)
r.add genTagsItem(d, n, n[namePos], skIterator)
of nkMacroDef:
r.add genTagsItem(d, n, n[namePos], skMacro)
of nkTemplateDef:
r.add genTagsItem(d, n, n[namePos], skTemplate)
of nkConverterDef:
when useEffectSystem: documentRaises(d.cache, n)
r.add genTagsItem(d, n, n[namePos], skConverter)
of nkTypeSection, nkVarSection, nkLetSection, nkConstSection:
for i in 0..<n.len:
if n[i].kind != nkCommentStmt:
# order is always 'type var let const':
r.add genTagsItem(d, n[i], n[i][0],
succ(skType, ord(n.kind)-ord(nkTypeSection)))
of nkStmtList:
for i in 0..<n.len:
generateTags(d, n[i], r)
of nkWhenStmt:
# generate documentation for the first branch only:
if not checkForFalse(n[0][0]):
generateTags(d, lastSon(n[0]), r)
else: discard
proc genSection(d: PDoc, kind: TSymKind, groupedToc = false) =
const sectionNames: array[skModule..skField, string] = [
"Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Funcs",
"Methods", "Iterators", "Converters", "Macros", "Templates", "Exports"
]
if d.section[kind].finalMarkup == "": return
var title = sectionNames[kind]
d.section[kind].finalMarkup = getConfigVar(d.conf, "doc.section") % [
"sectionid", $ord(kind), "sectionTitle", title,
"sectionTitleID", $(ord(kind) + 50), "content", d.section[kind].finalMarkup]
proc cmp(x, y: TocItem): int = cmpDecimalsIgnoreCase(x.sortName, y.sortName)
if groupedToc:
let overloadableNames = toSeq(keys(d.tocTable[kind]))
for plainName in overloadableNames.sorted(cmpDecimalsIgnoreCase):
var overloadChoices = d.tocTable[kind][plainName]
overloadChoices.sort(cmp)
var content: string
for item in overloadChoices:
content.add item.content
d.toc2[kind].add getConfigVar(d.conf, "doc.section.toc2") % [
"sectionid", $ord(kind), "sectionTitle", title,
"sectionTitleID", $(ord(kind) + 50),
"content", content, "plainName", plainName]
else:
for item in d.tocSimple[kind].sorted(cmp):
d.toc2[kind].add item.content
d.toc[kind] = getConfigVar(d.conf, "doc.section.toc") % [
"sectionid", $ord(kind), "sectionTitle", title,
"sectionTitleID", $(ord(kind) + 50), "content", d.toc2[kind]]
proc relLink(outDir: AbsoluteDir, destFile: AbsoluteFile, linkto: RelativeFile): string =
$relativeTo(outDir / linkto, destFile.splitFile().dir, '/')
proc genOutFile(d: PDoc, groupedToc = false): string =
var
code, content: string
title = ""
var j = 0
var toc = ""
renderTocEntries(d[], j, 1, toc)
for i in TSymKind:
var shouldSort = i in routineKinds and groupedToc
genSection(d, i, shouldSort)
toc.add(d.toc[i])
if toc != "" or d.target == outLatex:
# for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set
toc = getConfigVar(d.conf, "doc.toc") % ["content", toc]
for i in TSymKind: code.add(d.section[i].finalMarkup)
# Extract the title. Non API modules generate an entry in the index table.
if d.meta[metaTitle].len != 0:
title = d.meta[metaTitle]
let external = AbsoluteFile(d.destFile)
.relativeTo(d.conf.outDir, '/')
.changeFileExt(HtmlExt)
.string
setIndexTerm(d[], external, "", title)
else:
# Modules get an automatic title for the HTML, but no entry in the index.
title = canonicalImport(d.conf, AbsoluteFile d.filename)
title = esc(d.target, title)
var subtitle = ""
if d.meta[metaSubtitle] != "":
dispA(d.conf, subtitle, "<h2 class=\"subtitle\">$1</h2>",
"\\\\\\vspace{0.5em}\\large $1", [esc(d.target, d.meta[metaSubtitle])])
var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection")
let bodyname = if d.hasToc and not d.isPureRst and not d.conf.isLatexCmd:
groupsection.setLen 0
"doc.body_toc_group"
elif d.hasToc: "doc.body_toc"
else: "doc.body_no_toc"
let seeSrc = genSeeSrc(d, d.filename, 1)
content = getConfigVar(d.conf, bodyname) % [
"title", title, "subtitle", subtitle,
"tableofcontents", toc, "moduledesc", d.modDescFinal, "date", getDateStr(),
"time", getClockStr(), "content", code,
"deprecationMsg", d.modDeprecationMsg,
"theindexhref", relLink(d.conf.outDir, d.destFile.AbsoluteFile,
theindexFname.RelativeFile),
"body_toc_groupsection", groupsection, "seeSrc", seeSrc]
if optCompileOnly notin d.conf.globalOptions:
# XXX what is this hack doing here? 'optCompileOnly' means raw output!?
code = getConfigVar(d.conf, "doc.file") % [
"nimdoccss", relLink(d.conf.outDir, d.destFile.AbsoluteFile,
nimdocOutCss.RelativeFile),
"dochackjs", relLink(d.conf.outDir, d.destFile.AbsoluteFile,
docHackJsFname.RelativeFile),
"title", title, "subtitle", subtitle, "tableofcontents", toc,
"moduledesc", d.modDescFinal, "date", getDateStr(), "time", getClockStr(),
"content", content, "author", d.meta[metaAuthor],
"version", esc(d.target, d.meta[metaVersion]), "analytics", d.analytics,
"deprecationMsg", d.modDeprecationMsg]
else:
code = content
result = code
proc generateIndex*(d: PDoc) =
if optGenIndex in d.conf.globalOptions:
let dir = d.conf.outDir
createDir(dir)
let dest = dir / changeFileExt(presentationPath(d.conf, AbsoluteFile d.filename), IndexExt)
writeIndexFile(d[], dest.string)
proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
if d.module == nil or sfMainModule in d.module.flags: # nil for e.g. for commandRst2Html
if d.conf.outFile.isEmpty:
d.conf.outFile = outfile.relativeTo(d.conf.outDir)
if isAbsolute(d.conf.outFile.string):
d.conf.outFile = splitPath(d.conf.outFile.string)[1].RelativeFile
proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) =
runAllExamples(d)
var content = genOutFile(d, groupedToc)
if optStdout in d.conf.globalOptions:
write(stdout, content)
else:
template outfile: untyped = d.destFile.AbsoluteFile
#let outfile = getOutFile2(d.conf, shortenDir(d.conf, filename), outExt)
let dir = outfile.splitFile.dir
createDir(dir)
updateOutfile(d, outfile)
try:
writeFile(outfile, content)
except IOError:
rawMessage(d.conf, if useWarning: warnCannotOpenFile else: errCannotOpenFile,
outfile.string)
if not d.wroteSupportFiles: # nimdoc.css + dochack.js
let nimr = $d.conf.getPrefixDir()
copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss)
if optGenIndex in d.conf.globalOptions:
let docHackJs2 = getDocHacksJs(nimr, nim = getAppFilename())
copyFile(docHackJs2, $d.conf.outDir / docHackJs2.lastPathPart)
d.wroteSupportFiles = true
proc writeOutputJson*(d: PDoc, useWarning = false) =
runAllExamples(d)
var modDesc: string
for desc in d.modDescFinal:
modDesc &= desc
let content = %*{"orig": d.filename,
"nimble": getPackageName(d.conf, d.filename),
"moduleDescription": modDesc,
"entries": d.jEntriesFinal}
if optStdout in d.conf.globalOptions:
write(stdout, $content)
else:
var f: File
if open(f, d.destFile, fmWrite):
write(f, $content)
close(f)
updateOutfile(d, d.destFile.AbsoluteFile)
else:
localError(d.conf, newLineInfo(d.conf, AbsoluteFile d.filename, -1, -1),
warnUser, "unable to open file \"" & d.destFile &
"\" for writing")
proc handleDocOutputOptions*(conf: ConfigRef) =
if optWholeProject in conf.globalOptions:
# Backward compatibility with previous versions
# xxx this is buggy when user provides `nim doc --project -o:sub/bar.html main`,
# it'd write to `sub/bar.html/main.html`
conf.outDir = AbsoluteDir(conf.outDir / conf.outFile)
proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
## implementation of deprecated ``doc0`` command (without semantic checking)
handleDocOutputOptions conf
var ast = parseFile(conf.projectMainIdx, cache, conf)
if ast == nil: return
var d = newDocumentor(conf.projectFull, cache, conf)
d.hasToc = true
generateDoc(d, ast, ast)
finishGenerateDoc(d)
writeOutput(d)
generateIndex(d)
proc commandRstAux(cache: IdentCache, conf: ConfigRef;
filename: AbsoluteFile, outExt: string) =
var filen = addFileExt(filename, "txt")
var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true)
let rst = parseRst(readFile(filen.string),
line=LineRstInit, column=ColRstInit,
conf, d.sharedState)
d.modDescPre = @[ItemFragment(isRst: true, rst: rst)]
finishGenerateDoc(d)
writeOutput(d)
generateIndex(d)
proc commandRst2Html*(cache: IdentCache, conf: ConfigRef) =
commandRstAux(cache, conf, conf.projectFull, HtmlExt)
proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef) =
commandRstAux(cache, conf, conf.projectFull, TexExt)
proc commandJson*(cache: IdentCache, conf: ConfigRef) =
## implementation of a deprecated jsondoc0 command
var ast = parseFile(conf.projectMainIdx, cache, conf)
if ast == nil: return
var d = newDocumentor(conf.projectFull, cache, conf)
d.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string;
status: int; content: string) =
localError(conf, newLineInfo(conf, AbsoluteFile d.filename, -1, -1),
warnUser, "the ':test:' attribute is not supported by this backend")
d.hasToc = true
generateJson(d, ast)
finishGenerateDoc(d)
let json = d.jEntriesFinal
let content = pretty(json)
if optStdout in d.conf.globalOptions:
write(stdout, content)
else:
#echo getOutFile(gProjectFull, JsonExt)
let filename = getOutFile(conf, RelativeFile conf.projectName, JsonExt)
try:
writeFile(filename, content)
except:
rawMessage(conf, errCannotOpenFile, filename.string)
proc commandTags*(cache: IdentCache, conf: ConfigRef) =
var ast = parseFile(conf.projectMainIdx, cache, conf)
if ast == nil: return
var d = newDocumentor(conf.projectFull, cache, conf)
d.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string;
status: int; content: string) =
localError(conf, newLineInfo(conf, AbsoluteFile d.filename, -1, -1),
warnUser, "the ':test:' attribute is not supported by this backend")
d.hasToc = true
var
content = ""
generateTags(d, ast, content)
if optStdout in d.conf.globalOptions:
write(stdout, content)
else:
#echo getOutFile(gProjectFull, TagsExt)
let filename = getOutFile(conf, RelativeFile conf.projectName, TagsExt)
try:
writeFile(filename, content)
except:
rawMessage(conf, errCannotOpenFile, filename.string)
proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") =
var content = mergeIndexes(dir)
var outFile = outFile
if outFile.isEmpty: outFile = theindexFname.RelativeFile.changeFileExt("")
let filename = getOutFile(conf, outFile, HtmlExt)
let code = getConfigVar(conf, "doc.file") % [
"nimdoccss", relLink(conf.outDir, filename, nimdocOutCss.RelativeFile),
"dochackjs", relLink(conf.outDir, filename, docHackJsFname.RelativeFile),
"title", "Index",
"subtitle", "", "tableofcontents", "", "moduledesc", "",
"date", getDateStr(), "time", getClockStr(),
"content", content, "author", "", "version", "", "analytics", ""]
# no analytics because context is not available
try:
writeFile(filename, code)
except:
rawMessage(conf, errCannotOpenFile, filename.string)