diff options
Diffstat (limited to 'tools/kochdocs.nim')
| -rw-r--r-- | tools/kochdocs.nim | 388 |
1 files changed, 388 insertions, 0 deletions
diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim new file mode 100644 index 000000000..477fb29fa --- /dev/null +++ b/tools/kochdocs.nim @@ -0,0 +1,388 @@ +## Part of 'koch' responsible for the documentation generation. + +import std/[os, strutils, osproc, sets, pathnorm, sequtils, pegs] + +import officialpackages +export exec + +when defined(nimPreviewSlimSystem): + import std/assertions + +from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry +import "../compiler/nimpaths" + +const + gaCode* = " --doc.googleAnalytics:UA-48159761-1" + paCode* = " --doc.plausibleAnalytics:nim-lang.org" + # errormax: subsequent errors are probably consequences of 1st one; a simple + # bug could cause unlimited number of errors otherwise, hard to debug in CI. + docDefines = "-d:nimExperimentalLinenoiseExtra" + nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] + gitUrl = "https://github.com/nim-lang/Nim" + docHtmlOutput = "doc/html" + webUploadOutput = "web/upload" + +var nimExe*: string +const allowList = ["jsbigints.nim", "jsheaders.nim", "jsformdata.nim", "jsfetch.nim", "jsutils.nim"] + +template isJsOnly(file: string): bool = + file.isRelativeTo("lib/js") or + file.extractFilename in allowList + +proc exe*(f: string): string = + result = addFileExt(f, ExeExt) + when defined(windows): + result = result.replace('/','\\') + +proc findNimImpl*(): tuple[path: string, ok: bool] = + if nimExe.len > 0: return (nimExe, true) + let nim = "nim".exe + result.path = "bin" / nim + result.ok = true + if fileExists(result.path): return + for dir in split(getEnv("PATH"), PathSep): + result.path = dir / nim + if fileExists(result.path): return + # assume there is a symlink to the exe or something: + return (nim, false) + +proc findNim*(): string = findNimImpl().path + +template inFold*(desc, body) = + if existsEnv("GITHUB_ACTIONS"): + echo "::group::" & desc + elif existsEnv("TF_BUILD"): + echo "##[group]" & desc + body + if existsEnv("GITHUB_ACTIONS"): + echo "::endgroup::" + elif existsEnv("TF_BUILD"): + echo "##[endgroup]" + +proc execFold*(desc, cmd: string, errorcode: int = QuitFailure, additionalPath = "") = + ## Execute shell command. Add log folding for various CI services. + let desc = if desc.len == 0: cmd else: desc + inFold(desc): + exec(cmd, errorcode, additionalPath) + +proc execCleanPath*(cmd: string, + additionalPath = ""; errorcode: int = QuitFailure) = + # simulate a poor man's virtual environment + let prevPath = getEnv("PATH") + when defined(windows): + let cleanPath = r"$1\system32;$1;$1\System32\Wbem" % getEnv"SYSTEMROOT" + else: + const cleanPath = r"/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin" + putEnv("PATH", cleanPath & PathSep & additionalPath) + echo(cmd) + if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) + putEnv("PATH", prevPath) + +proc nimexec*(cmd: string) = + # Consider using `nimCompile` instead + exec findNim().quoteShell() & " " & cmd + +proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") = + let output = outputDir / input.splitFile.name.exe + let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input + exec cmd + +proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "", outputName = "") = + let outputName2 = if outputName.len == 0: input.splitFile.name.exe else: outputName.exe + let output = outputDir / outputName2 + let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input + execFold(desc, cmd) + +const officialPackagesMarkdown = """ +pkgs/atlas/doc/atlas.md +""".splitWhitespace() + +proc getMd2html(): seq[string] = + for a in walkDirRecFilter("doc"): + let path = a.path + if a.kind == pcFile and path.splitFile.ext == ".md" and path.lastPathPart notin + ["docs.md", + "docstyle.md" # docstyle.md shouldn't be converted to html separately; + # it's included in contributing.md. + ]: + # `docs` is redundant with `overview`, might as well remove that file? + result.add path + for md in officialPackagesMarkdown: + result.add md + doAssert "doc/manual/var_t_return.md".unixToNativePath in result # sanity check + +const + mdPdfList = """ +manual.md +lib.md +tut1.md +tut2.md +tut3.md +nimc.md +niminst.md +mm.md +""".splitWhitespace().mapIt("doc" / it) + + withoutIndex = """ +lib/wrappers/tinyc.nim +lib/wrappers/pcre.nim +lib/wrappers/openssl.nim +lib/posix/posix.nim +lib/posix/linux.nim +lib/posix/termios.nim +""".splitWhitespace() + + # some of these are include files so shouldn't be docgen'd + ignoredModules = """ +lib/pure/future.nim +lib/pure/collections/hashcommon.nim +lib/pure/collections/tableimpl.nim +lib/pure/collections/setimpl.nim +lib/pure/ioselects/ioselectors_kqueue.nim +lib/pure/ioselects/ioselectors_select.nim +lib/pure/ioselects/ioselectors_poll.nim +lib/pure/ioselects/ioselectors_epoll.nim +lib/posix/posix_macos_amd64.nim +lib/posix/posix_other.nim +lib/posix/posix_nintendoswitch.nim +lib/posix/posix_nintendoswitch_consts.nim +lib/posix/posix_linux_amd64.nim +lib/posix/posix_linux_amd64_consts.nim +lib/posix/posix_other_consts.nim +lib/posix/posix_freertos_consts.nim +lib/posix/posix_openbsd_amd64.nim +lib/posix/posix_haiku.nim +lib/pure/md5.nim +lib/std/sha1.nim +lib/pure/htmlparser.nim +""".splitWhitespace() + + officialPackagesList = """ +pkgs/asyncftpclient/src/asyncftpclient.nim +pkgs/smtp/src/smtp.nim +pkgs/punycode/src/punycode.nim +pkgs/db_connector/src/db_connector/db_common.nim +pkgs/db_connector/src/db_connector/db_mysql.nim +pkgs/db_connector/src/db_connector/db_odbc.nim +pkgs/db_connector/src/db_connector/db_postgres.nim +pkgs/db_connector/src/db_connector/db_sqlite.nim +pkgs/checksums/src/checksums/md5.nim +pkgs/checksums/src/checksums/sha1.nim +pkgs/checksums/src/checksums/sha2.nim +pkgs/checksums/src/checksums/sha3.nim +pkgs/checksums/src/checksums/bcrypt.nim +pkgs/htmlparser/src/htmlparser.nim +""".splitWhitespace() + + officialPackagesListWithoutIndex = """ +pkgs/db_connector/src/db_connector/mysql.nim +pkgs/db_connector/src/db_connector/sqlite3.nim +pkgs/db_connector/src/db_connector/postgres.nim +pkgs/db_connector/src/db_connector/odbcsql.nim +pkgs/db_connector/src/db_connector/private/dbutils.nim +""".splitWhitespace() + +when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): + proc isRelativeTo(path, base: string): bool = + let path = path.normalizedPath + let base = base.normalizedPath + let ret = relativePath(path, base) + result = path.len > 0 and not ret.startsWith ".." + +proc getDocList(): seq[string] = + ## + var docIgnore: HashSet[string] + for a in withoutIndex: docIgnore.incl a + for a in ignoredModules: docIgnore.incl a + + # don't ignore these even though in lib/system (not include files) + const goodSystem = """ +lib/system/nimscript.nim +lib/system/assertions.nim +lib/system/iterators.nim +lib/system/exceptions.nim +lib/system/dollars.nim +lib/system/ctypes.nim +""".splitWhitespace() + + proc follow(a: PathEntry): bool = + result = a.path.lastPathPart notin ["nimcache", htmldocsDirname, + "includes", "deprecated", "genode"] and + not a.path.isRelativeTo("lib/fusion") # fusion was un-bundled but we need to keep this in case user has it installed + for entry in walkDirRecFilter("lib", follow = follow): + let a = entry.path + if entry.kind != pcFile or a.splitFile.ext != ".nim" or + (a.isRelativeTo("lib/system") and a.nativeToUnixPath notin goodSystem) or + a.nativeToUnixPath in docIgnore: + continue + result.add a + result.add normalizePath("nimsuggest/sexp.nim") + +let doc = getDocList() + +proc sexec(cmds: openArray[string]) = + ## Serial queue wrapper around exec. + for cmd in cmds: + echo(cmd) + let (outp, exitCode) = osproc.execCmdEx(cmd) + if exitCode != 0: quit outp + +proc mexec(cmds: openArray[string]) = + ## Multiprocessor version of exec + let r = execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd}) + if r != 0: + echo "external program failed, retrying serial work queue for logs!" + sexec(cmds) + +proc buildDocSamples(nimArgs, destPath: string) = + ## Special case documentation sample proc. + ## + ## TODO: consider integrating into the existing generic documentation builders + ## now that we have a single `doc` command. + exec(findNim().quoteShell() & " doc $# -o:$# $#" % + [nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"]) + +proc buildDocPackages(nimArgs, destPath: string, indexOnly: bool) = + # compiler docs; later, other packages (perhaps tools, testament etc) + let nim = findNim().quoteShell() + # to avoid broken links to manual from compiler dir, but a multi-package + # structure could be supported later + + proc docProject(outdir, options, mainproj: string) = + exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $index $options $mainproj" % [ + "nim", nim, + "outdir", outdir, + "nimArgs", nimArgs, + "gitUrl", gitUrl, + "options", options, + "mainproj", mainproj, + "index", if indexOnly: "--index:only" else: "" + ]) + let extra = "-u:boot" + # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather, + # start using nims instead of nimdoc.cfg + docProject(destPath/"compiler", extra, "compiler/index.nim") + +proc buildDoc(nimArgs, destPath: string, indexOnly: bool) = + # call nim for the documentation: + let rst2html = getMd2html() + var + commands = newSeq[string](rst2html.len + len(doc) + withoutIndex.len + + officialPackagesList.len + officialPackagesListWithoutIndex.len) + i = 0 + let nim = findNim().quoteShell() + + let index = if indexOnly: "--index:only" else: "" + for d in items(rst2html): + commands[i] = nim & " md2html $# --git.url:$# -o:$# $# $#" % + [nimArgs, gitUrl, + destPath / changeFileExt(splitFile(d).name, "html"), index, d] + i.inc + for d in items(doc): + let extra = if isJsOnly(d): "--backend:js" else: "" + var nimArgs2 = nimArgs + if d.isRelativeTo("compiler"): doAssert false + commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# $# $#" % + [extra, nimArgs2, gitUrl, destPath, index, d] + i.inc + for d in items(withoutIndex): + commands[i] = nim & " doc $# --git.url:$# -o:$# $#" % + [nimArgs, gitUrl, + destPath / changeFileExt(splitFile(d).name, "html"), d] + i.inc + + + for d in items(officialPackagesList): + var nimArgs2 = nimArgs + if d.isRelativeTo("compiler"): doAssert false + commands[i] = nim & " doc $# --outdir:$# --index:on $#" % + [nimArgs2, destPath, d] + i.inc + for d in items(officialPackagesListWithoutIndex): + commands[i] = nim & " doc $# -o:$# $#" % + [nimArgs, + destPath / changeFileExt(splitFile(d).name, "html"), d] + i.inc + + mexec(commands) + +proc nim2pdf(src: string, dst: string, nimArgs: string) = + # xxx expose as a `nim` command or in some other reusable way. + let outDir = "build" / "xelatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 + # note: this will generate temporary files in gitignored `outDir`: aux toc log out tex + exec("$# md2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell]) + let texFile = outDir / src.lastPathPart.changeFileExt("tex") + for i in 0..<3: # call LaTeX three times to get cross references right: + let xelatexLog = outDir / "xelatex.log" + # `>` should work on windows, if not, we can use `execCmdEx` + let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell] + exec(cmd) # on error, user can inspect `xelatexLog` + if i == 1: # build .ind file + var texFileBase = texFile + texFileBase.removeSuffix(".tex") + let cmd = "makeindex $# > $#" % [ + texFileBase.quoteShell, xelatexLog.quoteShell] + exec(cmd) + moveFile(texFile.changeFileExt("pdf"), dst) + +proc buildPdfDoc*(args: string, destPath: string) = + let args = nimArgs & " " & args + var pdfList: seq[string] + createDir(destPath) + if os.execShellCmd("xelatex -version") != 0: + doAssert false, "xelatex not found" # or, raise an exception + else: + for src in items(mdPdfList): + let dst = destPath / src.lastPathPart.changeFileExt("pdf") + pdfList.add dst + nim2pdf(src, dst, args) + echo "\nOutput PDF files: \n ", pdfList.join(" ") # because `nim2pdf` is a bit verbose + +proc buildJS(): string = + let nim = findNim() + exec("$# js -d:release --out:$# tools/nimblepkglist.nim" % + [nim.quoteShell(), webUploadOutput / "nimblepkglist.js"]) + # xxx deadcode? and why is it only for webUploadOutput, not for local docs? + result = getDocHacksJs(nimr = getCurrentDir(), nim) + +proc buildDocsDir*(args: string, dir: string) = + let args = nimArgs & " " & args + let docHackJsSource = buildJS() + gitClonePackages(@["asyncftpclient", "punycode", "smtp", "db_connector", "checksums", "atlas", "htmlparser"]) + createDir(dir) + buildDocSamples(args, dir) + + # generate `.idx` files and top-level `theindex.html`: + buildDoc(args, dir, indexOnly=true) # bottleneck + let nim = findNim().quoteShell() + exec(nim & " buildIndex -o:$1/theindex.html $1" % [dir]) + # caveat: this works so long it's called before `buildDocPackages` which + # populates `compiler/` with unrelated idx files that shouldn't be in index, + # so should work in CI but you may need to remove your generated html files + # locally after calling `./koch docs`. The clean fix would be for `idx` files + # to be transient with `--project` (eg all in memory). + buildDocPackages(args, dir, indexOnly=true) + + # generate HTML and package-level `theindex.html`: + buildDoc(args, dir, indexOnly=false) # bottleneck + buildDocPackages(args, dir, indexOnly=false) + + copyFile(dir / "overview.html", dir / "index.html") + copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart) + +proc buildDocs*(args: string, localOnly = false, localOutDir = "") = + let localOutDir = + if localOutDir.len == 0: + docHtmlOutput + else: + localOutDir + + var args = args + + if not localOnly: + buildDocsDir(args, webUploadOutput / NimVersion) + + let gaFilter = peg"@( y'--doc.googleAnalytics:' @(\s / $) )" + args = args.replace(gaFilter) + + buildDocsDir(args, localOutDir) |