diff options
Diffstat (limited to 'tools/kochdocs.nim')
| -rw-r--r-- | tools/kochdocs.nim | 493 |
1 files changed, 249 insertions, 244 deletions
diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 073dff919..477fb29fa 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -1,57 +1,67 @@ ## Part of 'koch' responsible for the documentation generation. -import os, strutils, osproc, sets +import std/[os, strutils, osproc, sets, pathnorm, sequtils, pegs] + +import officialpackages +export exec + +when defined(nimPreviewSlimSystem): + import std/assertions + +from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry +import "../compiler/nimpaths" const gaCode* = " --doc.googleAnalytics:UA-48159761-1" - # --warning[LockLevel]:off pending #13218 - nimArgs = "--warning[LockLevel]:off --hint[Conf]:off --hint[Path]:off --hint[Processing]:off -d:boot --putenv:nimversion=$#" % system.NimVersion + paCode* = " --doc.plausibleAnalytics:nim-lang.org" + # errormax: subsequent errors are probably consequences of 1st one; a simple + # bug could cause unlimited number of errors otherwise, hard to debug in CI. + docDefines = "-d:nimExperimentalLinenoiseExtra" + nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] gitUrl = "https://github.com/nim-lang/Nim" docHtmlOutput = "doc/html" webUploadOutput = "web/upload" - docHackDir = "tools/dochack" var nimExe*: string +const allowList = ["jsbigints.nim", "jsheaders.nim", "jsformdata.nim", "jsfetch.nim", "jsutils.nim"] + +template isJsOnly(file: string): bool = + file.isRelativeTo("lib/js") or + file.extractFilename in allowList proc exe*(f: string): string = result = addFileExt(f, ExeExt) when defined(windows): result = result.replace('/','\\') -proc findNim*(): string = - if nimExe.len > 0: return nimExe - var nim = "nim".exe - result = "bin" / nim - if existsFile(result): return +proc findNimImpl*(): tuple[path: string, ok: bool] = + if nimExe.len > 0: return (nimExe, true) + let nim = "nim".exe + result.path = "bin" / nim + result.ok = true + if fileExists(result.path): return for dir in split(getEnv("PATH"), PathSep): - if existsFile(dir / nim): return dir / nim + result.path = dir / nim + if fileExists(result.path): return # assume there is a symlink to the exe or something: - return nim + return (nim, false) -proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = - let prevPath = getEnv("PATH") - if additionalPath.len > 0: - var absolute = additionalPath - if not absolute.isAbsolute: - absolute = getCurrentDir() / absolute - echo("Adding to $PATH: ", absolute) - putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & absolute) - echo(cmd) - if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) - putEnv("PATH", prevPath) +proc findNim*(): string = findNimImpl().path template inFold*(desc, body) = - if existsEnv("TRAVIS"): - echo "travis_fold:start:" & desc.replace(" ", "_") - + if existsEnv("GITHUB_ACTIONS"): + echo "::group::" & desc + elif existsEnv("TF_BUILD"): + echo "##[group]" & desc body - - if existsEnv("TRAVIS"): - echo "travis_fold:end:" & desc.replace(" ", "_") + if existsEnv("GITHUB_ACTIONS"): + echo "::endgroup::" + elif existsEnv("TF_BUILD"): + echo "##[endgroup]" proc execFold*(desc, cmd: string, errorcode: int = QuitFailure, additionalPath = "") = - ## Execute shell command. Add log folding on Travis CI. - # https://github.com/travis-ci/travis-ci/issues/2285#issuecomment-42724719 + ## Execute shell command. Add log folding for various CI services. + let desc = if desc.len == 0: cmd else: desc inFold(desc): exec(cmd, errorcode, additionalPath) @@ -70,86 +80,61 @@ proc execCleanPath*(cmd: string, proc nimexec*(cmd: string) = # Consider using `nimCompile` instead - exec findNim() & " " & cmd + exec findNim().quoteShell() & " " & cmd proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") = let output = outputDir / input.splitFile.name.exe - let cmd = findNim() & " " & mode & " -o:" & output & " " & options & " " & input + let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input exec cmd -proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "") = - let output = outputDir / input.splitFile.name.exe - let cmd = findNim() & " " & mode & " -o:" & output & " " & options & " " & input +proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "", outputName = "") = + let outputName2 = if outputName.len == 0: input.splitFile.name.exe else: outputName.exe + let output = outputDir / outputName2 + let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input execFold(desc, cmd) -const - pdf = """ -doc/manual.rst -doc/lib.rst -doc/tut1.rst -doc/tut2.rst -doc/tut3.rst -doc/nimc.rst -doc/niminst.rst -doc/gc.rst +const officialPackagesMarkdown = """ +pkgs/atlas/doc/atlas.md """.splitWhitespace() - rst2html = """ -doc/intern.rst -doc/apis.rst -doc/lib.rst -doc/manual.rst -doc/manual_experimental.rst -doc/destructors.rst -doc/tut1.rst -doc/tut2.rst -doc/tut3.rst -doc/nimc.rst -doc/hcr.rst -doc/overview.rst -doc/filters.rst -doc/tools.rst -doc/niminst.rst -doc/nimgrep.rst -doc/gc.rst -doc/estp.rst -doc/idetools.rst -doc/docgen.rst -doc/koch.rst -doc/backends.rst -doc/nimsuggest.rst -doc/nep1.rst -doc/nims.rst -doc/contributing.rst -doc/codeowners.rst -doc/packaging.rst -doc/manual/var_t_return.rst -""".splitWhitespace() +proc getMd2html(): seq[string] = + for a in walkDirRecFilter("doc"): + let path = a.path + if a.kind == pcFile and path.splitFile.ext == ".md" and path.lastPathPart notin + ["docs.md", + "docstyle.md" # docstyle.md shouldn't be converted to html separately; + # it's included in contributing.md. + ]: + # `docs` is redundant with `overview`, might as well remove that file? + result.add path + for md in officialPackagesMarkdown: + result.add md + doAssert "doc/manual/var_t_return.md".unixToNativePath in result # sanity check - doc0 = """ -lib/system/threads.nim -lib/system/channels.nim -""".splitWhitespace() # ran by `nim doc0` instead of `nim doc` +const + mdPdfList = """ +manual.md +lib.md +tut1.md +tut2.md +tut3.md +nimc.md +niminst.md +mm.md +""".splitWhitespace().mapIt("doc" / it) withoutIndex = """ -lib/wrappers/mysql.nim -lib/wrappers/iup.nim -lib/wrappers/sqlite3.nim -lib/wrappers/postgres.nim lib/wrappers/tinyc.nim -lib/wrappers/odbcsql.nim lib/wrappers/pcre.nim lib/wrappers/openssl.nim lib/posix/posix.nim lib/posix/linux.nim lib/posix/termios.nim -lib/js/jscore.nim """.splitWhitespace() + # some of these are include files so shouldn't be docgen'd ignoredModules = """ lib/pure/future.nim -lib/impure/osinfo_posix.nim -lib/impure/osinfo_win.nim lib/pure/collections/hashcommon.nim lib/pure/collections/tableimpl.nim lib/pure/collections/setimpl.nim @@ -164,119 +149,74 @@ lib/posix/posix_nintendoswitch_consts.nim lib/posix/posix_linux_amd64.nim lib/posix/posix_linux_amd64_consts.nim lib/posix/posix_other_consts.nim +lib/posix/posix_freertos_consts.nim lib/posix/posix_openbsd_amd64.nim +lib/posix/posix_haiku.nim +lib/pure/md5.nim +lib/std/sha1.nim +lib/pure/htmlparser.nim +""".splitWhitespace() + + officialPackagesList = """ +pkgs/asyncftpclient/src/asyncftpclient.nim +pkgs/smtp/src/smtp.nim +pkgs/punycode/src/punycode.nim +pkgs/db_connector/src/db_connector/db_common.nim +pkgs/db_connector/src/db_connector/db_mysql.nim +pkgs/db_connector/src/db_connector/db_odbc.nim +pkgs/db_connector/src/db_connector/db_postgres.nim +pkgs/db_connector/src/db_connector/db_sqlite.nim +pkgs/checksums/src/checksums/md5.nim +pkgs/checksums/src/checksums/sha1.nim +pkgs/checksums/src/checksums/sha2.nim +pkgs/checksums/src/checksums/sha3.nim +pkgs/checksums/src/checksums/bcrypt.nim +pkgs/htmlparser/src/htmlparser.nim +""".splitWhitespace() + + officialPackagesListWithoutIndex = """ +pkgs/db_connector/src/db_connector/mysql.nim +pkgs/db_connector/src/db_connector/sqlite3.nim +pkgs/db_connector/src/db_connector/postgres.nim +pkgs/db_connector/src/db_connector/odbcsql.nim +pkgs/db_connector/src/db_connector/private/dbutils.nim """.splitWhitespace() - # some of these (eg lib/posix/posix_macos_amd64.nim) are include files - # but contain potentially valuable docs on OS-specific symbols (eg OSX) that - # don't end up in the main docs; we ignore these for now. when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): proc isRelativeTo(path, base: string): bool = - # pending #13212 use os.isRelativeTo let path = path.normalizedPath let base = base.normalizedPath let ret = relativePath(path, base) result = path.len > 0 and not ret.startsWith ".." proc getDocList(): seq[string] = - var t: HashSet[string] - for a in doc0: - doAssert a notin t - t.incl a - for a in withoutIndex: - doAssert a notin t, a - t.incl a - - for a in ignoredModules: - doAssert a notin t, a - t.incl a - - var t2: HashSet[string] - template myadd(a)= - result.add a - doAssert a notin t2, a - t2.incl a + ## + var docIgnore: HashSet[string] + for a in withoutIndex: docIgnore.incl a + for a in ignoredModules: docIgnore.incl a - # don't ignore these even though in lib/system + # don't ignore these even though in lib/system (not include files) const goodSystem = """ -lib/system/io.nim lib/system/nimscript.nim lib/system/assertions.nim lib/system/iterators.nim +lib/system/exceptions.nim lib/system/dollars.nim -lib/system/widestrs.nim +lib/system/ctypes.nim """.splitWhitespace() - for a in walkDirRec("lib"): - if a.splitFile.ext != ".nim": continue - if a.isRelativeTo("lib/pure/includes"): continue - if a.isRelativeTo("lib/genode"): continue - if a.isRelativeTo("lib/deprecated"): - if a notin @["lib/deprecated/pure/ospaths.nim"]: # REMOVE - continue - if a.isRelativeTo("lib/system"): - if a notin goodSystem: - continue - if a notin t: - result.add a - doAssert a notin t2, a - t2.incl a - - myadd "nimsuggest/sexp.nim" - # these are include files, even though some of them don't specify `included from ...` - const ignore = """ -compiler/ccgcalls.nim -compiler/ccgexprs.nim -compiler/ccgliterals.nim -compiler/ccgstmts.nim -compiler/ccgthreadvars.nim -compiler/ccgtrav.nim -compiler/ccgtypes.nim -compiler/jstypes.nim -compiler/semcall.nim -compiler/semexprs.nim -compiler/semfields.nim -compiler/semgnrc.nim -compiler/seminst.nim -compiler/semmagic.nim -compiler/semobjconstr.nim -compiler/semstmts.nim -compiler/semtempl.nim -compiler/semtypes.nim -compiler/sizealignoffsetimpl.nim -compiler/suggest.nim -compiler/packagehandling.nim -compiler/hlo.nim -compiler/rodimpl.nim -compiler/vmops.nim -compiler/vmhooks.nim -""".splitWhitespace() - - # not include files but doesn't work; not included/imported anywhere; dead code? - const bad = """ -compiler/debuginfo.nim -compiler/canonicalizer.nim -compiler/forloops.nim -""".splitWhitespace() - - # these cause errors even though they're imported (some of which are mysterious) - const bad2 = """ -compiler/closureiters.nim -compiler/tccgen.nim -compiler/lambdalifting.nim -compiler/layouter.nim -compiler/evalffi.nim -compiler/nimfix/nimfix.nim -compiler/plugins/active.nim -compiler/plugins/itersgen.nim -""".splitWhitespace() - - for a in walkDirRec("compiler"): - if a.splitFile.ext != ".nim": continue - if a in ignore: continue - if a in bad: continue - if a in bad2: continue + proc follow(a: PathEntry): bool = + result = a.path.lastPathPart notin ["nimcache", htmldocsDirname, + "includes", "deprecated", "genode"] and + not a.path.isRelativeTo("lib/fusion") # fusion was un-bundled but we need to keep this in case user has it installed + for entry in walkDirRecFilter("lib", follow = follow): + let a = entry.path + if entry.kind != pcFile or a.splitFile.ext != ".nim" or + (a.isRelativeTo("lib/system") and a.nativeToUnixPath notin goodSystem) or + a.nativeToUnixPath in docIgnore: + continue result.add a + result.add normalizePath("nimsuggest/sexp.nim") let doc = getDocList() @@ -299,85 +239,150 @@ proc buildDocSamples(nimArgs, destPath: string) = ## ## TODO: consider integrating into the existing generic documentation builders ## now that we have a single `doc` command. - exec(findNim() & " doc $# -o:$# $#" % + exec(findNim().quoteShell() & " doc $# -o:$# $#" % [nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"]) -proc buildDoc(nimArgs, destPath: string) = +proc buildDocPackages(nimArgs, destPath: string, indexOnly: bool) = + # compiler docs; later, other packages (perhaps tools, testament etc) + let nim = findNim().quoteShell() + # to avoid broken links to manual from compiler dir, but a multi-package + # structure could be supported later + + proc docProject(outdir, options, mainproj: string) = + exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $index $options $mainproj" % [ + "nim", nim, + "outdir", outdir, + "nimArgs", nimArgs, + "gitUrl", gitUrl, + "options", options, + "mainproj", mainproj, + "index", if indexOnly: "--index:only" else: "" + ]) + let extra = "-u:boot" + # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather, + # start using nims instead of nimdoc.cfg + docProject(destPath/"compiler", extra, "compiler/index.nim") + +proc buildDoc(nimArgs, destPath: string, indexOnly: bool) = # call nim for the documentation: + let rst2html = getMd2html() var - commands = newSeq[string](rst2html.len + len(doc0) + len(doc) + withoutIndex.len) + commands = newSeq[string](rst2html.len + len(doc) + withoutIndex.len + + officialPackagesList.len + officialPackagesListWithoutIndex.len) i = 0 - let nim = findNim() + let nim = findNim().quoteShell() + + let index = if indexOnly: "--index:only" else: "" for d in items(rst2html): - commands[i] = nim & " rst2html $# --git.url:$# -o:$# --index:on $#" % - [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] - i.inc - for d in items(doc0): - commands[i] = nim & " doc0 $# --git.url:$# -o:$# --index:on $#" % + commands[i] = nim & " md2html $# --git.url:$# -o:$# $# $#" % [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] + destPath / changeFileExt(splitFile(d).name, "html"), index, d] i.inc for d in items(doc): - commands[i] = nim & " doc $# --git.url:$# -o:$# --index:on $#" % - [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] + let extra = if isJsOnly(d): "--backend:js" else: "" + var nimArgs2 = nimArgs + if d.isRelativeTo("compiler"): doAssert false + commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# $# $#" % + [extra, nimArgs2, gitUrl, destPath, index, d] i.inc for d in items(withoutIndex): - commands[i] = nim & " doc2 $# --git.url:$# -o:$# $#" % + commands[i] = nim & " doc $# --git.url:$# -o:$# $#" % [nimArgs, gitUrl, destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc + + for d in items(officialPackagesList): + var nimArgs2 = nimArgs + if d.isRelativeTo("compiler"): doAssert false + commands[i] = nim & " doc $# --outdir:$# --index:on $#" % + [nimArgs2, destPath, d] + i.inc + for d in items(officialPackagesListWithoutIndex): + commands[i] = nim & " doc $# -o:$# $#" % + [nimArgs, + destPath / changeFileExt(splitFile(d).name, "html"), d] + i.inc + mexec(commands) - exec(nim & " buildIndex -o:$1/theindex.html $1" % [destPath]) -proc buildPdfDoc*(nimArgs, destPath: string) = +proc nim2pdf(src: string, dst: string, nimArgs: string) = + # xxx expose as a `nim` command or in some other reusable way. + let outDir = "build" / "xelatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 + # note: this will generate temporary files in gitignored `outDir`: aux toc log out tex + exec("$# md2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell]) + let texFile = outDir / src.lastPathPart.changeFileExt("tex") + for i in 0..<3: # call LaTeX three times to get cross references right: + let xelatexLog = outDir / "xelatex.log" + # `>` should work on windows, if not, we can use `execCmdEx` + let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell] + exec(cmd) # on error, user can inspect `xelatexLog` + if i == 1: # build .ind file + var texFileBase = texFile + texFileBase.removeSuffix(".tex") + let cmd = "makeindex $# > $#" % [ + texFileBase.quoteShell, xelatexLog.quoteShell] + exec(cmd) + moveFile(texFile.changeFileExt("pdf"), dst) + +proc buildPdfDoc*(args: string, destPath: string) = + let args = nimArgs & " " & args + var pdfList: seq[string] createDir(destPath) - if os.execShellCmd("pdflatex -version") != 0: - echo "pdflatex not found; no PDF documentation generated" + if os.execShellCmd("xelatex -version") != 0: + doAssert false, "xelatex not found" # or, raise an exception else: - const pdflatexcmd = "pdflatex -interaction=nonstopmode " - for d in items(pdf): - exec(findNim() & " rst2tex $# $#" % [nimArgs, d]) - # call LaTeX twice to get cross references right: - exec(pdflatexcmd & changeFileExt(d, "tex")) - exec(pdflatexcmd & changeFileExt(d, "tex")) - # delete all the crappy temporary files: - let pdf = splitFile(d).name & ".pdf" - let dest = destPath / pdf - removeFile(dest) - moveFile(dest=dest, source=pdf) - removeFile(changeFileExt(pdf, "aux")) - if existsFile(changeFileExt(pdf, "toc")): - removeFile(changeFileExt(pdf, "toc")) - removeFile(changeFileExt(pdf, "log")) - removeFile(changeFileExt(pdf, "out")) - removeFile(changeFileExt(d, "tex")) - -proc buildJS() = - exec(findNim() & " js -d:release --out:$1 tools/nimblepkglist.nim" % - [webUploadOutput / "nimblepkglist.js"]) - exec(findNim() & " js " & (docHackDir / "dochack.nim")) - -proc buildDocs*(args: string) = - const - docHackJs = "dochack.js" - let - a = nimArgs & " " & args - docHackJsSource = docHackDir / docHackJs - docHackJsDest = docHtmlOutput / docHackJs - - buildJS() # This call generates docHackJsSource - let docup = webUploadOutput / NimVersion - createDir(docup) - buildDocSamples(a, docup) - buildDoc(a, docup) - - # 'nimArgs' instead of 'a' is correct here because we don't want - # that the offline docs contain the 'gaCode'! - createDir(docHtmlOutput) - buildDocSamples(nimArgs, docHtmlOutput) - buildDoc(nimArgs, docHtmlOutput) - copyFile(docHackJsSource, docHackJsDest) - copyFile(docHackJsSource, docup / docHackJs) + for src in items(mdPdfList): + let dst = destPath / src.lastPathPart.changeFileExt("pdf") + pdfList.add dst + nim2pdf(src, dst, args) + echo "\nOutput PDF files: \n ", pdfList.join(" ") # because `nim2pdf` is a bit verbose + +proc buildJS(): string = + let nim = findNim() + exec("$# js -d:release --out:$# tools/nimblepkglist.nim" % + [nim.quoteShell(), webUploadOutput / "nimblepkglist.js"]) + # xxx deadcode? and why is it only for webUploadOutput, not for local docs? + result = getDocHacksJs(nimr = getCurrentDir(), nim) + +proc buildDocsDir*(args: string, dir: string) = + let args = nimArgs & " " & args + let docHackJsSource = buildJS() + gitClonePackages(@["asyncftpclient", "punycode", "smtp", "db_connector", "checksums", "atlas", "htmlparser"]) + createDir(dir) + buildDocSamples(args, dir) + + # generate `.idx` files and top-level `theindex.html`: + buildDoc(args, dir, indexOnly=true) # bottleneck + let nim = findNim().quoteShell() + exec(nim & " buildIndex -o:$1/theindex.html $1" % [dir]) + # caveat: this works so long it's called before `buildDocPackages` which + # populates `compiler/` with unrelated idx files that shouldn't be in index, + # so should work in CI but you may need to remove your generated html files + # locally after calling `./koch docs`. The clean fix would be for `idx` files + # to be transient with `--project` (eg all in memory). + buildDocPackages(args, dir, indexOnly=true) + + # generate HTML and package-level `theindex.html`: + buildDoc(args, dir, indexOnly=false) # bottleneck + buildDocPackages(args, dir, indexOnly=false) + + copyFile(dir / "overview.html", dir / "index.html") + copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart) + +proc buildDocs*(args: string, localOnly = false, localOutDir = "") = + let localOutDir = + if localOutDir.len == 0: + docHtmlOutput + else: + localOutDir + + var args = args + + if not localOnly: + buildDocsDir(args, webUploadOutput / NimVersion) + + let gaFilter = peg"@( y'--doc.googleAnalytics:' @(\s / $) )" + args = args.replace(gaFilter) + + buildDocsDir(args, localOutDir) |