diff options
Diffstat (limited to 'tools/kochdocs.nim')
| -rw-r--r-- | tools/kochdocs.nim | 261 |
1 files changed, 155 insertions, 106 deletions
diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 43b802ee2..477fb29fa 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -1,21 +1,33 @@ ## Part of 'koch' responsible for the documentation generation. -import os, strutils, osproc, sets, pathnorm, pegs +import std/[os, strutils, osproc, sets, pathnorm, sequtils, pegs] + +import officialpackages +export exec + +when defined(nimPreviewSlimSystem): + import std/assertions + from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry import "../compiler/nimpaths" const gaCode* = " --doc.googleAnalytics:UA-48159761-1" + paCode* = " --doc.plausibleAnalytics:nim-lang.org" # errormax: subsequent errors are probably consequences of 1st one; a simple # bug could cause unlimited number of errors otherwise, hard to debug in CI. - nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off -d:boot --putenv:nimversion=$#" % system.NimVersion + docDefines = "-d:nimExperimentalLinenoiseExtra" + nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] gitUrl = "https://github.com/nim-lang/Nim" docHtmlOutput = "doc/html" webUploadOutput = "web/upload" var nimExe*: string +const allowList = ["jsbigints.nim", "jsheaders.nim", "jsformdata.nim", "jsfetch.nim", "jsutils.nim"] -template isJsOnly(file: string): bool = file.isRelativeTo("lib/js") +template isJsOnly(file: string): bool = + file.isRelativeTo("lib/js") or + file.extractFilename in allowList proc exe*(f: string): string = result = addFileExt(f, ExeExt) @@ -36,38 +48,20 @@ proc findNimImpl*(): tuple[path: string, ok: bool] = proc findNim*(): string = findNimImpl().path -proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = - let prevPath = getEnv("PATH") - if additionalPath.len > 0: - var absolute = additionalPath - if not absolute.isAbsolute: - absolute = getCurrentDir() / absolute - echo("Adding to $PATH: ", absolute) - putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & absolute) - echo(cmd) - if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) - putEnv("PATH", prevPath) - template inFold*(desc, body) = - if existsEnv("TRAVIS"): - echo "travis_fold:start:" & desc.replace(" ", "_") - elif existsEnv("GITHUB_ACTIONS"): + if existsEnv("GITHUB_ACTIONS"): echo "::group::" & desc elif existsEnv("TF_BUILD"): echo "##[group]" & desc - body - - if existsEnv("TRAVIS"): - echo "travis_fold:end:" & desc.replace(" ", "_") - elif existsEnv("GITHUB_ACTIONS"): + if existsEnv("GITHUB_ACTIONS"): echo "::endgroup::" elif existsEnv("TF_BUILD"): echo "##[endgroup]" proc execFold*(desc, cmd: string, errorcode: int = QuitFailure, additionalPath = "") = ## Execute shell command. Add log folding for various CI services. - # https://github.com/travis-ci/travis-ci/issues/2285#issuecomment-42724719 + let desc = if desc.len == 0: cmd else: desc inFold(desc): exec(cmd, errorcode, additionalPath) @@ -93,56 +87,53 @@ proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") = let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input exec cmd -proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "") = - let output = outputDir / input.splitFile.name.exe +proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "", outputName = "") = + let outputName2 = if outputName.len == 0: input.splitFile.name.exe else: outputName.exe + let output = outputDir / outputName2 let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input execFold(desc, cmd) -proc getRst2html(): seq[string] = +const officialPackagesMarkdown = """ +pkgs/atlas/doc/atlas.md +""".splitWhitespace() + +proc getMd2html(): seq[string] = for a in walkDirRecFilter("doc"): let path = a.path - if a.kind == pcFile and path.splitFile.ext == ".rst" and path.lastPathPart notin - ["docs.rst", "nimfix.rst"]: - # maybe we should still show nimfix, could help reviving it + if a.kind == pcFile and path.splitFile.ext == ".md" and path.lastPathPart notin + ["docs.md", + "docstyle.md" # docstyle.md shouldn't be converted to html separately; + # it's included in contributing.md. + ]: # `docs` is redundant with `overview`, might as well remove that file? result.add path - doAssert "doc/manual/var_t_return.rst".unixToNativePath in result # sanity check + for md in officialPackagesMarkdown: + result.add md + doAssert "doc/manual/var_t_return.md".unixToNativePath in result # sanity check const - pdf = """ -doc/manual.rst -doc/lib.rst -doc/tut1.rst -doc/tut2.rst -doc/tut3.rst -doc/nimc.rst -doc/niminst.rst -doc/gc.rst -""".splitWhitespace() - - doc0 = """ -lib/system/threads.nim -lib/system/channels.nim -""".splitWhitespace() # ran by `nim doc0` instead of `nim doc` + mdPdfList = """ +manual.md +lib.md +tut1.md +tut2.md +tut3.md +nimc.md +niminst.md +mm.md +""".splitWhitespace().mapIt("doc" / it) withoutIndex = """ -lib/wrappers/mysql.nim -lib/wrappers/iup.nim -lib/wrappers/sqlite3.nim -lib/wrappers/postgres.nim lib/wrappers/tinyc.nim -lib/wrappers/odbcsql.nim lib/wrappers/pcre.nim lib/wrappers/openssl.nim lib/posix/posix.nim lib/posix/linux.nim lib/posix/termios.nim -lib/js/jscore.nim """.splitWhitespace() # some of these are include files so shouldn't be docgen'd ignoredModules = """ -lib/prelude.nim lib/pure/future.nim lib/pure/collections/hashcommon.nim lib/pure/collections/tableimpl.nim @@ -158,8 +149,37 @@ lib/posix/posix_nintendoswitch_consts.nim lib/posix/posix_linux_amd64.nim lib/posix/posix_linux_amd64_consts.nim lib/posix/posix_other_consts.nim +lib/posix/posix_freertos_consts.nim lib/posix/posix_openbsd_amd64.nim lib/posix/posix_haiku.nim +lib/pure/md5.nim +lib/std/sha1.nim +lib/pure/htmlparser.nim +""".splitWhitespace() + + officialPackagesList = """ +pkgs/asyncftpclient/src/asyncftpclient.nim +pkgs/smtp/src/smtp.nim +pkgs/punycode/src/punycode.nim +pkgs/db_connector/src/db_connector/db_common.nim +pkgs/db_connector/src/db_connector/db_mysql.nim +pkgs/db_connector/src/db_connector/db_odbc.nim +pkgs/db_connector/src/db_connector/db_postgres.nim +pkgs/db_connector/src/db_connector/db_sqlite.nim +pkgs/checksums/src/checksums/md5.nim +pkgs/checksums/src/checksums/sha1.nim +pkgs/checksums/src/checksums/sha2.nim +pkgs/checksums/src/checksums/sha3.nim +pkgs/checksums/src/checksums/bcrypt.nim +pkgs/htmlparser/src/htmlparser.nim +""".splitWhitespace() + + officialPackagesListWithoutIndex = """ +pkgs/db_connector/src/db_connector/mysql.nim +pkgs/db_connector/src/db_connector/sqlite3.nim +pkgs/db_connector/src/db_connector/postgres.nim +pkgs/db_connector/src/db_connector/odbcsql.nim +pkgs/db_connector/src/db_connector/private/dbutils.nim """.splitWhitespace() when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): @@ -170,23 +190,25 @@ when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): result = path.len > 0 and not ret.startsWith ".." proc getDocList(): seq[string] = + ## var docIgnore: HashSet[string] - for a in doc0: docIgnore.incl a for a in withoutIndex: docIgnore.incl a for a in ignoredModules: docIgnore.incl a # don't ignore these even though in lib/system (not include files) const goodSystem = """ -lib/system/io.nim lib/system/nimscript.nim lib/system/assertions.nim lib/system/iterators.nim +lib/system/exceptions.nim lib/system/dollars.nim -lib/system/widestrs.nim +lib/system/ctypes.nim """.splitWhitespace() proc follow(a: PathEntry): bool = - a.path.lastPathPart notin ["nimcache", "htmldocs", "includes", "deprecated", "genode"] + result = a.path.lastPathPart notin ["nimcache", htmldocsDirname, + "includes", "deprecated", "genode"] and + not a.path.isRelativeTo("lib/fusion") # fusion was un-bundled but we need to keep this in case user has it installed for entry in walkDirRecFilter("lib", follow = follow): let a = entry.path if entry.kind != pcFile or a.splitFile.ext != ".nim" or @@ -220,105 +242,132 @@ proc buildDocSamples(nimArgs, destPath: string) = exec(findNim().quoteShell() & " doc $# -o:$# $#" % [nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"]) -proc buildDocPackages(nimArgs, destPath: string) = +proc buildDocPackages(nimArgs, destPath: string, indexOnly: bool) = # compiler docs; later, other packages (perhaps tools, testament etc) let nim = findNim().quoteShell() # to avoid broken links to manual from compiler dir, but a multi-package # structure could be supported later proc docProject(outdir, options, mainproj: string) = - exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $options $mainproj" % [ + exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $index $options $mainproj" % [ "nim", nim, "outdir", outdir, "nimArgs", nimArgs, "gitUrl", gitUrl, "options", options, "mainproj", mainproj, + "index", if indexOnly: "--index:only" else: "" ]) let extra = "-u:boot" # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather, # start using nims instead of nimdoc.cfg docProject(destPath/"compiler", extra, "compiler/index.nim") -proc buildDoc(nimArgs, destPath: string) = +proc buildDoc(nimArgs, destPath: string, indexOnly: bool) = # call nim for the documentation: - let rst2html = getRst2html() + let rst2html = getMd2html() var - commands = newSeq[string](rst2html.len + len(doc0) + len(doc) + withoutIndex.len) + commands = newSeq[string](rst2html.len + len(doc) + withoutIndex.len + + officialPackagesList.len + officialPackagesListWithoutIndex.len) i = 0 let nim = findNim().quoteShell() + + let index = if indexOnly: "--index:only" else: "" for d in items(rst2html): - commands[i] = nim & " rst2html $# --git.url:$# -o:$# --index:on $#" % + commands[i] = nim & " md2html $# --git.url:$# -o:$# $# $#" % [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] - i.inc - for d in items(doc0): - commands[i] = nim & " doc0 $# --git.url:$# -o:$# --index:on $#" % - [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] + destPath / changeFileExt(splitFile(d).name, "html"), index, d] i.inc for d in items(doc): - let extra = if isJsOnly(d): " --backend:js " else: "" + let extra = if isJsOnly(d): "--backend:js" else: "" var nimArgs2 = nimArgs if d.isRelativeTo("compiler"): doAssert false - commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# --index:on $#" % - [extra, nimArgs2, gitUrl, destPath, d] + commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# $# $#" % + [extra, nimArgs2, gitUrl, destPath, index, d] i.inc for d in items(withoutIndex): - commands[i] = nim & " doc2 $# --git.url:$# -o:$# $#" % + commands[i] = nim & " doc $# --git.url:$# -o:$# $#" % [nimArgs, gitUrl, destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc + + for d in items(officialPackagesList): + var nimArgs2 = nimArgs + if d.isRelativeTo("compiler"): doAssert false + commands[i] = nim & " doc $# --outdir:$# --index:on $#" % + [nimArgs2, destPath, d] + i.inc + for d in items(officialPackagesListWithoutIndex): + commands[i] = nim & " doc $# -o:$# $#" % + [nimArgs, + destPath / changeFileExt(splitFile(d).name, "html"), d] + i.inc + mexec(commands) - exec(nim & " buildIndex -o:$1/theindex.html $1" % [destPath]) - # caveat: this works so long it's called before `buildDocPackages` which - # populates `compiler/` with unrelated idx files that shouldn't be in index, - # so should work in CI but you may need to remove your generated html files - # locally after calling `./koch docs`. The clean fix would be for `idx` files - # to be transient with `--project` (eg all in memory). -proc buildPdfDoc*(nimArgs, destPath: string) = +proc nim2pdf(src: string, dst: string, nimArgs: string) = + # xxx expose as a `nim` command or in some other reusable way. + let outDir = "build" / "xelatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 + # note: this will generate temporary files in gitignored `outDir`: aux toc log out tex + exec("$# md2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell]) + let texFile = outDir / src.lastPathPart.changeFileExt("tex") + for i in 0..<3: # call LaTeX three times to get cross references right: + let xelatexLog = outDir / "xelatex.log" + # `>` should work on windows, if not, we can use `execCmdEx` + let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell] + exec(cmd) # on error, user can inspect `xelatexLog` + if i == 1: # build .ind file + var texFileBase = texFile + texFileBase.removeSuffix(".tex") + let cmd = "makeindex $# > $#" % [ + texFileBase.quoteShell, xelatexLog.quoteShell] + exec(cmd) + moveFile(texFile.changeFileExt("pdf"), dst) + +proc buildPdfDoc*(args: string, destPath: string) = + let args = nimArgs & " " & args + var pdfList: seq[string] createDir(destPath) - if os.execShellCmd("pdflatex -version") != 0: - echo "pdflatex not found; no PDF documentation generated" + if os.execShellCmd("xelatex -version") != 0: + doAssert false, "xelatex not found" # or, raise an exception else: - const pdflatexcmd = "pdflatex -interaction=nonstopmode " - for d in items(pdf): - exec(findNim().quoteShell() & " rst2tex $# $#" % [nimArgs, d]) - let tex = splitFile(d).name & ".tex" - removeFile("doc" / tex) - moveFile(tex, "doc" / tex) - # call LaTeX twice to get cross references right: - exec(pdflatexcmd & changeFileExt(d, "tex")) - exec(pdflatexcmd & changeFileExt(d, "tex")) - # delete all the crappy temporary files: - let pdf = splitFile(d).name & ".pdf" - let dest = destPath / pdf - removeFile(dest) - moveFile(dest=dest, source=pdf) - removeFile(changeFileExt(pdf, "aux")) - if fileExists(changeFileExt(pdf, "toc")): - removeFile(changeFileExt(pdf, "toc")) - removeFile(changeFileExt(pdf, "log")) - removeFile(changeFileExt(pdf, "out")) - removeFile(changeFileExt(d, "tex")) + for src in items(mdPdfList): + let dst = destPath / src.lastPathPart.changeFileExt("pdf") + pdfList.add dst + nim2pdf(src, dst, args) + echo "\nOutput PDF files: \n ", pdfList.join(" ") # because `nim2pdf` is a bit verbose proc buildJS(): string = let nim = findNim() - exec(nim.quoteShell() & " js -d:release --out:$1 tools/nimblepkglist.nim" % - [webUploadOutput / "nimblepkglist.js"]) + exec("$# js -d:release --out:$# tools/nimblepkglist.nim" % + [nim.quoteShell(), webUploadOutput / "nimblepkglist.js"]) # xxx deadcode? and why is it only for webUploadOutput, not for local docs? result = getDocHacksJs(nimr = getCurrentDir(), nim) proc buildDocsDir*(args: string, dir: string) = let args = nimArgs & " " & args let docHackJsSource = buildJS() + gitClonePackages(@["asyncftpclient", "punycode", "smtp", "db_connector", "checksums", "atlas", "htmlparser"]) createDir(dir) buildDocSamples(args, dir) - buildDoc(args, dir) # bottleneck + + # generate `.idx` files and top-level `theindex.html`: + buildDoc(args, dir, indexOnly=true) # bottleneck + let nim = findNim().quoteShell() + exec(nim & " buildIndex -o:$1/theindex.html $1" % [dir]) + # caveat: this works so long it's called before `buildDocPackages` which + # populates `compiler/` with unrelated idx files that shouldn't be in index, + # so should work in CI but you may need to remove your generated html files + # locally after calling `./koch docs`. The clean fix would be for `idx` files + # to be transient with `--project` (eg all in memory). + buildDocPackages(args, dir, indexOnly=true) + + # generate HTML and package-level `theindex.html`: + buildDoc(args, dir, indexOnly=false) # bottleneck + buildDocPackages(args, dir, indexOnly=false) + copyFile(dir / "overview.html", dir / "index.html") - buildDocPackages(args, dir) copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart) proc buildDocs*(args: string, localOnly = false, localOutDir = "") = |