diff options
Diffstat (limited to 'tools/kochdocs.nim')
| -rw-r--r-- | tools/kochdocs.nim | 214 |
1 files changed, 130 insertions, 84 deletions
diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 621dc643f..477fb29fa 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -1,14 +1,23 @@ ## Part of 'koch' responsible for the documentation generation. -import os, strutils, osproc, sets, pathnorm, pegs, sequtils +import std/[os, strutils, osproc, sets, pathnorm, sequtils, pegs] + +import officialpackages +export exec + +when defined(nimPreviewSlimSystem): + import std/assertions + from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry import "../compiler/nimpaths" const gaCode* = " --doc.googleAnalytics:UA-48159761-1" + paCode* = " --doc.plausibleAnalytics:nim-lang.org" # errormax: subsequent errors are probably consequences of 1st one; a simple # bug could cause unlimited number of errors otherwise, hard to debug in CI. - nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$#" % system.NimVersion + docDefines = "-d:nimExperimentalLinenoiseExtra" + nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] gitUrl = "https://github.com/nim-lang/Nim" docHtmlOutput = "doc/html" webUploadOutput = "web/upload" @@ -39,38 +48,19 @@ proc findNimImpl*(): tuple[path: string, ok: bool] = proc findNim*(): string = findNimImpl().path -proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = - let prevPath = getEnv("PATH") - if additionalPath.len > 0: - var absolute = additionalPath - if not absolute.isAbsolute: - absolute = getCurrentDir() / absolute - echo("Adding to $PATH: ", absolute) - putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & absolute) - echo(cmd) - if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) - putEnv("PATH", prevPath) - template inFold*(desc, body) = - if existsEnv("TRAVIS"): - echo "travis_fold:start:" & desc.replace(" ", "_") - elif existsEnv("GITHUB_ACTIONS"): + if existsEnv("GITHUB_ACTIONS"): echo "::group::" & desc elif existsEnv("TF_BUILD"): echo "##[group]" & desc - body - - if existsEnv("TRAVIS"): - echo "travis_fold:end:" & desc.replace(" ", "_") - elif existsEnv("GITHUB_ACTIONS"): + if existsEnv("GITHUB_ACTIONS"): echo "::endgroup::" elif existsEnv("TF_BUILD"): echo "##[endgroup]" proc execFold*(desc, cmd: string, errorcode: int = QuitFailure, additionalPath = "") = ## Execute shell command. Add log folding for various CI services. - # https://github.com/travis-ci/travis-ci/issues/2285#issuecomment-42724719 let desc = if desc.len == 0: cmd else: desc inFold(desc): exec(cmd, errorcode, additionalPath) @@ -97,44 +87,44 @@ proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") = let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input exec cmd -proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "") = - let output = outputDir / input.splitFile.name.exe +proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "", outputName = "") = + let outputName2 = if outputName.len == 0: input.splitFile.name.exe else: outputName.exe + let output = outputDir / outputName2 let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input execFold(desc, cmd) -proc getRst2html(): seq[string] = +const officialPackagesMarkdown = """ +pkgs/atlas/doc/atlas.md +""".splitWhitespace() + +proc getMd2html(): seq[string] = for a in walkDirRecFilter("doc"): let path = a.path - if a.kind == pcFile and path.splitFile.ext == ".rst" and path.lastPathPart notin - ["docs.rst", "nimfix.rst"]: - # maybe we should still show nimfix, could help reviving it + if a.kind == pcFile and path.splitFile.ext == ".md" and path.lastPathPart notin + ["docs.md", + "docstyle.md" # docstyle.md shouldn't be converted to html separately; + # it's included in contributing.md. + ]: # `docs` is redundant with `overview`, might as well remove that file? result.add path - doAssert "doc/manual/var_t_return.rst".unixToNativePath in result # sanity check + for md in officialPackagesMarkdown: + result.add md + doAssert "doc/manual/var_t_return.md".unixToNativePath in result # sanity check const - rstPdfList = """ -manual.rst -lib.rst -tut1.rst -tut2.rst -tut3.rst -nimc.rst -niminst.rst -gc.rst + mdPdfList = """ +manual.md +lib.md +tut1.md +tut2.md +tut3.md +nimc.md +niminst.md +mm.md """.splitWhitespace().mapIt("doc" / it) - doc0 = """ -lib/system/threads.nim -lib/system/channels_builtin.nim -""".splitWhitespace() # ran by `nim doc0` instead of `nim doc` - withoutIndex = """ -lib/wrappers/mysql.nim -lib/wrappers/sqlite3.nim -lib/wrappers/postgres.nim lib/wrappers/tinyc.nim -lib/wrappers/odbcsql.nim lib/wrappers/pcre.nim lib/wrappers/openssl.nim lib/posix/posix.nim @@ -162,6 +152,34 @@ lib/posix/posix_other_consts.nim lib/posix/posix_freertos_consts.nim lib/posix/posix_openbsd_amd64.nim lib/posix/posix_haiku.nim +lib/pure/md5.nim +lib/std/sha1.nim +lib/pure/htmlparser.nim +""".splitWhitespace() + + officialPackagesList = """ +pkgs/asyncftpclient/src/asyncftpclient.nim +pkgs/smtp/src/smtp.nim +pkgs/punycode/src/punycode.nim +pkgs/db_connector/src/db_connector/db_common.nim +pkgs/db_connector/src/db_connector/db_mysql.nim +pkgs/db_connector/src/db_connector/db_odbc.nim +pkgs/db_connector/src/db_connector/db_postgres.nim +pkgs/db_connector/src/db_connector/db_sqlite.nim +pkgs/checksums/src/checksums/md5.nim +pkgs/checksums/src/checksums/sha1.nim +pkgs/checksums/src/checksums/sha2.nim +pkgs/checksums/src/checksums/sha3.nim +pkgs/checksums/src/checksums/bcrypt.nim +pkgs/htmlparser/src/htmlparser.nim +""".splitWhitespace() + + officialPackagesListWithoutIndex = """ +pkgs/db_connector/src/db_connector/mysql.nim +pkgs/db_connector/src/db_connector/sqlite3.nim +pkgs/db_connector/src/db_connector/postgres.nim +pkgs/db_connector/src/db_connector/odbcsql.nim +pkgs/db_connector/src/db_connector/private/dbutils.nim """.splitWhitespace() when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): @@ -172,19 +190,19 @@ when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): result = path.len > 0 and not ret.startsWith ".." proc getDocList(): seq[string] = + ## var docIgnore: HashSet[string] - for a in doc0: docIgnore.incl a for a in withoutIndex: docIgnore.incl a for a in ignoredModules: docIgnore.incl a # don't ignore these even though in lib/system (not include files) const goodSystem = """ -lib/system/io.nim lib/system/nimscript.nim lib/system/assertions.nim lib/system/iterators.nim +lib/system/exceptions.nim lib/system/dollars.nim -lib/system/widestrs.nim +lib/system/ctypes.nim """.splitWhitespace() proc follow(a: PathEntry): bool = @@ -224,49 +242,48 @@ proc buildDocSamples(nimArgs, destPath: string) = exec(findNim().quoteShell() & " doc $# -o:$# $#" % [nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"]) -proc buildDocPackages(nimArgs, destPath: string) = +proc buildDocPackages(nimArgs, destPath: string, indexOnly: bool) = # compiler docs; later, other packages (perhaps tools, testament etc) let nim = findNim().quoteShell() # to avoid broken links to manual from compiler dir, but a multi-package # structure could be supported later proc docProject(outdir, options, mainproj: string) = - exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $options $mainproj" % [ + exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $index $options $mainproj" % [ "nim", nim, "outdir", outdir, "nimArgs", nimArgs, "gitUrl", gitUrl, "options", options, "mainproj", mainproj, + "index", if indexOnly: "--index:only" else: "" ]) let extra = "-u:boot" # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather, # start using nims instead of nimdoc.cfg docProject(destPath/"compiler", extra, "compiler/index.nim") -proc buildDoc(nimArgs, destPath: string) = +proc buildDoc(nimArgs, destPath: string, indexOnly: bool) = # call nim for the documentation: - let rst2html = getRst2html() + let rst2html = getMd2html() var - commands = newSeq[string](rst2html.len + len(doc0) + len(doc) + withoutIndex.len) + commands = newSeq[string](rst2html.len + len(doc) + withoutIndex.len + + officialPackagesList.len + officialPackagesListWithoutIndex.len) i = 0 let nim = findNim().quoteShell() + + let index = if indexOnly: "--index:only" else: "" for d in items(rst2html): - commands[i] = nim & " rst2html $# --git.url:$# -o:$# --index:on $#" % - [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] - i.inc - for d in items(doc0): - commands[i] = nim & " doc0 $# --git.url:$# -o:$# --index:on $#" % + commands[i] = nim & " md2html $# --git.url:$# -o:$# $# $#" % [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] + destPath / changeFileExt(splitFile(d).name, "html"), index, d] i.inc for d in items(doc): let extra = if isJsOnly(d): "--backend:js" else: "" var nimArgs2 = nimArgs if d.isRelativeTo("compiler"): doAssert false - commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# --index:on $#" % - [extra, nimArgs2, gitUrl, destPath, d] + commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# $# $#" % + [extra, nimArgs2, gitUrl, destPath, index, d] i.inc for d in items(withoutIndex): commands[i] = nim & " doc $# --git.url:$# -o:$# $#" % @@ -274,37 +291,51 @@ proc buildDoc(nimArgs, destPath: string) = destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc + + for d in items(officialPackagesList): + var nimArgs2 = nimArgs + if d.isRelativeTo("compiler"): doAssert false + commands[i] = nim & " doc $# --outdir:$# --index:on $#" % + [nimArgs2, destPath, d] + i.inc + for d in items(officialPackagesListWithoutIndex): + commands[i] = nim & " doc $# -o:$# $#" % + [nimArgs, + destPath / changeFileExt(splitFile(d).name, "html"), d] + i.inc + mexec(commands) - exec(nim & " buildIndex -o:$1/theindex.html $1" % [destPath]) - # caveat: this works so long it's called before `buildDocPackages` which - # populates `compiler/` with unrelated idx files that shouldn't be in index, - # so should work in CI but you may need to remove your generated html files - # locally after calling `./koch docs`. The clean fix would be for `idx` files - # to be transient with `--project` (eg all in memory). proc nim2pdf(src: string, dst: string, nimArgs: string) = # xxx expose as a `nim` command or in some other reusable way. - let outDir = "build" / "pdflatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 + let outDir = "build" / "xelatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 # note: this will generate temporary files in gitignored `outDir`: aux toc log out tex - exec("$# rst2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell]) + exec("$# md2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell]) let texFile = outDir / src.lastPathPart.changeFileExt("tex") - for i in 0..<2: # call LaTeX twice to get cross references right: - let pdflatexLog = outDir / "pdflatex.log" + for i in 0..<3: # call LaTeX three times to get cross references right: + let xelatexLog = outDir / "xelatex.log" # `>` should work on windows, if not, we can use `execCmdEx` - let cmd = "pdflatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, pdflatexLog.quoteShell] - exec(cmd) # on error, user can inspect `pdflatexLog` + let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell] + exec(cmd) # on error, user can inspect `xelatexLog` + if i == 1: # build .ind file + var texFileBase = texFile + texFileBase.removeSuffix(".tex") + let cmd = "makeindex $# > $#" % [ + texFileBase.quoteShell, xelatexLog.quoteShell] + exec(cmd) moveFile(texFile.changeFileExt("pdf"), dst) -proc buildPdfDoc*(nimArgs, destPath: string) = +proc buildPdfDoc*(args: string, destPath: string) = + let args = nimArgs & " " & args var pdfList: seq[string] createDir(destPath) - if os.execShellCmd("pdflatex -version") != 0: - doAssert false, "pdflatex not found" # or, raise an exception + if os.execShellCmd("xelatex -version") != 0: + doAssert false, "xelatex not found" # or, raise an exception else: - for src in items(rstPdfList): + for src in items(mdPdfList): let dst = destPath / src.lastPathPart.changeFileExt("pdf") pdfList.add dst - nim2pdf(src, dst, nimArgs) + nim2pdf(src, dst, args) echo "\nOutput PDF files: \n ", pdfList.join(" ") # because `nim2pdf` is a bit verbose proc buildJS(): string = @@ -317,11 +348,26 @@ proc buildJS(): string = proc buildDocsDir*(args: string, dir: string) = let args = nimArgs & " " & args let docHackJsSource = buildJS() + gitClonePackages(@["asyncftpclient", "punycode", "smtp", "db_connector", "checksums", "atlas", "htmlparser"]) createDir(dir) buildDocSamples(args, dir) - buildDoc(args, dir) # bottleneck + + # generate `.idx` files and top-level `theindex.html`: + buildDoc(args, dir, indexOnly=true) # bottleneck + let nim = findNim().quoteShell() + exec(nim & " buildIndex -o:$1/theindex.html $1" % [dir]) + # caveat: this works so long it's called before `buildDocPackages` which + # populates `compiler/` with unrelated idx files that shouldn't be in index, + # so should work in CI but you may need to remove your generated html files + # locally after calling `./koch docs`. The clean fix would be for `idx` files + # to be transient with `--project` (eg all in memory). + buildDocPackages(args, dir, indexOnly=true) + + # generate HTML and package-level `theindex.html`: + buildDoc(args, dir, indexOnly=false) # bottleneck + buildDocPackages(args, dir, indexOnly=false) + copyFile(dir / "overview.html", dir / "index.html") - buildDocPackages(args, dir) copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart) proc buildDocs*(args: string, localOnly = false, localOutDir = "") = |