kochdocs: use a glob instead of hardcoded list; generate docs for compiler/; bugfixes (#13221)

* kochdocs: use a glob instead of hardcoded list; generate docs for compiler/; bugfixes
* fixup after #13212 isRelativeTo got merged

1 files changed, 138 insertions, 145 deletions

diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim
index 3b3df75a7..073dff919 100644
--- a/tools/kochdocs.nim
+++ b/tools/kochdocs.nim
@@ -1,11 +1,11 @@
 ## Part of 'koch' responsible for the documentation generation.
 
-import os, strutils, osproc
+import os, strutils, osproc, sets
 
 const
   gaCode* = " --doc.googleAnalytics:UA-48159761-1"
-
-  nimArgs = "--hint[Conf]:off --hint[Path]:off --hint[Processing]:off -d:boot --putenv:nimversion=$#" % system.NimVersion
+  # --warning[LockLevel]:off pending #13218
+  nimArgs = "--warning[LockLevel]:off --hint[Conf]:off --hint[Path]:off --hint[Processing]:off -d:boot --putenv:nimversion=$#" % system.NimVersion
   gitUrl = "https://github.com/nim-lang/Nim"
   docHtmlOutput = "doc/html"
   webUploadOutput = "web/upload"
@@ -126,150 +126,10 @@ doc/packaging.rst
 doc/manual/var_t_return.rst
 """.splitWhitespace()
 
-  doc = """
-lib/system.nim
-lib/system/io.nim
-lib/system/nimscript.nim
-lib/system/assertions.nim
-lib/system/iterators.nim
-lib/system/dollars.nim
-lib/system/widestrs.nim
-lib/deprecated/pure/ospaths.nim
-lib/pure/parsejson.nim
-lib/pure/cstrutils.nim
-lib/core/macros.nim
-lib/pure/marshal.nim
-lib/core/typeinfo.nim
-lib/impure/re.nim
-lib/pure/typetraits.nim
-nimsuggest/sexp.nim
-lib/pure/concurrency/threadpool.nim
-lib/pure/concurrency/cpuinfo.nim
-lib/pure/concurrency/cpuload.nim
-lib/js/dom.nim
-lib/js/jsffi.nim
-lib/js/jsconsole.nim
-lib/js/asyncjs.nim
-lib/pure/os.nim
-lib/pure/strutils.nim
-lib/pure/math.nim
-lib/std/editdistance.nim
-lib/std/wordwrap.nim
-lib/std/wrapnils.nim
-lib/experimental/diff.nim
-lib/pure/algorithm.nim
-lib/pure/stats.nim
-lib/windows/winlean.nim
-lib/windows/registry.nim
-lib/pure/random.nim
-lib/pure/complex.nim
-lib/pure/times.nim
-lib/std/monotimes.nim
-lib/pure/osproc.nim
-lib/pure/pegs.nim
-lib/pure/dynlib.nim
-lib/pure/strscans.nim
-lib/pure/parseopt.nim
-lib/pure/hashes.nim
-lib/pure/strtabs.nim
-lib/pure/lexbase.nim
-lib/pure/parsecfg.nim
-lib/pure/parsexml.nim
-lib/pure/parsecsv.nim
-lib/pure/parsesql.nim
-lib/pure/streams.nim
-lib/pure/terminal.nim
-lib/pure/cgi.nim
-lib/pure/unicode.nim
-lib/pure/strmisc.nim
-lib/pure/htmlgen.nim
-lib/pure/parseutils.nim
-lib/pure/browsers.nim
-lib/impure/db_postgres.nim
-lib/impure/db_mysql.nim
-lib/impure/db_sqlite.nim
-lib/impure/db_odbc.nim
-lib/pure/db_common.nim
-lib/pure/httpclient.nim
-lib/pure/smtp.nim
-lib/pure/ropes.nim
-lib/pure/unidecode/unidecode.nim
-lib/pure/xmlparser.nim
-lib/pure/htmlparser.nim
-lib/pure/xmltree.nim
-lib/pure/colors.nim
-lib/pure/mimetypes.nim
-lib/pure/json.nim
-lib/pure/base64.nim
-lib/impure/nre.nim
-lib/impure/nre/private/util.nim
-lib/pure/collections/tables.nim
-lib/pure/collections/sets.nim
-lib/pure/collections/lists.nim
-lib/pure/collections/sharedlist.nim
-lib/pure/collections/sharedtables.nim
-lib/pure/collections/intsets.nim
-lib/pure/collections/deques.nim
-lib/pure/encodings.nim
-lib/pure/collections/sequtils.nim
-lib/pure/collections/rtarrays.nim
-lib/pure/cookies.nim
-lib/pure/memfiles.nim
-lib/pure/collections/critbits.nim
-lib/core/locks.nim
-lib/core/rlocks.nim
-lib/pure/oids.nim
-lib/pure/endians.nim
-lib/pure/uri.nim
-lib/pure/nimprof.nim
-lib/pure/unittest.nim
-lib/packages/docutils/highlite.nim
-lib/packages/docutils/rst.nim
-lib/packages/docutils/rstast.nim
-lib/packages/docutils/rstgen.nim
-lib/pure/logging.nim
-lib/pure/options.nim
-lib/pure/asyncdispatch.nim
-lib/pure/asyncnet.nim
-lib/pure/asyncstreams.nim
-lib/pure/asyncfutures.nim
-lib/pure/nativesockets.nim
-lib/pure/asynchttpserver.nim
-lib/pure/net.nim
-lib/pure/selectors.nim
-lib/pure/sugar.nim
-lib/pure/collections/chains.nim
-lib/pure/asyncfile.nim
-lib/pure/asyncftpclient.nim
-lib/pure/lenientops.nim
-lib/pure/md5.nim
-lib/pure/rationals.nim
-lib/pure/distros.nim
-lib/pure/oswalkdir.nim
-lib/pure/collections/heapqueue.nim
-lib/pure/fenv.nim
-lib/std/sha1.nim
-lib/std/sums.nim
-lib/std/varints.nim
-lib/std/time_t.nim
-lib/impure/rdstdin.nim
-lib/wrappers/linenoise/linenoise.nim
-lib/pure/strformat.nim
-lib/pure/segfaults.nim
-lib/pure/mersenne.nim
-lib/pure/coro.nim
-lib/pure/httpcore.nim
-lib/pure/bitops.nim
-lib/pure/nimtracker.nim
-lib/pure/punycode.nim
-lib/pure/volatile.nim
-lib/posix/posix_utils.nim
-""".splitWhitespace()
-
   doc0 = """
 lib/system/threads.nim
 lib/system/channels.nim
-""".splitWhitespace()
+""".splitWhitespace() # ran by `nim doc0` instead of `nim doc`
 
   withoutIndex = """
 lib/wrappers/mysql.nim
@@ -283,10 +143,143 @@ lib/wrappers/openssl.nim
 lib/posix/posix.nim
 lib/posix/linux.nim
 lib/posix/termios.nim
-lib/wrappers/odbcsql.nim
 lib/js/jscore.nim
 """.splitWhitespace()
 
+  ignoredModules = """
+lib/pure/future.nim
+lib/impure/osinfo_posix.nim
+lib/impure/osinfo_win.nim
+lib/pure/collections/hashcommon.nim
+lib/pure/collections/tableimpl.nim
+lib/pure/collections/setimpl.nim
+lib/pure/ioselects/ioselectors_kqueue.nim
+lib/pure/ioselects/ioselectors_select.nim
+lib/pure/ioselects/ioselectors_poll.nim
+lib/pure/ioselects/ioselectors_epoll.nim
+lib/posix/posix_macos_amd64.nim
+lib/posix/posix_other.nim
+lib/posix/posix_nintendoswitch.nim
+lib/posix/posix_nintendoswitch_consts.nim
+lib/posix/posix_linux_amd64.nim
+lib/posix/posix_linux_amd64_consts.nim
+lib/posix/posix_other_consts.nim
+lib/posix/posix_openbsd_amd64.nim
+""".splitWhitespace()
+  # some of these (eg lib/posix/posix_macos_amd64.nim) are include files
+  # but contain potentially valuable docs on OS-specific symbols (eg OSX) that
+  # don't end up in the main docs; we ignore these for now.
+
+when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo):
+  proc isRelativeTo(path, base: string): bool =
+    # pending #13212 use os.isRelativeTo
+    let path = path.normalizedPath
+    let base = base.normalizedPath
+    let ret = relativePath(path, base)
+    result = path.len > 0 and not ret.startsWith ".."
+
+proc getDocList(): seq[string] =
+  var t: HashSet[string]
+  for a in doc0:
+    doAssert a notin t
+    t.incl a
+  for a in withoutIndex:
+    doAssert a notin t, a
+    t.incl a
+
+  for a in ignoredModules:
+    doAssert a notin t, a
+    t.incl a
+
+  var t2: HashSet[string]
+  template myadd(a)=
+    result.add a
+    doAssert a notin t2, a
+    t2.incl a
+
+  # don't ignore these even though in lib/system
+  const goodSystem = """
+lib/system/io.nim
+lib/system/nimscript.nim
+lib/system/assertions.nim
+lib/system/iterators.nim
+lib/system/dollars.nim
+lib/system/widestrs.nim
+""".splitWhitespace()
+
+  for a in walkDirRec("lib"):
+    if a.splitFile.ext != ".nim": continue
+    if a.isRelativeTo("lib/pure/includes"): continue
+    if a.isRelativeTo("lib/genode"): continue
+    if a.isRelativeTo("lib/deprecated"):
+      if a notin @["lib/deprecated/pure/ospaths.nim"]: # REMOVE
+        continue
+    if a.isRelativeTo("lib/system"):
+      if a notin goodSystem:
+        continue
+    if a notin t:
+      result.add a
+      doAssert a notin t2, a
+      t2.incl a
+
+  myadd "nimsuggest/sexp.nim"
+  # these are include files, even though some of them don't specify `included from ...`
+  const ignore = """
+compiler/ccgcalls.nim
+compiler/ccgexprs.nim
+compiler/ccgliterals.nim
+compiler/ccgstmts.nim
+compiler/ccgthreadvars.nim
+compiler/ccgtrav.nim
+compiler/ccgtypes.nim
+compiler/jstypes.nim
+compiler/semcall.nim
+compiler/semexprs.nim
+compiler/semfields.nim
+compiler/semgnrc.nim
+compiler/seminst.nim
+compiler/semmagic.nim
+compiler/semobjconstr.nim
+compiler/semstmts.nim
+compiler/semtempl.nim
+compiler/semtypes.nim
+compiler/sizealignoffsetimpl.nim
+compiler/suggest.nim
+compiler/packagehandling.nim
+compiler/hlo.nim
+compiler/rodimpl.nim
+compiler/vmops.nim
+compiler/vmhooks.nim
+""".splitWhitespace()
+
+  # not include files but doesn't work; not included/imported anywhere; dead code?
+  const bad = """
+compiler/debuginfo.nim
+compiler/canonicalizer.nim
+compiler/forloops.nim
+""".splitWhitespace()
+
+  # these cause errors even though they're imported (some of which are mysterious)
+  const bad2 = """
+compiler/closureiters.nim
+compiler/tccgen.nim
+compiler/lambdalifting.nim
+compiler/layouter.nim
+compiler/evalffi.nim
+compiler/nimfix/nimfix.nim
+compiler/plugins/active.nim
+compiler/plugins/itersgen.nim
+""".splitWhitespace()
+
+  for a in walkDirRec("compiler"):
+    if a.splitFile.ext != ".nim": continue
+    if a in ignore: continue
+    if a in bad: continue
+    if a in bad2: continue
+    result.add a
+
+let doc = getDocList()
+
 proc sexec(cmds: openArray[string]) =
   ## Serial queue wrapper around exec.
   for cmd in cmds: