fix #6583, fix #14376, index+search now generated for all projects, many bug fixes with nim doc (#14324)

* refs #6583 fix nim doc output
* changelog
* change default for outDir when unspecified
* cleanups
* --project implies --index

6 files changed, 139 insertions, 107 deletions

diff --git a/compiler/commands.nim b/compiler/commands.nim
index e2e1a4558..ce4f7cc0f 100644
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -440,7 +440,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectArg(conf, switch, arg, pass, info)
     conf.docSeeSrcUrl = arg
   of "docroot":
-    conf.docRoot = if arg.len == 0: "@default" else: arg
+    conf.docRoot = if arg.len == 0: docRootDefault else: arg
   of "backend", "b":
     let backend = parseEnum(arg.normalize, TBackend.default)
     if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
@@ -485,7 +485,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "forcebuild", "f":
     processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
   of "project":
-    processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
+    processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info)
   of "gc":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index a67dc48ec..bd967d403 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -17,11 +17,10 @@ import
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, cgi, trees, types,
   typesrenderer, astalgo, lineinfos, intsets,
-  pathutils, trees, tables
+  pathutils, trees, tables, nimpaths
 
 const
   exportSection = skField
-  htmldocsDir = RelativeDir"htmldocs"
   docCmdSkip = "skip"
 
 type
@@ -51,7 +50,7 @@ type
     destFile*: AbsoluteFile
     thisDir*: AbsoluteDir
     exampleGroups: OrderedTable[string, ExampleGroup]
-    wroteCss*: bool
+    wroteSupportFiles*: bool
 
   PDoc* = ref TDocumentor ## Alias to type less.
 
@@ -72,7 +71,7 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
   proc nimbleDir(): AbsoluteDir =
     getNimbleFile(conf, file2).parentDir.AbsoluteDir
   case conf.docRoot:
-  of "@default": # using `@` instead of `$` to avoid shell quoting complications
+  of docRootDefault:
     result = getRelativePathFromConfigPath(conf, file)
     let dir = nimbleDir()
     if not dir.isEmpty:
@@ -88,9 +87,12 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
     result = getRelativePathFromConfigPath(conf, file)
     if result.isEmpty: bail()
   elif conf.docRoot.len > 0:
-    doAssert conf.docRoot.isAbsolute, conf.docRoot # or globalError
-    doAssert conf.docRoot.existsDir, conf.docRoot
-    result = relativeTo(file, conf.docRoot.AbsoluteDir)
+    # we're (currently) requiring `isAbsolute` to avoid confusion when passing
+    # a relative path (would it be relative wrt $PWD or to projectfile)
+    conf.globalAssert conf.docRoot.isAbsolute, arg=conf.docRoot
+    conf.globalAssert conf.docRoot.existsDir, arg=conf.docRoot
+    # needed because `canonicalizePath` called on `file`
+    result = file.relativeTo conf.docRoot.expandFilename.AbsoluteDir
   else:
     bail()
   if isAbsolute(result.string):
@@ -1125,11 +1127,8 @@ proc genSection(d: PDoc, kind: TSymKind) =
       "sectionid", "sectionTitle", "sectionTitleID", "content"], [
       ord(kind).rope, title, rope(ord(kind) + 50), d.toc[kind]])
 
-const nimdocOutCss = "nimdoc.out.css"
-  # `out` to make it easier to use with gitignore in user's repos
-
-proc cssHref(outDir: AbsoluteDir, destFile: AbsoluteFile): Rope =
-  rope($relativeTo(outDir / nimdocOutCss.RelativeFile, destFile.splitFile().dir, '/'))
+proc relLink(outDir: AbsoluteDir, destFile: AbsoluteFile, linkto: RelativeFile): Rope =
+  rope($relativeTo(outDir / linkto, destFile.splitFile().dir, '/'))
 
 proc genOutFile(d: PDoc): Rope =
   var
@@ -1160,15 +1159,17 @@ proc genOutFile(d: PDoc): Rope =
                  elif d.hasToc: "doc.body_toc"
                  else: "doc.body_no_toc"
   content = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, bodyname), ["title",
-      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg"],
+      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg", "theindexhref"],
       [title.rope, toc, d.modDesc, rope(getDateStr()),
-      rope(getClockStr()), code, d.modDeprecationMsg])
+      rope(getClockStr()), code, d.modDeprecationMsg, relLink(d.conf.outDir, d.destFile, theindexFname.RelativeFile)])
   if optCompileOnly notin d.conf.globalOptions:
     # XXX what is this hack doing here? 'optCompileOnly' means raw output!?
     code = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.file"), [
-        "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
+        "nimdoccss", "dochackjs",  "title", "tableofcontents", "moduledesc", "date", "time",
         "content", "author", "version", "analytics", "deprecationMsg"],
-        [cssHref(d.conf.outDir, d.destFile), title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
+        [relLink(d.conf.outDir, d.destFile, nimdocOutCss.RelativeFile),
+        relLink(d.conf.outDir, d.destFile, docHackJsFname.RelativeFile),
+        title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
         content, d.meta[metaAuthor].rope, d.meta[metaVersion].rope, d.analytics.rope, d.modDeprecationMsg])
   else:
     code = content
@@ -1203,11 +1204,13 @@ proc writeOutput*(d: PDoc, useWarning = false) =
     if not writeRope(content, outfile):
       rawMessage(d.conf, if useWarning: warnCannotOpenFile else: errCannotOpenFile,
         outfile.string)
-    elif not d.wroteCss:
-      let cssSource = $d.conf.getPrefixDir() / "doc" / "nimdoc.css"
-      let cssDest = $dir / nimdocOutCss
-      copyFile(cssSource, cssDest)
-      d.wroteCss = true
+    elif not d.wroteSupportFiles: # nimdoc.css + dochack.js
+      let nimr = $d.conf.getPrefixDir()
+      copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss)
+      if optGenIndex in d.conf.globalOptions:
+        let docHackJs2 = getDocHacksJs(nimr, nim = getAppFilename())
+        copyFile(docHackJs2, $d.conf.outDir / docHackJs2.lastPathPart)
+      d.wroteSupportFiles = true
 
 proc writeOutputJson*(d: PDoc, useWarning = false) =
   runAllExamples(d)
@@ -1234,6 +1237,8 @@ proc writeOutputJson*(d: PDoc, useWarning = false) =
 proc handleDocOutputOptions*(conf: ConfigRef) =
   if optWholeProject in conf.globalOptions:
     # Backward compatibility with previous versions
+    # xxx this is buggy when user provides `nim doc --project -o:sub/bar.html main`,
+    # it'd write to `sub/bar.html/main.html`
     conf.outDir = AbsoluteDir(conf.outDir / conf.outFile)
 
 proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
@@ -1308,20 +1313,23 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) =
     if not writeRope(content, filename):
       rawMessage(conf, errCannotOpenFile, filename.string)
 
-proc commandBuildIndex*(cache: IdentCache, conf: ConfigRef) =
-  var content = mergeIndexes(conf.projectFull.string).rope
+proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") =
+  var content = mergeIndexes(dir).rope
 
-  var outFile = RelativeFile"theindex"
-  if conf.outFile != RelativeFile"":
-    outFile = conf.outFile
+  var outFile = outFile
+  if outFile.isEmpty: outFile = theindexFname.RelativeFile.changeFileExt("")
   let filename = getOutFile(conf, outFile, HtmlExt)
 
   let code = ropeFormatNamedVars(conf, getConfigVar(conf, "doc.file"), [
-      "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
+      "nimdoccss", "dochackjs",
+      "title", "tableofcontents", "moduledesc", "date", "time",
       "content", "author", "version", "analytics"],
-      [cssHref(conf.outDir, filename), rope"Index", nil, nil, rope(getDateStr()),
+      [relLink(conf.outDir, filename, nimdocOutCss.RelativeFile),
+      relLink(conf.outDir, filename, docHackJsFname.RelativeFile),
+      rope"Index", nil, nil, rope(getDateStr()),
       rope(getClockStr()), content, nil, nil, nil])
   # no analytics because context is not available
 
   if not writeRope(code, filename):
     rawMessage(conf, errCannotOpenFile, filename.string)
+
diff --git a/compiler/main.nim b/compiler/main.nim
index 88f1890de..55b2f2899 100644
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -66,16 +66,8 @@ when not defined(leanCompiler):
     compileProject(graph)
     finishDoc2Pass(graph.config.projectName)
 
-proc setOutDir(conf: ConfigRef) =
-  if conf.outDir.isEmpty:
-    if optUseNimcache in conf.globalOptions:
-      conf.outDir = getNimcacheDir(conf)
-    else:
-      conf.outDir = conf.projectPath
-
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
-  setOutDir(conf)
   if conf.outFile.isEmpty:
     let base = conf.projectName
     let targetName = if optGenDynLib in conf.globalOptions:
@@ -121,7 +113,6 @@ proc commandCompileToJS(graph: ModuleGraph) =
     let conf = graph.config
     conf.exc = excCpp
 
-    setOutDir(conf)
     if conf.outFile.isEmpty:
       conf.outFile = RelativeFile(conf.projectName & ".js")
 
@@ -191,11 +182,6 @@ proc mainCommand*(graph: ModuleGraph) =
   conf.searchPaths.add(conf.libpath)
   setId(100)
 
-  ## Calling `setOutDir(conf)` unconditionally would fix regression
-  ## https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
-  when false: setOutDir(conf)
-  if optUseNimcache in conf.globalOptions: setOutDir(conf)
-
   proc customizeForBackend(backend: TBackend) =
     ## Sets backend specific options but don't compile to backend yet in
     ## case command doesn't require it. This must be called by all commands.
@@ -234,15 +220,40 @@ proc mainCommand*(graph: ModuleGraph) =
     of backendJs: commandCompileToJS(graph)
     of backendInvalid: doAssert false
 
+  template docLikeCmd(body) =
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      wantMainModule(conf)
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
+      defineSymbol(conf.symbols, "nimdoc")
+      body
+
+  block: ## command prepass
+    var docLikeCmd2 = false # includes what calls `docLikeCmd` + some more
+    case conf.command.normalize
+    of "r": conf.globalOptions.incl {optRun, optUseNimcache}
+    of "doc0",  "doc2", "doc", "rst2html", "rst2tex", "jsondoc0", "jsondoc2",
+      "jsondoc", "ctags", "buildindex": docLikeCmd2 = true
+    else: discard
+    if conf.outDir.isEmpty:
+      # doc like commands can generate a lot of files (especially with --project)
+      # so by default should not end up in $PWD nor in $projectPath.
+      conf.outDir = block:
+        var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
+        else: conf.projectPath
+        doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
+        if docLikeCmd2: ret = ret / htmldocsDir
+        ret
+
   ## process all backend commands
   case conf.command.normalize
   of "c", "cc", "compile", "compiletoc": compileToBackend(backendC) # compile means compileToC currently
   of "cpp", "compiletocpp": compileToBackend(backendCpp)
   of "objc", "compiletooc": compileToBackend(backendObjc)
   of "js", "compiletojs": compileToBackend(backendJs)
-  of "r": # different from `"run"`!
-    conf.globalOptions.incl {optRun, optUseNimcache}
-    compileToBackend(backendC)
+  of "r": compileToBackend(backendC) # different from `"run"`!
   of "run":
     when hasTinyCBackend:
       extccomp.setCC(conf, "tcc", unknownLineInfo)
@@ -254,29 +265,19 @@ proc mainCommand*(graph: ModuleGraph) =
   else: customizeForBackend(backendC) # fallback for other commands
 
   ## process all other commands
-  case conf.command.normalize
-  of "doc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandDoc(cache, conf)
+  case conf.command.normalize # synchronize with `cmdUsingHtmlDocs`
+  of "doc0": docLikeCmd commandDoc(cache, conf)
   of "doc2", "doc":
-    conf.setNoteDefaults(warnLockLevel, false) # issue #13218
-    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
-      # because currently generates lots of false positives due to conflation
-      # of labels links in doc comments, eg for random.rand:
-      #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
-      #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
+    docLikeCmd():
+      conf.setNoteDefaults(warnLockLevel, false) # issue #13218
+      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+        # because currently generates lots of false positives due to conflation
+        # of labels links in doc comments, eg for random.rand:
+        #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
+        #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
       commandDoc2(graph, false)
+      if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
+        commandBuildIndex(conf, $conf.outDir)
   of "rst2html":
     conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
     when defined(leanCompiler):
@@ -292,41 +293,10 @@ proc mainCommand*(graph: ModuleGraph) =
       conf.cmd = cmdRst2tex
       loadConfigs(DocTexConfig, cache, conf)
       commandRst2TeX(cache, conf)
-  of "jsondoc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandJson(cache, conf)
-  of "jsondoc2", "jsondoc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandDoc2(graph, true)
-  of "ctags":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandTags(cache, conf)
-  of "buildindex":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandBuildIndex(cache, conf)
+  of "jsondoc0": docLikeCmd commandJson(cache, conf)
+  of "jsondoc2", "jsondoc": docLikeCmd commandDoc2(graph, true)
+  of "ctags": docLikeCmd commandTags(cache, conf)
+  of "buildindex": docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
   of "gendepend":
     conf.cmd = cmdGenDepend
     commandGenDepend(graph)
diff --git a/compiler/msgs.nim b/compiler/msgs.nim
index 6bf4d82d2..a26aa0a3c 100644
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -510,7 +510,7 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
         styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg)
         if conf.hasHint(hintSource) and info != unknownLineInfo:
           conf.writeSurroundingSrc(info)
-        if conf.hasHint(hintMsgOrigin):
+        if hintMsgOrigin in conf.mainPackageNotes:
           styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle,
             " compiler msg initiated here", KindColor,
             KindFormat % hintMsgOrigin.msgToStr,
@@ -529,6 +529,14 @@ template fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
   conf.m.errorOutputs = {eStdOut, eStdErr}
   liMessage(conf, info, msg, arg, doAbort, instLoc())
 
+template globalAssert*(conf: ConfigRef; cond: untyped, info: TLineInfo = unknownLineInfo, arg = "") =
+  ## avoids boilerplate
+  if not cond:
+    const info2 = instantiationInfo(-1, fullPaths = true)
+    var arg2 = "'$1' failed" % [astToStr(cond)]
+    if arg.len > 0: arg2.add "; " & astToStr(arg) & ": " & arg
+    liMessage(conf, info, errGenerated, arg2, doRaise, info2)
+
 template globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
   liMessage(conf, info, msg, arg, doRaise, instLoc())
 
diff --git a/compiler/nimpaths.nim b/compiler/nimpaths.nim
new file mode 100644
index 000000000..2d7fa53cb
--- /dev/null
+++ b/compiler/nimpaths.nim
@@ -0,0 +1,44 @@
+##[
+Represents absolute paths, but using a symbolic variables (eg $nimr) which can be
+resolved at runtime; this avoids hardcoding at compile time absolute paths so
+that the project root can be relocated.
+
+xxx consider some refactoring with $nim/testament/lib/stdtest/specialpaths.nim;
+specialpaths is simpler because it doesn't need variables to be relocatable at
+runtime (eg for use in testament)
+
+interpolation variables:
+  $nimr: such that `$nimr/lib/system.nim` exists (avoids confusion with $nim binary)
+         in compiler, it's obtainable via getPrefixDir(); for other tools (eg koch),
+        this could be getCurrentDir() or getAppFilename().parentDir.parentDir,
+        depending on use case
+
+Unstable API
+]##
+
+import std/[os,strutils]
+
+const
+  docCss* = "$nimr/doc/nimdoc.css"
+  docHackNim* = "$nimr/tools/dochack/dochack.nim"
+  docHackJs* = docHackNim.changeFileExt("js")
+  docHackJsFname* = docHackJs.lastPathPart
+  theindexFname* = "theindex.html"
+  nimdocOutCss* = "nimdoc.out.css"
+    # `out` to make it easier to use with gitignore in user's repos
+  htmldocsDirname* = "htmldocs"
+
+proc interp*(path: string, nimr: string): string =
+  result = path % ["nimr", nimr]
+  doAssert '$' notin result, $(path, nimr, result) # avoids un-interpolated variables in output
+
+proc getDocHacksJs*(nimr: string, nim = getCurrentCompilerExe(), forceRebuild = false): string =
+  ## return absolute path to dochhack.js, rebuilding if it doesn't exist or if
+  ## `forceRebuild`.
+  let docHackJs2 = docHackJs.interp(nimr = nimr)
+  if forceRebuild or not docHackJs2.fileExists:
+    let cmd =  "$nim js $file" % ["nim", nim.quoteShell, "file", docHackNim.interp(nimr = nimr).quoteShell]
+    echo "getDocHacksJs: cmd: " & cmd
+    doAssert execShellCmd(cmd) == 0, $(cmd)
+  doAssert docHackJs2.fileExists
+  result = docHackJs2
diff --git a/compiler/options.nim b/compiler/options.nim
index 1418fff63..d990f2fd4 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -9,7 +9,7 @@
 
 import
   os, strutils, strtabs, sets, lineinfos, platform,
-  prefixmatches, pathutils
+  prefixmatches, pathutils, nimpaths
 
 from terminal import isatty
 from times import utc, fromUnix, local, getTime, format, DateTime
@@ -518,8 +518,9 @@ const
   DefaultConfigNims* = RelativeFile"config.nims"
   DocConfig* = RelativeFile"nimdoc.cfg"
   DocTexConfig* = RelativeFile"nimdoc.tex.cfg"
-
-const oKeepVariableNames* = true
+  htmldocsDir* = htmldocsDirname.RelativeDir
+  docRootDefault* = "@default" # using `@` instead of `$` to avoid shell quoting complications
+  oKeepVariableNames* = true
 
 proc mainCommandArg*(conf: ConfigRef): string =
   ## This is intended for commands like check or parse
@@ -543,6 +544,7 @@ proc getOutFile*(conf: ConfigRef; filename: RelativeFile, ext: string): Absolute
   # explains regression https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
   # Yet another reason why "" should not mean ".";  `""/something` should raise
   # instead of implying "" == "." as it's bug prone.
+  doAssert conf.outDir.string.len > 0
   result = conf.outDir / changeFileExt(filename, ext)
 
 proc absOutFile*(conf: ConfigRef): AbsoluteFile =