IC: first steps towards 'nim check --def --ic:on' (#17714)

* IC: first steps towards 'nim check --def --ic:on'
* IC navigator: deduplicate output lines
* IC navigator: progress
* IC navigator: use a different nimcache entry
* IC navigator: special logic for templates/macros
* IC navigator: proper error messages
* IC navigator: prepare for testing code; document only what currently works somewhat

11 files changed, 211 insertions, 12 deletions

diff --git a/compiler/commands.nim b/compiler/commands.nim
index f36a4f515..e918979fb 100644
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -859,6 +859,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "usages":
     expectNoArg(conf, switch, arg, pass, info)
     conf.ideCmd = ideUse
+  of "defusages":
+    expectNoArg(conf, switch, arg, pass, info)
+    conf.ideCmd = ideDus
   of "stdout":
     processOnOffSwitchG(conf, {optStdout}, arg, pass, info)
   of "listfullpaths":
diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim
index e882b7a85..4613b9cb8 100644
--- a/compiler/ic/ic.nim
+++ b/compiler/ic/ic.nim
@@ -89,14 +89,27 @@ proc rememberConfig(c: var PackedEncoder; m: var PackedModule; config: ConfigRef
   #primConfigFields rem
   m.cfg = pc
 
+const
+  debugConfigDiff = defined(debugConfigDiff)
+
+when debugConfigDiff:
+  import std / [hashes, tables, intsets, sha1, strutils, sets]
+
 proc configIdentical(m: PackedModule; config: ConfigRef): bool =
   result = m.definedSymbols == definedSymbolsAsString(config)
-  #if not result:
-  #  echo "A ", m.definedSymbols, " ", definedSymbolsAsString(config)
+  when debugConfigDiff:
+    if not result:
+      var wordsA = m.definedSymbols.split(Whitespace).toHashSet()
+      var wordsB = definedSymbolsAsString(config).split(Whitespace).toHashSet()
+      for c in wordsA - wordsB:
+        echo "in A but not in B ", c
+      for c in wordsB - wordsA:
+        echo "in B but not in A ", c
   template eq(x) =
     result = result and m.cfg.x == config.x
-    #if not result:
-    #  echo "B ", m.cfg.x, " ", config.x
+    when debugConfigDiff:
+      if m.cfg.x != config.x:
+        echo "B ", m.cfg.x, " ", config.x
   primConfigFields eq
 
 proc rememberStartupConfig*(dest: var PackedConfig, config: ConfigRef) =
@@ -124,7 +137,7 @@ proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId =
       c.filenames[x] = result
     c.lastFile = x
     c.lastLit = result
-    assert result != LitId(0)
+  assert result != LitId(0)
 
 proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex =
   result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x])
@@ -144,6 +157,8 @@ proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; co
   m.bodies = newTreeFrom(m.topLevel)
   m.toReplay = newTreeFrom(m.topLevel)
 
+  c.lastFile = FileIndex(-10)
+
   let thisNimFile = FileIndex c.thisModule
   var h = msgs.getHash(config, thisNimFile)
   if h.len == 0:
@@ -464,6 +479,9 @@ proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i:
                                           concreteTypes: t)
   toPackedGeneratedProcDef(i.sym, c, m)
 
+proc storeExpansion*(c: var PackedEncoder; m: var PackedModule; info: TLineInfo; s: PSym) =
+  toPackedNode(newSymNode(s, info), m.bodies, c, m)
+
 proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) =
   case err
   of cannotOpen:
diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim
new file mode 100644
index 000000000..ad2b29f42
--- /dev/null
+++ b/compiler/ic/navigator.nim
@@ -0,0 +1,147 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2021 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Supports the "nim check --ic:on --def --track:FILE,LINE,COL"
+## IDE-like features. It uses the set of .rod files to accomplish
+## its task. The set must cover a complete Nim project.
+
+import std / sets
+
+from os import nil
+from std/private/miscdollars import toLocation
+
+import ".." / [ast, modulegraphs, msgs, options]
+import packed_ast, bitabs, ic
+
+type
+  NavContext = object
+    g: ModuleGraph
+    thisModule: int32
+    trackPos: PackedLineInfo
+    alreadyEmitted: HashSet[string]
+    outputSep: char # for easier testing, use short filenames and spaces instead of tabs.
+
+proc isTracked(current, trackPos: PackedLineInfo, tokenLen: int): bool =
+  if current.file == trackPos.file and current.line == trackPos.line:
+    let col = trackPos.col
+    if col >= current.col and col < current.col+tokenLen:
+      return true
+
+proc searchLocalSym(c: var NavContext; s: PackedSym; info: PackedLineInfo): bool =
+  result = s.name != LitId(0) and
+    isTracked(info, c.trackPos, c.g.packed[c.thisModule].fromDisk.sh.strings[s.name].len)
+
+proc searchForeignSym(c: var NavContext; s: ItemId; info: PackedLineInfo): bool =
+  let name = c.g.packed[s.module].fromDisk.sh.syms[s.item].name
+  result = name != LitId(0) and
+    isTracked(info, c.trackPos, c.g.packed[s.module].fromDisk.sh.strings[name].len)
+
+const
+  EmptyItemId = ItemId(module: -1'i32, item: -1'i32)
+
+proc search(c: var NavContext; tree: PackedTree): ItemId =
+  # We use the linear representation here directly:
+  for i in 0..high(tree.nodes):
+    case tree.nodes[i].kind
+    of nkSym:
+      let item = tree.nodes[i].operand
+      if searchLocalSym(c, c.g.packed[c.thisModule].fromDisk.sh.syms[item], tree.nodes[i].info):
+        return ItemId(module: c.thisModule, item: item)
+    of nkModuleRef:
+      if tree.nodes[i].info.line == c.trackPos.line and tree.nodes[i].info.file == c.trackPos.file:
+        let (n1, n2) = sons2(tree, NodePos i)
+        assert n1.kind == nkInt32Lit
+        assert n2.kind == nkInt32Lit
+        let pId = PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)
+        let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config)
+        if searchForeignSym(c, itemId, tree.nodes[i].info):
+          return itemId
+    else: discard
+  return EmptyItemId
+
+proc isDecl(tree: PackedTree; n: NodePos): bool =
+  # XXX This is not correct yet.
+  const declarativeNodes = procDefs + {nkMacroDef, nkTemplateDef,
+    nkLetSection, nkVarSection, nkUsingStmt, nkConstSection, nkTypeSection,
+    nkIdentDefs, nkEnumTy, nkVarTuple}
+  result = n.int >= 0 and tree[n.int].kind in declarativeNodes
+
+proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) =
+  var m = ""
+  var file = c.g.packed[c.thisModule].fromDisk.sh.strings[info.file]
+  if c.outputSep == ' ':
+    file = os.extractFilename file
+  toLocation(m, file, info.line.int, info.col.int + ColOffset)
+  if not c.alreadyEmitted.containsOrIncl(m):
+    echo (if isDecl: "def" else: "usage"), c.outputSep, m
+
+proc list(c: var NavContext; tree: PackedTree; sym: ItemId) =
+  for i in 0..high(tree.nodes):
+    case tree.nodes[i].kind
+    of nkSym:
+      let item = tree.nodes[i].operand
+      if sym.item == item and sym.module == c.thisModule:
+        usage(c, tree.nodes[i].info, isDecl(tree, parent(NodePos i)))
+    of nkModuleRef:
+      let (n1, n2) = sons2(tree, NodePos i)
+      assert n1.kind == nkInt32Lit
+      assert n2.kind == nkInt32Lit
+      let pId = PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)
+      let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config)
+      if itemId.item == sym.item and sym.module == itemId.module:
+        usage(c, tree.nodes[i].info, isDecl(tree, parent(NodePos i)))
+    else: discard
+
+proc nav(g: ModuleGraph) =
+  # translate the track position to a packed position:
+  let unpacked = g.config.m.trackPos
+  let mid = unpacked.fileIndex
+  let fileId = g.packed[int32 mid].fromDisk.sh.strings.getKeyId(toFullPath(g.config, mid))
+
+  if fileId == LitId(0):
+    internalError(g.config, unpacked, "cannot find a valid file ID")
+    return
+
+  var c = NavContext(
+    g: g,
+    thisModule: int32 mid,
+    trackPos: PackedLineInfo(line: unpacked.line, col: unpacked.col, file: fileId),
+    outputSep: if isDefined(g.config, "nimIcNavigatorTests"): ' ' else: '\t'
+  )
+  var symId = search(c, g.packed[int32 mid].fromDisk.topLevel)
+  if symId == EmptyItemId:
+    symId = search(c, g.packed[int32 mid].fromDisk.bodies)
+
+  if symId == EmptyItemId:
+    localError(g.config, unpacked, "no symbol at this position")
+    return
+
+  for i in 0..high(g.packed):
+    # case statement here to enforce exhaustive checks.
+    case g.packed[i].status
+    of undefined:
+      discard "nothing to do"
+    of loading:
+      assert false, "cannot check integrity: Module still loading"
+    of stored, storing, outdated, loaded:
+      c.thisModule = int32 i
+      list(c, g.packed[i].fromDisk.topLevel, symId)
+      list(c, g.packed[i].fromDisk.bodies, symId)
+
+proc navDefinition*(g: ModuleGraph) = nav(g)
+proc navUsages*(g: ModuleGraph) = nav(g)
+proc navDefusages*(g: ModuleGraph) = nav(g)
+
+proc writeRodFiles*(g: ModuleGraph) =
+  for i in 0..high(g.packed):
+    case g.packed[i].status
+    of undefined, loading, stored, loaded:
+      discard "nothing to do"
+    of storing, outdated:
+      closeRodFile(g, g.packed[i].module)
diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim
index 1075664cb..5cc3b1476 100644
--- a/compiler/ic/packed_ast.nim
+++ b/compiler/ic/packed_ast.nim
@@ -258,9 +258,9 @@ iterator sonsWithoutLast2*(tree: PackedTree; n: NodePos): NodePos =
 proc parentImpl(tree: PackedTree; n: NodePos): NodePos =
   # finding the parent of a node is rather easy:
   var pos = n.int - 1
-  while pos >= 0 and isAtom(tree, pos) or (pos + tree.nodes[pos].operand - 1 < n.int):
+  while pos >= 0 and (isAtom(tree, pos) or (pos + tree.nodes[pos].operand - 1 < n.int)):
     dec pos
-  assert pos >= 0, "node has no parent"
+  #assert pos >= 0, "node has no parent"
   result = NodePos(pos)
 
 template parent*(n: NodePos): NodePos = parentImpl(tree, n)
diff --git a/compiler/main.nim b/compiler/main.nim
index 3c5018546..475ba7773 100644
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -21,7 +21,7 @@ import
   modules,
   modulegraphs, tables, lineinfos, pathutils, vmprofiler
 
-import ic / [cbackend, integrity]
+import ic / [cbackend, integrity, navigator]
 from ic / ic import rodViewer
 
 when not defined(leanCompiler):
@@ -54,11 +54,20 @@ proc commandGenDepend(graph: ModuleGraph) =
       ' ' & changeFileExt(project, "dot").string)
 
 proc commandCheck(graph: ModuleGraph) =
-  graph.config.setErrorMaxHighMaybe
-  defineSymbol(graph.config.symbols, "nimcheck")
+  let conf = graph.config
+  conf.setErrorMaxHighMaybe
+  defineSymbol(conf.symbols, "nimcheck")
   semanticPasses(graph)  # use an empty backend for semantic checking only
   compileProject(graph)
 
+  if conf.symbolFiles != disabledSf:
+    case conf.ideCmd
+    of ideDef: navDefinition(graph)
+    of ideUse: navUsages(graph)
+    of ideDus: navDefusages(graph)
+    else: discard
+    writeRodFiles(graph)
+
 when not defined(leanCompiler):
   proc commandDoc2(graph: ModuleGraph; json: bool) =
     handleDocOutputOptions graph.config
diff --git a/compiler/msgs.nim b/compiler/msgs.nim
index 24855ae18..452b1cd89 100644
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -418,7 +418,9 @@ proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string)
     inc(conf.errorCounter)
     conf.exitcode = 1'i8
     if conf.errorCounter >= conf.errorMax:
-      quit(conf, msg)
+      # only really quit when we're not in the new 'nim check --def' mode:
+      if conf.ideCmd == ideNone:
+        quit(conf, msg)
     elif eh == doAbort and conf.cmd != cmdIdeTools:
       quit(conf, msg)
     elif eh == doRaise:
diff --git a/compiler/options.nim b/compiler/options.nim
index 7408ce675..9eeffb4fb 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -685,6 +685,11 @@ proc getOsCacheDir(): string =
     result = getHomeDir() / genSubDir.string
 
 proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir =
+  proc nimcacheSuffix(conf: ConfigRef): string =
+    if conf.cmd == cmdCheck: "_check"
+    elif isDefined(conf, "release") or isDefined(conf, "danger"): "_r"
+    else: "_d"
+
   # XXX projectName should always be without a file extension!
   result = if not conf.nimcacheDir.isEmpty:
              conf.nimcacheDir
@@ -692,7 +697,7 @@ proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir =
              conf.projectPath / genSubDir
            else:
             AbsoluteDir(getOsCacheDir() / splitFile(conf.projectName).name &
-               (if isDefined(conf, "release") or isDefined(conf, "danger"): "_r" else: "_d"))
+               nimcacheSuffix(conf))
 
 proc pathSubs*(conf: ConfigRef; p, config: string): string =
   let home = removeTrailingDirSep(os.getHomeDir())
diff --git a/compiler/sem.nim b/compiler/sem.nim
index 6b3f0c80d..ccc634b4d 100644
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -452,6 +452,7 @@ const
 
 proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
                   flags: TExprFlags = {}): PNode =
+  rememberExpansion(c, nOrig.info, sym)
   pushInfoContext(c.config, nOrig.info, sym.detailedInfo)
 
   let info = getCallLineInfo(n)
diff --git a/compiler/semdata.nim b/compiler/semdata.nim
index daf1265e8..6b0dcb0e2 100644
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -570,3 +570,13 @@ proc sealRodFile*(c: PContext) =
         if m == c.module:
           addPragmaComputation(c, n)
     c.idgen.sealed = true # no further additions are allowed
+
+proc rememberExpansion*(c: PContext; info: TLineInfo; expandedSym: PSym) =
+  ## Templates and macros are very special in Nim; these have
+  ## inlining semantics so after semantic checking they leave no trace
+  ## in the sem'checked AST. This is very bad for IDE-like tooling
+  ## ("find all usages of this template" would not work). We need special
+  ## logic to remember macro/template expansions. This is done here and
+  ## delegated to the "rod" file mechanism.
+  if c.config.symbolFiles != disabledSf:
+    storeExpansion(c.encoder, c.packedRepr, info, expandedSym)
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index 256c008c9..cbd4318f7 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -27,6 +27,7 @@ const
 
 proc semTemplateExpr(c: PContext, n: PNode, s: PSym,
                      flags: TExprFlags = {}): PNode =
+  rememberExpansion(c, n.info, s)
   let info = getCallLineInfo(n)
   markUsed(c, info, s)
   onUse(info, s)
diff --git a/doc/advopt.txt b/doc/advopt.txt
index 4842d090e..c7b63d50f 100644
--- a/doc/advopt.txt
+++ b/doc/advopt.txt
@@ -15,6 +15,7 @@ Advanced commands:
   //dump                    dump all defined conditionals and search paths
                             see also: --dump.format:json (useful with: `| jq`)
   //check                   checks the project for syntax and semantic
+                            (can be combined with --track and --defusages)
 
 Runtime checks (see -x):
   --objChecks:on|off        turn obj conversion checks on|off
@@ -27,6 +28,8 @@ Runtime checks (see -x):
   --infChecks:on|off        turn Inf checks on|off
 
 Advanced options:
+  --track:FILE,LINE,COL     set the tracking position for (--defusages)
+  --defusages               find the definition and all usages of a symbol
   -o:FILE, --out:FILE       set the output filename
   --outdir:DIR              set the path where the output file will be written
   --usenimcache             will use `outdir=$$nimcache`, whichever it resolves