IC: compilerprocs are handled correctly (#17265)

* IC: compilerprocs are handled correctly
* IC: special logic for .exportC'ed procs
* IC: 'hello world' compiles for the first round (2nd round fails)
* IC: hello world seems to work

5 files changed, 88 insertions, 23 deletions

diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim
index eda54c438..52a4a3339 100644
--- a/compiler/ic/cbackend.nim
+++ b/compiler/ic/cbackend.nim
@@ -83,6 +83,7 @@ proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool
 proc generateCode*(g: ModuleGraph) =
   ## The single entry point, generate C(++) code for the entire
   ## Nim program aka `ModuleGraph`.
+  initStrTable(g.compilerprocs)
   var alive = computeAliveSyms(g.packed, g.config)
 
   for i in 0..high(g.packed):
diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim
index d60c707aa..c7d66465d 100644
--- a/compiler/ic/dce.nim
+++ b/compiler/ic/dce.nim
@@ -47,14 +47,19 @@ proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: i
   ## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will
   ## be analysed.
   if not c.alive[module].containsOrIncl(item):
-    let body = g[module].fromDisk.sh.syms[item].ast
+    var body = g[module].fromDisk.sh.syms[item].ast
     if body != emptyNodeId:
       let opt = g[module].fromDisk.sh.syms[item].options
+      if g[module].fromDisk.sh.syms[item].kind in routineKinds:
+        body = NodeId ithSon(g[module].fromDisk.bodies, NodePos body, bodyPos)
       c.stack.add((module, opt, NodePos(body)))
 
     when false:
-      let name = g[module].fromDisk.sh.strings[g[module].fromDisk.sh.syms[item].name]
-      echo "I was called! ", name, " body exists: ", body != emptyNodeId
+      let nid = g[module].fromDisk.sh.syms[item].name
+      if nid != LitId(0):
+        let name = g[module].fromDisk.sh.strings[nid]
+        if name in ["nimFrame", "callDepthLimitReached"]:
+          echo "I was called! ", name, " body exists: ", body != emptyNodeId, " ", module, " ", item
 
 proc requestCompilerProc(c: var AliveContext; g: PackedModuleGraph; name: string) =
   let (module, item) = c.compilerProcs[name]
@@ -118,8 +123,8 @@ proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: N
     rangeCheckAnalysis(c, g, tree, n)
   of nkProcDef, nkConverterDef, nkMethodDef, nkLambda, nkDo, nkFuncDef:
     if n.firstSon.kind == nkSym and isNotGeneric(n):
-      if isExportedToC(c, g, n.firstSon.operand):
-        let item = n.operand
+      let item = n.firstSon.operand
+      if isExportedToC(c, g, item):
         # This symbol is alive and everything its body references.
         followLater(c, g, c.thisModule, item)
   else:
@@ -145,6 +150,7 @@ proc computeAliveSyms*(g: PackedModuleGraph; conf: ConfigRef): AliveSyms =
       c.thisModule = i
       for p in allNodes(g[i].fromDisk.topLevel):
         aliveCode(c, g, g[i].fromDisk.topLevel, p)
+
   followNow(c, g)
   result = move(c.alive)
 
diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim
index 86636f902..353cc3a42 100644
--- a/compiler/ic/packed_ast.nim
+++ b/compiler/ic/packed_ast.nim
@@ -464,3 +464,6 @@ iterator allNodes*(tree: PackedTree): NodePos =
     yield NodePos(p)
     let s = span(tree, p)
     inc p, s
+
+proc toPackedItemId*(item: int32): PackedItemId {.inline.} =
+  PackedItemId(module: LitId(0), item: item)
diff --git a/compiler/ic/replayer.nim b/compiler/ic/replayer.nim
index b0ea557e4..05c473090 100644
--- a/compiler/ic/replayer.nim
+++ b/compiler/ic/replayer.nim
@@ -128,9 +128,11 @@ proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
                             PackedItemId(module: LitId(0), item: it))
     methodDef(g, g.idgen, sym)
 
-  for it in mitems(g.packed[module].fromDisk.compilerProcs):
-    let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it[1]))
-    g.lazyCompilerprocs[g.packed[module].fromDisk.sh.strings[it[0]]] = symId
+  when false:
+    # not used anymore:
+    for it in mitems(g.packed[module].fromDisk.compilerProcs):
+      let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it[1]))
+      g.lazyCompilerprocs[g.packed[module].fromDisk.sh.strings[it[0]]] = symId
 
   for it in mitems(g.packed[module].fromDisk.converters):
     let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it))
diff --git a/compiler/ic/to_packed_ast.nim b/compiler/ic/to_packed_ast.nim
index 58b71e3a4..44902143d 100644
--- a/compiler/ic/to_packed_ast.nim
+++ b/compiler/ic/to_packed_ast.nim
@@ -11,7 +11,7 @@ import std / [hashes, tables, intsets, sha1]
 import packed_ast, bitabs, rodfiles
 import ".." / [ast, idents, lineinfos, msgs, ropes, options,
   pathutils, condsyms]
-
+#import ".." / [renderer, astalgo]
 from std / os import removeFile, isAbsolute
 
 type
@@ -57,6 +57,9 @@ type
     symMarker*: IntSet #Table[ItemId, SymId]    # ItemId.item -> SymId
     config*: ConfigRef
 
+proc isActive*(e: PackedEncoder): bool = e.config != nil
+proc disable*(e: var PackedEncoder) = e.config = nil
+
 template primConfigFields(fn: untyped) {.dirty.} =
   fn backend
   fn selectedGC
@@ -216,7 +219,8 @@ proc addMissing(c: var PackedEncoder; p: PSym) =
   ## consider queuing a symbol for later addition to the packed tree
   if p != nil and p.itemId.module == c.thisModule:
     if p.itemId.item notin c.symMarker:
-      c.pendingSyms.add p
+      if not (sfForward in p.flags and p.kind in routineKinds):
+        c.pendingSyms.add p
 
 proc addMissing(c: var PackedEncoder; p: PType) =
   ## consider queuing a type for later addition to the packed tree
@@ -332,6 +336,8 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
     if s.itemId.item >= m.sh.syms.len:
       setLen m.sh.syms, s.itemId.item+1
 
+    assert sfForward notin s.flags
+
     var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic,
       position: s.position, offset: s.offset, options: s.options,
       name: s.name.s.toLitId(m))
@@ -418,28 +424,40 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa
       toPackedNode(n[i], ir, c, m)
     ir.patch patchPos
 
-proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) =
-  var t = newSeq[PackedItemId](i.concreteTypes.len)
-  for j in 0..high(i.concreteTypes):
-    t[j] = storeTypeLater(i.concreteTypes[j], c, m)
-  m.procInstCache.add PackedInstantiation(key: storeSymLater(s, c, m),
-                                          sym: storeSymLater(i.sym, c, m),
-                                          concreteTypes: t)
-
 proc storeTypeInst*(c: var PackedEncoder; m: var PackedModule; s: PSym; inst: PType) =
   m.typeInstCache.add (storeSymLater(s, c, m), storeTypeLater(inst, c, m))
 
 proc addPragmaComputation*(c: var PackedEncoder; m: var PackedModule; n: PNode) =
   toPackedNode(n, m.toReplay, c, m)
 
+proc toPackedProcDef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
+  let info = toPackedInfo(n.info, c, m)
+  let patchPos = ir.prepare(n.kind, n.flags,
+                            storeTypeLater(n.typ, c, m), info)
+  for i in 0..<n.len:
+    if i != bodyPos:
+      toPackedNode(n[i], ir, c, m)
+    else:
+      # do not serialize the body of the proc, it's unnecessary since
+      # n[0].sym.ast has the sem'checked variant of it which is what
+      # everybody should use instead.
+      ir.nodes.add PackedNode(kind: nkEmpty, flags: {}, operand: 0,
+                              typeId: nilItemId, info: info)
+  ir.patch patchPos
+
 proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var PackedModule) =
   case n.kind
   of routineDefs:
-    # we serialize n[namePos].sym instead
-    if n[namePos].kind == nkSym:
-      discard storeSym(n[namePos].sym, encoder, m)
-    else:
-      toPackedNode(n, m.topLevel, encoder, m)
+    toPackedProcDef(n, m.topLevel, encoder, m)
+    when false:
+      # we serialize n[namePos].sym instead
+      if n[namePos].kind == nkSym:
+        let s = n[namePos].sym
+        discard storeSym(s, encoder, m)
+        if s.flags * {sfExportc, sfCompilerProc, sfCompileTime} == {sfExportc}:
+          m.exportCProcs.add(s.itemId.item)
+      else:
+        toPackedNode(n, m.topLevel, encoder, m)
   of nkStmtList, nkStmtListExpr:
     for it in n:
       toPackedNodeIgnoreProcDefs(it, encoder, m)
@@ -450,6 +468,24 @@ proc toPackedNodeTopLevel*(n: PNode, encoder: var PackedEncoder; m: var PackedMo
   toPackedNodeIgnoreProcDefs(n, encoder, m)
   flush encoder, m
 
+proc toPackedGeneratedProcDef*(s: PSym, encoder: var PackedEncoder; m: var PackedModule) =
+  ## Generic procs and generated `=hook`'s need explicit top-level entries so
+  ## that the code generator can work without having to special case these. These
+  ## entries will also be useful for other tools and are the cleanest design
+  ## I can come up with.
+  assert s.kind in routineKinds
+  toPackedProcDef(s.ast, m.topLevel, encoder, m)
+  #flush encoder, m
+
+proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) =
+  var t = newSeq[PackedItemId](i.concreteTypes.len)
+  for j in 0..high(i.concreteTypes):
+    t[j] = storeTypeLater(i.concreteTypes[j], c, m)
+  m.procInstCache.add PackedInstantiation(key: storeSymLater(s, c, m),
+                                          sym: storeSymLater(i.sym, c, m),
+                                          concreteTypes: t)
+  toPackedGeneratedProcDef(i.sym, c, m)
+
 proc loadError(err: RodFileError; filename: AbsoluteFile) =
   echo "Error: ", $err, " loading file: ", filename.string
 
@@ -520,6 +556,7 @@ proc storeError(err: RodFileError; filename: AbsoluteFile) =
   removeFile(filename.string)
 
 proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var PackedModule) =
+  flush encoder, m
   #rememberConfig(encoder, encoder.config)
 
   var f = rodfiles.create(filename.string)
@@ -572,6 +609,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeSeqSection enumToStringProcsSection, m.enumToStringProcs
 
   close(f)
+  encoder.disable()
   if f.err != ok:
     storeError(f.err, filename)
 
@@ -848,6 +886,8 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
                   info: newLineInfo(fileIdx, 1, 1),
                   position: int(fileIdx))
   m.module.owner = newPackage(conf, cache, fileIdx)
+  if fileIdx == conf.projectMainIdx2:
+    m.module.flags.incl sfMainModule
 
 proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                        fileIdx: FileIndex; m: var LoadedModule) =
@@ -895,6 +935,7 @@ proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache
       loadError(err, rod)
       g[m].status = outdated
       result = true
+    when false: loadError(err, rod)
   of loading, loaded:
     # For loading: Assume no recompile is required.
     result = false
@@ -1061,6 +1102,13 @@ proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator =
   IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.sh.syms.len,
               typeId: int32 m.fromDisk.sh.types.len)
 
+proc searchForCompilerproc*(m: LoadedModule; name: string): int32 =
+  # slow, linear search, but the results are cached:
+  for it in items(m.fromDisk.compilerProcs):
+    if m.fromDisk.sh.strings[it[0]] == name:
+      return it[1]
+  return -1
+
 # ------------------------- .rod file viewer ---------------------------------
 
 proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
@@ -1081,6 +1129,11 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
     for ex in m.reexports:
       echo "  ", m.sh.strings[ex[0]]
     #  reexports*: seq[(LitId, PackedItemId)]
+
+  echo "all symbols"
+  for i in 0..high(m.sh.syms):
+    echo "  ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i
+
   echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len,
     " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len,
     " strings: ", m.sh.strings.len, " integers: ", m.sh.integers.len,