IC: final implementation steps (#16801)

* removed dead code
* we need even more laziness for the generic caches
* make it bootstrap on older Nims
* wrote more deserialization code
* IC: replay required methods information

4 files changed, 127 insertions, 23 deletions

diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim
index 213b21b23..95cac4700 100644
--- a/compiler/ic/packed_ast.nim
+++ b/compiler/ic/packed_ast.nim
@@ -74,11 +74,9 @@ type
     flags*: TTypeFlags
     types*: seq[PackedItemId]
     n*: NodeId
-    methods*: seq[(int, PackedItemId)]
     #nodeflags*: TNodeFlags
     sym*: PackedItemId
     owner*: PackedItemId
-    attachedOps*: array[TTypeAttachedOp, PackedItemId]
     size*: BiggestInt
     align*: int16
     paddingAtEnd*: int16
@@ -111,6 +109,10 @@ type
     floats*: BiTable[BiggestFloat]
     #config*: ConfigRef
 
+  PackedInstantiation* = object
+    key*, sym*: PackedItemId
+    concreteTypes*: seq[PackedItemId]
+
 proc `==`*(a, b: SymId): bool {.borrow.}
 proc hash*(a: SymId): Hash {.borrow.}
 
diff --git a/compiler/ic/replayer.nim b/compiler/ic/replayer.nim
index b4dd05965..579fa8f1b 100644
--- a/compiler/ic/replayer.nim
+++ b/compiler/ic/replayer.nim
@@ -12,10 +12,12 @@
 ## support.
 
 import ".." / [ast, modulegraphs, trees, extccomp, btrees,
-  msgs, lineinfos, pathutils, options]
+  msgs, lineinfos, pathutils, options, cgmeth]
 
 import tables
 
+import packed_ast, to_packed_ast, bitabs
+
 proc replayStateChanges*(module: PSym; g: ModuleGraph) =
   let list = module.ast
   assert list != nil
@@ -84,6 +86,44 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph) =
       else:
         internalAssert g.config, false
 
-#  of nkMethodDef:
-#    methodDef(g, n[namePos].sym, fromCache=true)
+proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
+  ## We remember the generic instantiations a module performed
+  ## in order to to avoid the code bloat that generic code tends
+  ## to imply. This is cheaper than deduplication of identical
+  ## generic instantiations. However, deduplication is more
+  ## powerful and general and I hope to implement it soon too
+  ## (famous last words).
+  assert g.packed[module].status == loaded
+  for it in g.packed[module].fromDisk.typeInstCache:
+    let key = translateId(it[0], g.packed, module, g.config)
+    g.typeInstCache.mgetOrPut(key, @[]).add LazyType(id: FullId(module: module, packed: it[1]), typ: nil)
+
+  for it in mitems(g.packed[module].fromDisk.procInstCache):
+    let key = translateId(it.key, g.packed, module, g.config)
+    let sym = translateId(it.sym, g.packed, module, g.config)
+    var concreteTypes = newSeq[FullId](it.concreteTypes.len)
+    for i in 0..high(it.concreteTypes):
+      let tmp = translateId(it.concreteTypes[i], g.packed, module, g.config)
+      concreteTypes[i] = FullId(module: tmp.module, packed: it.concreteTypes[i])
+
+    g.procInstCache.mgetOrPut(key, @[]).add LazyInstantiation(
+      module: module, sym: FullId(module: sym.module, packed: it.sym),
+      concreteTypes: concreteTypes, inst: nil)
+
+  for it in mitems(g.packed[module].fromDisk.methodsPerType):
+    let key = translateId(it[0], g.packed, module, g.config)
+    let col = it[1]
+    let tmp = translateId(it[2], g.packed, module, g.config)
+    let symId = FullId(module: tmp.module, packed: it[2])
+    g.methodsPerType.mgetOrPut(key, @[]).add (col, LazySym(id: symId, sym: nil))
+
+  for it in mitems(g.packed[module].fromDisk.enumToStringProcs):
+    let key = translateId(it[0], g.packed, module, g.config)
+    let tmp = translateId(it[1], g.packed, module, g.config)
+    let symId = FullId(module: tmp.module, packed: it[1])
+    g.enumToStringProcs[key] = LazySym(id: symId, sym: nil)
 
+  for it in mitems(g.packed[module].fromDisk.methods):
+    let sym = loadSymFromId(g.config, g.cache, g.packed, module,
+                            PackedItemId(module: LitId(0), item: it))
+    methodDef(g, g.idgen, sym)
diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim
index 98399cede..fa0a7c734 100644
--- a/compiler/ic/rodfiles.nim
+++ b/compiler/ic/rodfiles.nim
@@ -31,6 +31,11 @@ type
     bodiesSection
     symsSection
     typesSection
+    typeInstCacheSection
+    procInstCacheSection
+    attachedOpsSection
+    methodsPerTypeSection
+    enumToStringProcsSection
     aliveSymsSection # beware, this is stored in a `.alivesyms` file.
 
   RodFileError* = enum
@@ -73,6 +78,12 @@ proc storePrim*[T](f: var RodFile; x: T) =
   elif T is tuple:
     for y in fields(x):
       storePrim(f, y)
+  elif T is object:
+    for y in fields(x):
+      when y is seq:
+        storeSeq(f, y)
+      else:
+        storePrim(f, y)
   else:
     {.error: "unsupported type for 'storePrim'".}
 
@@ -107,6 +118,12 @@ proc loadPrim*[T](f: var RodFile; x: var T) =
   elif T is tuple:
     for y in fields(x):
       loadPrim(f, y)
+  elif T is object:
+    for y in fields(x):
+      when y is seq:
+        loadSeq(f, y)
+      else:
+        loadPrim(f, y)
   else:
     {.error: "unsupported type for 'loadPrim'".}
 
diff --git a/compiler/ic/to_packed_ast.nim b/compiler/ic/to_packed_ast.nim
index d3e68d6ca..8b6e63f9e 100644
--- a/compiler/ic/to_packed_ast.nim
+++ b/compiler/ic/to_packed_ast.nim
@@ -33,8 +33,15 @@ type
     exports*: seq[(LitId, int32)]
     reexports*: seq[(LitId, PackedItemId)]
     compilerProcs*, trmacros*, converters*, pureEnums*: seq[(LitId, int32)]
-    methods*: seq[(LitId, PackedItemId, int32)]
+    methods*: seq[int32]
     macroUsages*: seq[(PackedItemId, PackedLineInfo)]
+
+    typeInstCache*: seq[(PackedItemId, PackedItemId)]
+    procInstCache*: seq[PackedInstantiation]
+    attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)]
+    methodsPerType*: seq[(PackedItemId, int, PackedItemId)]
+    enumToStringProcs*: seq[(PackedItemId, PackedItemId)]
+
     sh*: Shared
     cfg: PackedConfig
 
@@ -160,9 +167,7 @@ proc addPureEnum*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
   m.pureEnums.add((nameId, s.itemId.item))
 
 proc addMethod*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
-  let nameId = getOrIncl(m.sh.strings, s.name.s)
-  discard "to do"
-  # c.m.methods.add((nameId, s.itemId.item))
+  m.methods.add s.itemId.item
 
 proc addReexport*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
   let nameId = getOrIncl(m.sh.strings, s.name.s)
@@ -415,6 +420,17 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa
       toPackedNode(n[i], ir, c, m)
     ir.patch patchPos
 
+proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) =
+  var t = newSeq[PackedItemId](i.concreteTypes.len)
+  for j in 0..high(i.concreteTypes):
+    t[j] = storeTypeLater(i.concreteTypes[j], c, m)
+  m.procInstCache.add PackedInstantiation(key: storeSymLater(s, c, m),
+                                          sym: storeSymLater(i.sym, c, m),
+                                          concreteTypes: t)
+
+proc storeTypeInst*(c: var PackedEncoder; m: var PackedModule; s: PSym; inst: PType) =
+  m.typeInstCache.add (storeSymLater(s, c, m), storeTypeLater(inst, c, m))
+
 proc addPragmaComputation*(c: var PackedEncoder; m: var PackedModule; n: PNode) =
   toPackedNode(n, m.toReplay, c, m)
 
@@ -436,20 +452,6 @@ proc toPackedNodeTopLevel*(n: PNode, encoder: var PackedEncoder; m: var PackedMo
   toPackedNodeIgnoreProcDefs(n, encoder, m)
   flush encoder, m
 
-proc storePrim*(f: var RodFile; x: PackedType) =
-  for y in fields(x):
-    when y is seq:
-      storeSeq(f, y)
-    else:
-      storePrim(f, y)
-
-proc loadPrim*(f: var RodFile; x: var PackedType) =
-  for y in fields(x):
-    when y is seq:
-      loadSeq(f, y)
-    else:
-      loadPrim(f, y)
-
 proc loadError(err: RodFileError; filename: AbsoluteFile) =
   echo "Error: ", $err, "\nloading file: ", filename.string
 
@@ -503,6 +505,12 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   loadSeqSection symsSection, m.sh.syms
   loadSeqSection typesSection, m.sh.types
 
+  loadSeqSection typeInstCacheSection, m.typeInstCache
+  loadSeqSection procInstCacheSection, m.procInstCache
+  loadSeqSection attachedOpsSection, m.attachedOps
+  loadSeqSection methodsPerTypeSection, m.methodsPerType
+  loadSeqSection enumToStringProcsSection, m.enumToStringProcs
+
   close(f)
   result = f.err
 
@@ -557,6 +565,13 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeSeqSection symsSection, m.sh.syms
 
   storeSeqSection typesSection, m.sh.types
+
+  storeSeqSection typeInstCacheSection, m.typeInstCache
+  storeSeqSection procInstCacheSection, m.procInstCache
+  storeSeqSection attachedOpsSection, m.attachedOps
+  storeSeqSection methodsPerTypeSection, m.methodsPerType
+  storeSeqSection enumToStringProcsSection, m.enumToStringProcs
+
   close(f)
   if f.err != ok:
     storeError(f.err, filename)
@@ -902,6 +917,36 @@ proc loadProcBody*(config: ConfigRef, cache: IdentCache;
   assert pos != emptyNodeId
   result = loadProcBody(decoder, g, mId, g[mId].fromDisk.bodies, NodePos pos)
 
+proc loadTypeFromId*(config: ConfigRef, cache: IdentCache;
+                     g: var PackedModuleGraph; module: int; id: PackedItemId): PType =
+  result = g[module].types[id.item]
+  if result == nil:
+    var decoder = PackedDecoder(
+      lastModule: int32(-1),
+      lastLit: LitId(0),
+      lastFile: FileIndex(-1),
+      config: config,
+      cache: cache)
+    result = loadType(decoder, g, module, id)
+
+proc loadSymFromId*(config: ConfigRef, cache: IdentCache;
+                    g: var PackedModuleGraph; module: int; id: PackedItemId): PSym =
+  result = g[module].syms[id.item]
+  if result == nil:
+    var decoder = PackedDecoder(
+      lastModule: int32(-1),
+      lastLit: LitId(0),
+      lastFile: FileIndex(-1),
+      config: config,
+      cache: cache)
+    result = loadSym(decoder, g, module, id)
+
+proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; config: ConfigRef): ItemId =
+  if id.module == LitId(0):
+    ItemId(module: thisModule.int32, item: id.item)
+  else:
+    ItemId(module: toFileIndex(id.module, g[thisModule].fromDisk, config).int32, item: id.item)
+
 proc checkForHoles(m: PackedModule; config: ConfigRef; moduleId: int) =
   var bugs = 0
   for i in 1 .. high(m.sh.syms):