ic refactoring (#17778)

* minor IC documentation update

* IC: refactoring: removed the 'shared' type and fields, these were a leftover from an earlier design

1 files changed, 111 insertions, 64 deletions

diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim
index 6f208a2a4..a23685be7 100644
--- a/compiler/ic/ic.nim
+++ b/compiler/ic/ic.nim
@@ -51,7 +51,12 @@ type
     emittedTypeInfo*: seq[string]
     backendFlags*: set[ModuleBackendFlag]
 
-    sh*: Shared
+    syms*: seq[PackedSym]
+    types*: seq[PackedType]
+    strings*: BiTable[string] # we could share these between modules.
+    numbers*: BiTable[BiggestInt] # we also store floats in here so
+                                  # that we can assure that every bit is kept
+
     cfg: PackedConfig
 
   PackedEncoder* = object
@@ -66,6 +71,50 @@ type
     symMarker*: IntSet #Table[ItemId, SymId]    # ItemId.item -> SymId
     config*: ConfigRef
 
+proc toString*(tree: PackedTree; n: NodePos; m: PackedModule; nesting: int;
+               result: var string) =
+  let pos = n.int
+  if result.len > 0 and result[^1] notin {' ', '\n'}:
+    result.add ' '
+
+  result.add $tree[pos].kind
+  case tree.nodes[pos].kind
+  of nkNone, nkEmpty, nkNilLit, nkType: discard
+  of nkIdent, nkStrLit..nkTripleStrLit:
+    result.add " "
+    result.add m.strings[LitId tree.nodes[pos].operand]
+  of nkSym:
+    result.add " "
+    result.add m.strings[m.syms[tree.nodes[pos].operand].name]
+  of directIntLit:
+    result.add " "
+    result.addInt tree.nodes[pos].operand
+  of externSIntLit:
+    result.add " "
+    result.addInt m.numbers[LitId tree.nodes[pos].operand]
+  of externUIntLit:
+    result.add " "
+    result.add $cast[uint64](m.numbers[LitId tree.nodes[pos].operand])
+  of nkFloatLit..nkFloat128Lit:
+    result.add " "
+    result.add $cast[BiggestFloat](m.numbers[LitId tree.nodes[pos].operand])
+  else:
+    result.add "(\n"
+    for i in 1..(nesting+1)*2: result.add ' '
+    for child in sonsReadonly(tree, n):
+      toString(tree, child, m, nesting + 1, result)
+    result.add "\n"
+    for i in 1..nesting*2: result.add ' '
+    result.add ")"
+    #for i in 1..nesting*2: result.add ' '
+
+proc toString*(tree: PackedTree; n: NodePos; m: PackedModule): string =
+  result = ""
+  toString(tree, n, m, 0, result)
+
+proc debug*(tree: PackedTree; m: PackedModule) =
+  stdout.write toString(tree, NodePos 0, m)
+
 proc isActive*(e: PackedEncoder): bool = e.config != nil
 proc disable(e: var PackedEncoder) = e.config = nil
 
@@ -134,14 +183,14 @@ proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId =
     result = c.filenames.getOrDefault(x)
     if result == LitId(0):
       let p = msgs.toFullPath(c.config, x)
-      result = getOrIncl(m.sh.strings, p)
+      result = getOrIncl(m.strings, p)
       c.filenames[x] = result
     c.lastFile = x
     c.lastLit = result
   assert result != LitId(0)
 
 proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex =
-  result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x])
+  result = msgs.fileInfoIdx(config, AbsoluteFile m.strings[x])
 
 proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
   for it in mitems(m.includes):
@@ -151,7 +200,6 @@ proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
 
 proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; config: ConfigRef; pc: PackedConfig) =
   ## setup a context for serializing to packed ast
-  m.sh = Shared()
   c.thisModule = moduleSym.itemId.module
   c.config = config
   m.moduleFlags = moduleSym.flags
@@ -179,11 +227,11 @@ proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex)
   m.imports.add toLitId(f, c, m)
 
 proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
-  let nameId = getOrIncl(m.sh.strings, s.name.s)
+  let nameId = getOrIncl(m.strings, s.name.s)
   m.hidden.add((nameId, s.itemId.item))
 
 proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
-  let nameId = getOrIncl(m.sh.strings, s.name.s)
+  let nameId = getOrIncl(m.strings, s.name.s)
   m.exports.add((nameId, s.itemId.item))
 
 proc addConverter*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
@@ -201,12 +249,12 @@ proc addMethod*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
   m.methods.add s.itemId.item
 
 proc addReexport*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
-  let nameId = getOrIncl(m.sh.strings, s.name.s)
+  let nameId = getOrIncl(m.strings, s.name.s)
   m.reexports.add((nameId, PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m),
                                         item: s.itemId.item)))
 
 proc addCompilerProc*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
-  let nameId = getOrIncl(m.sh.strings, s.name.s)
+  let nameId = getOrIncl(m.strings, s.name.s)
   m.compilerProcs.add((nameId, s.itemId.item))
 
 proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule)
@@ -225,11 +273,11 @@ proc flush(c: var PackedEncoder; m: var PackedModule) =
 
 proc toLitId(x: string; m: var PackedModule): LitId =
   ## store a string as a literal
-  result = getOrIncl(m.sh.strings, x)
+  result = getOrIncl(m.strings, x)
 
 proc toLitId(x: BiggestInt; m: var PackedModule): LitId =
   ## store an integer as a literal
-  result = getOrIncl(m.sh.numbers, x)
+  result = getOrIncl(m.numbers, x)
 
 proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo =
   PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m))
@@ -299,8 +347,8 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
   result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item)
 
   if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item):
-    if t.uniqueId.item >= m.sh.types.len:
-      setLen m.sh.types, t.uniqueId.item+1
+    if t.uniqueId.item >= m.types.len:
+      setLen m.types, t.uniqueId.item+1
 
     var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv,
       size: t.size, align: t.align, nonUniqueId: t.itemId.item,
@@ -315,7 +363,7 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
     p.owner = t.owner.safeItemId(c, m)
 
     # fill the reserved slot, nothing else:
-    m.sh.types[t.uniqueId.item] = p
+    m.types[t.uniqueId.item] = p
 
 proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib =
   ## the plib hangs off the psym via the .annex field
@@ -334,8 +382,8 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
   result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item)
 
   if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item):
-    if s.itemId.item >= m.sh.syms.len:
-      setLen m.sh.syms, s.itemId.item+1
+    if s.itemId.item >= m.syms.len:
+      setLen m.syms, s.itemId.item+1
 
     assert sfForward notin s.flags
 
@@ -363,7 +411,7 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
       p.cname = toLitId(s.cname, m)
 
     # fill the reserved slot, nothing else:
-    m.sh.syms[s.itemId.item] = p
+    m.syms[s.itemId.item] = p
 
 proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
   ## add a remote symbol reference to the tree
@@ -387,7 +435,7 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa
                             typeId: storeTypeLater(n.typ, c, m), info: info)
   of nkIdent:
     ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.sh.strings, n.ident.s),
+                            operand: int32 getOrIncl(m.strings, n.ident.s),
                             typeId: storeTypeLater(n.typ, c, m), info: info)
   of nkSym:
     if n.sym.itemId.module == c.thisModule:
@@ -405,15 +453,15 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa
                             typeId: storeTypeLater(n.typ, c, m), info: info)
   of externIntLit:
     ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.sh.numbers, n.intVal),
+                            operand: int32 getOrIncl(m.numbers, n.intVal),
                             typeId: storeTypeLater(n.typ, c, m), info: info)
   of nkStrLit..nkTripleStrLit:
     ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.sh.strings, n.strVal),
+                            operand: int32 getOrIncl(m.strings, n.strVal),
                             typeId: storeTypeLater(n.typ, c, m), info: info)
   of nkFloatLit..nkFloat128Lit:
     ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.sh.numbers, cast[BiggestInt](n.floatVal)),
+                            operand: int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)),
                             typeId: storeTypeLater(n.typ, c, m), info: info)
   else:
     let patchPos = ir.prepare(n.kind, n.flags,
@@ -499,7 +547,6 @@ proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) =
 
 proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef;
                   ignoreConfig = false): RodFileError =
-  m.sh = Shared()
   var f = rodfiles.open(filename.string)
   f.loadHeader()
   f.loadSection configSection
@@ -519,7 +566,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
     f.loadSection section
     f.load data
 
-  loadTabSection stringsSection, m.sh.strings
+  loadTabSection stringsSection, m.strings
 
   loadSeqSection checkSumsSection, m.includes
   if not includesIdentical(m, config):
@@ -527,7 +574,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
 
   loadSeqSection depsSection, m.imports
 
-  loadTabSection numbersSection, m.sh.numbers
+  loadTabSection numbersSection, m.numbers
 
   loadSeqSection exportsSection, m.exports
   loadSeqSection hiddenSection, m.hidden
@@ -545,8 +592,8 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   loadSeqSection toReplaySection, m.toReplay.nodes
   loadSeqSection topLevelSection, m.topLevel.nodes
   loadSeqSection bodiesSection, m.bodies.nodes
-  loadSeqSection symsSection, m.sh.syms
-  loadSeqSection typesSection, m.sh.types
+  loadSeqSection symsSection, m.syms
+  loadSeqSection typesSection, m.types
 
   loadSeqSection typeInstCacheSection, m.typeInstCache
   loadSeqSection procInstCacheSection, m.procInstCache
@@ -586,13 +633,13 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
     f.storeSection section
     f.store data
 
-  storeTabSection stringsSection, m.sh.strings
+  storeTabSection stringsSection, m.strings
 
   storeSeqSection checkSumsSection, m.includes
 
   storeSeqSection depsSection, m.imports
 
-  storeTabSection numbersSection, m.sh.numbers
+  storeTabSection numbersSection, m.numbers
 
   storeSeqSection exportsSection, m.exports
   storeSeqSection hiddenSection, m.hidden
@@ -610,9 +657,9 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeSeqSection topLevelSection, m.topLevel.nodes
 
   storeSeqSection bodiesSection, m.bodies.nodes
-  storeSeqSection symsSection, m.sh.syms
+  storeSeqSection symsSection, m.syms
 
-  storeSeqSection typesSection, m.sh.types
+  storeSeqSection typesSection, m.types
 
   storeSeqSection typeInstCacheSection, m.typeInstCache
   storeSeqSection procInstCacheSection, m.procInstCache
@@ -700,17 +747,17 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
   of nkEmpty, nkNilLit, nkType:
     discard
   of nkIdent:
-    result.ident = getIdent(c.cache, g[thisModule].fromDisk.sh.strings[n.litId])
+    result.ident = getIdent(c.cache, g[thisModule].fromDisk.strings[n.litId])
   of nkSym:
     result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand))
   of directIntLit:
     result.intVal = tree.nodes[n.int].operand
   of externIntLit:
-    result.intVal = g[thisModule].fromDisk.sh.numbers[n.litId]
+    result.intVal = g[thisModule].fromDisk.numbers[n.litId]
   of nkStrLit..nkTripleStrLit:
-    result.strVal = g[thisModule].fromDisk.sh.strings[n.litId]
+    result.strVal = g[thisModule].fromDisk.strings[n.litId]
   of nkFloatLit..nkFloat128Lit:
-    result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.sh.numbers[n.litId])
+    result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.numbers[n.litId])
   of nkModuleRef:
     let (n1, n2) = sons2(tree, n)
     assert n1.kind == nkInt32Lit
@@ -767,7 +814,7 @@ proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
     options: s.options,
     position: if s.kind in {skForVar, skVar, skLet, skTemp}: 0 else: s.position,
     offset: if s.kind in routineKinds: defaultOffset else: s.offset,
-    name: getIdent(c.cache, g[si].fromDisk.sh.strings[s.name])
+    name: getIdent(c.cache, g[si].fromDisk.strings[s.name])
   )
 
 template loadAstBody(p, field) =
@@ -785,7 +832,7 @@ proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph;
     result = nil
   else:
     result = PLib(generated: l.generated, isOverriden: l.isOverriden,
-                  kind: l.kind, name: rope g[si].fromDisk.sh.strings[l.name])
+                  kind: l.kind, name: rope g[si].fromDisk.strings[l.name])
     loadAstBody(l, path)
 
 proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
@@ -798,14 +845,14 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
     loadAstBody(s, ast)
   result.annex = loadLib(c, g, si, item, s.annex)
   when hasFFI:
-    result.cname = g[si].fromDisk.sh.strings[s.cname]
+    result.cname = g[si].fromDisk.strings[s.cname]
 
   if s.kind in {skLet, skVar, skField, skForVar}:
     result.guard = loadSym(c, g, si, s.guard)
     result.bitsize = s.bitsize
     result.alignment = s.alignment
   result.owner = loadSym(c, g, si, s.owner)
-  let externalName = g[si].fromDisk.sh.strings[s.externalName]
+  let externalName = g[si].fromDisk.strings[s.externalName]
   if externalName != "":
     result.loc.r = rope externalName
   result.loc.flags = s.locFlags
@@ -818,14 +865,14 @@ proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s:
     assert g[si].status in {loaded, storing, stored}
     if not g[si].symsInit:
       g[si].symsInit = true
-      setLen g[si].syms, g[si].fromDisk.sh.syms.len
+      setLen g[si].syms, g[si].fromDisk.syms.len
 
     if g[si].syms[s.item] == nil:
-      if g[si].fromDisk.sh.syms[s.item].kind != skModule:
-        result = symHeaderFromPacked(c, g, g[si].fromDisk.sh.syms[s.item], si, s.item)
+      if g[si].fromDisk.syms[s.item].kind != skModule:
+        result = symHeaderFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item)
         # store it here early on, so that recursions work properly:
         g[si].syms[s.item] = result
-        symBodyFromPacked(c, g, g[si].fromDisk.sh.syms[s.item], si, s.item, result)
+        symBodyFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item, result)
       else:
         result = g[si].module
         assert result != nil
@@ -866,13 +913,13 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t
 
     if not g[si].typesInit:
       g[si].typesInit = true
-      setLen g[si].types, g[si].fromDisk.sh.types.len
+      setLen g[si].types, g[si].fromDisk.types.len
 
     if g[si].types[t.item] == nil:
-      result = typeHeaderFromPacked(c, g, g[si].fromDisk.sh.types[t.item], si, t.item)
+      result = typeHeaderFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item)
       # store it here early on, so that recursions work properly:
       g[si].types[t.item] = result
-      typeBodyFromPacked(c, g, g[si].fromDisk.sh.types[t.item], si, t.item, result)
+      typeBodyFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item, result)
     else:
       result = g[si].types[t.item]
     assert result.itemId.item > 0
@@ -897,7 +944,7 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
     let e2 =
       when e[1] is PackedItemId: e[1]
       else: PackedItemId(module: LitId(0), item: e[1])
-    iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(e2)
+    iface.mgetOrPut(cache.getIdent(m.fromDisk.strings[nameLit]), @[]).add(e2)
 
   for e in m.fromDisk.exports:
     m.iface.impl(e)
@@ -1000,7 +1047,7 @@ proc loadProcBody*(config: ConfigRef, cache: IdentCache;
     lastFile: FileIndex(-1),
     config: config,
     cache: cache)
-  let pos = g[mId].fromDisk.sh.syms[s.itemId.item].ast
+  let pos = g[mId].fromDisk.syms[s.itemId.item].ast
   assert pos != emptyNodeId
   result = loadProcBody(decoder, g, mId, g[mId].fromDisk.bodies, NodePos pos)
 
@@ -1042,15 +1089,15 @@ proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; confi
 
 proc checkForHoles(m: PackedModule; config: ConfigRef; moduleId: int) =
   var bugs = 0
-  for i in 1 .. high(m.sh.syms):
-    if m.sh.syms[i].kind == skUnknown:
+  for i in 1 .. high(m.syms):
+    if m.syms[i].kind == skUnknown:
       echo "EMPTY ID ", i, " module ", moduleId, " ", toFullPath(config, FileIndex(moduleId))
       inc bugs
   assert bugs == 0
   when false:
     var nones = 0
-    for i in 1 .. high(m.sh.types):
-      inc nones, m.sh.types[i].kind == tyNone
+    for i in 1 .. high(m.types):
+      inc nones, m.types[i].kind == tyNone
     assert nones < 1
 
 proc simulateLoadedModule*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
@@ -1133,13 +1180,13 @@ proc interfaceSymbol*(config: ConfigRef, cache: IdentCache;
   result = loadSym(decoder, g, int(module), values[0])
 
 proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator =
-  IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.sh.syms.len,
-              typeId: int32 m.fromDisk.sh.types.len)
+  IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.syms.len,
+              typeId: int32 m.fromDisk.types.len)
 
 proc searchForCompilerproc*(m: LoadedModule; name: string): int32 =
   # slow, linear search, but the results are cached:
   for it in items(m.fromDisk.compilerProcs):
-    if m.fromDisk.sh.strings[it[0]] == name:
+    if m.fromDisk.strings[it[0]] == name:
       return it[1]
   return -1
 
@@ -1149,31 +1196,31 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
   var m: PackedModule
   let err = loadRodFile(rodfile, m, config, ignoreConfig=true)
   if err != ok:
-    config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err 
+    config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err
 
   when true:
     echo "exports:"
     for ex in m.exports:
-      echo "  ", m.sh.strings[ex[0]], " local ID: ", ex[1]
-      assert ex[0] == m.sh.syms[ex[1]].name
+      echo "  ", m.strings[ex[0]], " local ID: ", ex[1]
+      assert ex[0] == m.syms[ex[1]].name
       # ex[1] int32
 
     echo "reexports:"
     for ex in m.reexports:
-      echo "  ", m.sh.strings[ex[0]]
+      echo "  ", m.strings[ex[0]]
     #  reexports*: seq[(LitId, PackedItemId)]
 
     echo "hidden: " & $m.hidden.len
     for ex in m.hidden:
-      echo "  ", m.sh.strings[ex[0]], " local ID: ", ex[1]
+      echo "  ", m.strings[ex[0]], " local ID: ", ex[1]
 
   echo "all symbols"
-  for i in 0..high(m.sh.syms):
-    if m.sh.syms[i].name != LitId(0):
-      echo "  ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i, " kind ", m.sh.syms[i].kind
+  for i in 0..high(m.syms):
+    if m.syms[i].name != LitId(0):
+      echo "  ", m.strings[m.syms[i].name], " local ID: ", i, " kind ", m.syms[i].kind
     else:
-      echo "  <anon symbol?> local ID: ", i, " kind ", m.sh.syms[i].kind
+      echo "  <anon symbol?> local ID: ", i, " kind ", m.syms[i].kind
 
-  echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len,
+  echo "symbols: ", m.syms.len, " types: ", m.types.len,
     " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len,
-    " strings: ", m.sh.strings.len, " numbers: ", m.sh.numbers.len
+    " strings: ", m.strings.len, " numbers: ", m.numbers.len