IC: use better packed line information format (#22917)

7 files changed, 135 insertions, 147 deletions

diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim
index ce6422101..49669e4e2 100644
--- a/compiler/ic/dce.nim
+++ b/compiler/ic/dce.nim
@@ -112,8 +112,8 @@ proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: N
     followLater(c, g, c.thisModule, n.operand)
   of nkModuleRef:
     let (n1, n2) = sons2(tree, n)
-    assert n1.kind == nkInt32Lit
-    assert n2.kind == nkInt32Lit
+    assert n1.kind == nkNone
+    assert n2.kind == nkNone
     let m = n1.litId
     let item = n2.operand
     let otherModule = toFileIndexCached(c.decoder, g, c.thisModule, m).int
diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim
index 4d02822b2..1b52c82b0 100644
--- a/compiler/ic/ic.nim
+++ b/compiler/ic/ic.nim
@@ -16,6 +16,8 @@ from std/os import removeFile, isAbsolute
 
 import ../../dist/checksums/src/checksums/sha1
 
+import ".." / nir / nirlineinfos
+
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions, formatfloat]
 
@@ -60,6 +62,7 @@ type
     strings*: BiTable[string] # we could share these between modules.
     numbers*: BiTable[BiggestInt] # we also store floats in here so
                                   # that we can assure that every bit is kept
+    man*: LineInfoManager
 
     cfg: PackedConfig
 
@@ -75,37 +78,36 @@ type
     symMarker*: IntSet #Table[ItemId, SymId]    # ItemId.item -> SymId
     config*: ConfigRef
 
-proc toString*(tree: PackedTree; n: NodePos; m: PackedModule; nesting: int;
+proc toString*(tree: PackedTree; pos: NodePos; m: PackedModule; nesting: int;
                result: var string) =
-  let pos = n.int
   if result.len > 0 and result[^1] notin {' ', '\n'}:
     result.add ' '
 
   result.add $tree[pos].kind
-  case tree.nodes[pos].kind
-  of nkNone, nkEmpty, nkNilLit, nkType: discard
+  case tree[pos].kind
+  of nkEmpty, nkNilLit, nkType: discard
   of nkIdent, nkStrLit..nkTripleStrLit:
     result.add " "
-    result.add m.strings[LitId tree.nodes[pos].operand]
+    result.add m.strings[LitId tree[pos].operand]
   of nkSym:
     result.add " "
-    result.add m.strings[m.syms[tree.nodes[pos].operand].name]
+    result.add m.strings[m.syms[tree[pos].operand].name]
   of directIntLit:
     result.add " "
-    result.addInt tree.nodes[pos].operand
+    result.addInt tree[pos].operand
   of externSIntLit:
     result.add " "
-    result.addInt m.numbers[LitId tree.nodes[pos].operand]
+    result.addInt m.numbers[LitId tree[pos].operand]
   of externUIntLit:
     result.add " "
-    result.addInt cast[uint64](m.numbers[LitId tree.nodes[pos].operand])
+    result.addInt cast[uint64](m.numbers[LitId tree[pos].operand])
   of nkFloatLit..nkFloat128Lit:
     result.add " "
-    result.addFloat cast[BiggestFloat](m.numbers[LitId tree.nodes[pos].operand])
+    result.addFloat cast[BiggestFloat](m.numbers[LitId tree[pos].operand])
   else:
     result.add "(\n"
     for i in 1..(nesting+1)*2: result.add ' '
-    for child in sonsReadonly(tree, n):
+    for child in sonsReadonly(tree, pos):
       toString(tree, child, m, nesting + 1, result)
     result.add "\n"
     for i in 1..nesting*2: result.add ' '
@@ -284,7 +286,8 @@ proc toLitId(x: BiggestInt; m: var PackedModule): LitId =
   result = getOrIncl(m.numbers, x)
 
 proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo =
-  PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m))
+  pack(m.man, toLitId(x.fileIndex, c, m), x.line.int32, x.col.int32)
+  #PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m))
 
 proc safeItemId(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId {.inline.} =
   ## given a symbol, produce an ItemId with the correct properties
@@ -421,53 +424,49 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
 proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
   ## add a remote symbol reference to the tree
   let info = n.info.toPackedInfo(c, m)
-  ir.nodes.add PackedNode(kind: nkModuleRef, operand: 3.int32, # spans 3 nodes in total
-                          typeId: storeTypeLater(n.typ, c, m), info: info)
-  ir.nodes.add PackedNode(kind: nkInt32Lit, info: info,
-                          operand: toLitId(n.sym.itemId.module.FileIndex, c, m).int32)
-  ir.nodes.add PackedNode(kind: nkInt32Lit, info: info,
-                          operand: n.sym.itemId.item)
+  ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total
+             typeId = storeTypeLater(n.typ, c, m), info = info)
+  ir.addNode(kind = nkNone, info = info,
+             operand = toLitId(n.sym.itemId.module.FileIndex, c, m).int32)
+  ir.addNode(kind = nkNone, info = info,
+             operand = n.sym.itemId.item)
 
 proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
   ## serialize a node into the tree
   if n == nil:
-    ir.nodes.add PackedNode(kind: nkNilRodNode, flags: {}, operand: 1)
+    ir.addNode(kind = nkNilRodNode, operand = 1, info = NoLineInfo)
     return
   let info = toPackedInfo(n.info, c, m)
   case n.kind
   of nkNone, nkEmpty, nkNilLit, nkType:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, operand: 0,
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
+    ir.addNode(kind = n.kind, flags = n.flags, operand = 0,
+               typeId = storeTypeLater(n.typ, c, m), info = info)
   of nkIdent:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.strings, n.ident.s),
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
+    ir.addNode(kind = n.kind, flags = n.flags,
+                operand = int32 getOrIncl(m.strings, n.ident.s),
+                typeId = storeTypeLater(n.typ, c, m), info = info)
   of nkSym:
     if n.sym.itemId.module == c.thisModule:
       # it is a symbol that belongs to the module we're currently
       # packing:
       let id = n.sym.storeSymLater(c, m).item
-      ir.nodes.add PackedNode(kind: nkSym, flags: n.flags, operand: id,
-                              typeId: storeTypeLater(n.typ, c, m), info: info)
+      ir.addNode(kind = nkSym, flags = n.flags, operand = id,
+                 typeId = storeTypeLater(n.typ, c, m), info = info)
     else:
       # store it as an external module reference:
       addModuleRef(n, ir, c, m)
-  of directIntLit:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32(n.intVal),
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
   of externIntLit:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.numbers, n.intVal),
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
+    ir.addNode(kind = n.kind, flags = n.flags,
+               operand = int32 getOrIncl(m.numbers, n.intVal),
+               typeId = storeTypeLater(n.typ, c, m), info = info)
   of nkStrLit..nkTripleStrLit:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.strings, n.strVal),
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
+    ir.addNode(kind = n.kind, flags = n.flags,
+               operand = int32 getOrIncl(m.strings, n.strVal),
+               typeId = storeTypeLater(n.typ, c, m), info = info)
   of nkFloatLit..nkFloat128Lit:
-    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-                            operand: int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)),
-                            typeId: storeTypeLater(n.typ, c, m), info: info)
+    ir.addNode(kind = n.kind, flags = n.flags,
+               operand = int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)),
+               typeId = storeTypeLater(n.typ, c, m), info = info)
   else:
     let patchPos = ir.prepare(n.kind, n.flags,
                               storeTypeLater(n.typ, c, m), info)
@@ -492,8 +491,8 @@ proc toPackedProcDef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var
       # do not serialize the body of the proc, it's unnecessary since
       # n[0].sym.ast has the sem'checked variant of it which is what
       # everybody should use instead.
-      ir.nodes.add PackedNode(kind: nkEmpty, flags: {}, operand: 0,
-                              typeId: nilItemId, info: info)
+      ir.addNode(kind = nkEmpty, flags = {}, operand = 0,
+                 typeId = nilItemId, info = info)
   ir.patch patchPos
 
 proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var PackedModule) =
@@ -609,9 +608,9 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   loadSeqSection methodsSection, m.methods
   loadSeqSection pureEnumsSection, m.pureEnums
 
-  loadSeqSection toReplaySection, m.toReplay.nodes
-  loadSeqSection topLevelSection, m.topLevel.nodes
-  loadSeqSection bodiesSection, m.bodies.nodes
+  loadTabSection toReplaySection, m.toReplay
+  loadTabSection topLevelSection, m.topLevel
+  loadTabSection bodiesSection, m.bodies
   loadSeqSection symsSection, m.syms
   loadSeqSection typesSection, m.types
 
@@ -625,6 +624,9 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   f.loadSection backendFlagsSection
   f.loadPrim m.backendFlags
 
+  f.loadSection sideChannelSection
+  f.load m.man
+
   close(f)
   result = f.err
 
@@ -672,10 +674,10 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeSeqSection methodsSection, m.methods
   storeSeqSection pureEnumsSection, m.pureEnums
 
-  storeSeqSection toReplaySection, m.toReplay.nodes
-  storeSeqSection topLevelSection, m.topLevel.nodes
+  storeTabSection toReplaySection, m.toReplay
+  storeTabSection topLevelSection, m.topLevel
 
-  storeSeqSection bodiesSection, m.bodies.nodes
+  storeTabSection bodiesSection, m.bodies
   storeSeqSection symsSection, m.syms
 
   storeSeqSection typesSection, m.types
@@ -690,6 +692,9 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   f.storeSection backendFlagsSection
   f.storePrim m.backendFlags
 
+  f.storeSection sideChannelSection
+  f.store m.man
+
   close(f)
   encoder.disable()
   if f.err != ok:
@@ -748,8 +753,9 @@ proc toFileIndexCached*(c: var PackedDecoder; g: PackedModuleGraph; thisModule:
 proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
                        x: PackedLineInfo): TLineInfo =
   assert g[thisModule].status in {loaded, storing, stored}
-  result = TLineInfo(line: x.line, col: x.col,
-            fileIndex: toFileIndexCached(c, g, thisModule, x.file))
+  let (fileId, line, col) = unpack(g[thisModule].fromDisk.man, x)
+  result = TLineInfo(line: line.uint16, col: col.int16,
+            fileIndex: toFileIndexCached(c, g, thisModule, fileId))
 
 proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
                 tree: PackedTree; n: NodePos): PNode =
@@ -768,9 +774,9 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
   of nkIdent:
     result.ident = getIdent(c.cache, g[thisModule].fromDisk.strings[n.litId])
   of nkSym:
-    result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand))
+    result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree[n].operand))
   of directIntLit:
-    result.intVal = tree.nodes[n.int].operand
+    result.intVal = tree[n].operand
   of externIntLit:
     result.intVal = g[thisModule].fromDisk.numbers[n.litId]
   of nkStrLit..nkTripleStrLit:
@@ -779,10 +785,10 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
     result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.numbers[n.litId])
   of nkModuleRef:
     let (n1, n2) = sons2(tree, n)
-    assert n1.kind == nkInt32Lit
-    assert n2.kind == nkInt32Lit
+    assert n1.kind == nkNone
+    assert n2.kind == nkNone
     transitionNoneToSym(result)
-    result.sym = loadSym(c, g, thisModule, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand))
+    result.sym = loadSym(c, g, thisModule, PackedItemId(module: n1.litId, item: tree[n2].operand))
   else:
     for n0 in sonsReadonly(tree, n):
       result.addAllowNil loadNodes(c, g, thisModule, tree, n0)
@@ -1239,5 +1245,5 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
       echo "  <anon symbol?> local ID: ", i, " kind ", m.syms[i].kind
 
   echo "symbols: ", m.syms.len, " types: ", m.types.len,
-    " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len,
+    " top level nodes: ", m.topLevel.len, " other nodes: ", m.bodies.len,
     " strings: ", m.strings.len, " numbers: ", m.numbers.len
diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim
index ed87ae59d..00014f15a 100644
--- a/compiler/ic/integrity.nim
+++ b/compiler/ic/integrity.nim
@@ -72,15 +72,15 @@ proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) =
     c.thisModule = oldThisModule
 
 proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) =
-  if tree[n.int].typeId != nilItemId:
-    checkType(c, tree[n.int].typeId)
+  if tree[n].typeId != nilItemId:
+    checkType(c, tree[n].typeId)
   case n.kind
   of nkEmpty, nkNilLit, nkType, nkNilRodNode:
     discard
   of nkIdent:
     assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId
   of nkSym:
-    checkLocalSym(c, tree.nodes[n.int].operand)
+    checkLocalSym(c, tree[n].operand)
   of directIntLit:
     discard
   of externIntLit, nkFloatLit..nkFloat128Lit:
@@ -89,9 +89,9 @@ proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) =
     assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId
   of nkModuleRef:
     let (n1, n2) = sons2(tree, n)
-    assert n1.kind == nkInt32Lit
-    assert n2.kind == nkInt32Lit
-    checkForeignSym(c, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand))
+    assert n1.kind == nkNone
+    assert n2.kind == nkNone
+    checkForeignSym(c, PackedItemId(module: n1.litId, item: tree[n2].operand))
   else:
     for n0 in sonsReadonly(tree, n):
       checkNode(c, tree, n0)
diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim
index aea5e12e7..c2d7ee4ad 100644
--- a/compiler/ic/navigator.nim
+++ b/compiler/ic/navigator.nim
@@ -20,20 +20,25 @@ when defined(nimPreviewSlimSystem):
   import std/assertions
 
 import ".." / [ast, modulegraphs, msgs, options]
+import ".." / nir / nirlineinfos
 import packed_ast, bitabs, ic
 
 type
+  UnpackedLineInfo = object
+    file: LitId
+    line, col: int
   NavContext = object
     g: ModuleGraph
     thisModule: int32
-    trackPos: PackedLineInfo
+    trackPos: UnpackedLineInfo
     alreadyEmitted: HashSet[string]
     outputSep: char # for easier testing, use short filenames and spaces instead of tabs.
 
-proc isTracked(current, trackPos: PackedLineInfo, tokenLen: int): bool =
-  if current.file == trackPos.file and current.line == trackPos.line:
+proc isTracked(man: LineInfoManager; current: PackedLineInfo, trackPos: UnpackedLineInfo, tokenLen: int): bool =
+  let (currentFile, currentLine, currentCol) = man.unpack(current)
+  if currentFile == trackPos.file and currentLine == trackPos.line:
     let col = trackPos.col
-    if col >= current.col and col < current.col+tokenLen:
+    if col >= currentCol and col < currentCol+tokenLen:
       result = true
     else:
       result = false
@@ -42,32 +47,34 @@ proc isTracked(current, trackPos: PackedLineInfo, tokenLen: int): bool =
 
 proc searchLocalSym(c: var NavContext; s: PackedSym; info: PackedLineInfo): bool =
   result = s.name != LitId(0) and
-    isTracked(info, c.trackPos, c.g.packed[c.thisModule].fromDisk.strings[s.name].len)
+    isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[c.thisModule].fromDisk.strings[s.name].len)
 
 proc searchForeignSym(c: var NavContext; s: ItemId; info: PackedLineInfo): bool =
   let name = c.g.packed[s.module].fromDisk.syms[s.item].name
   result = name != LitId(0) and
-    isTracked(info, c.trackPos, c.g.packed[s.module].fromDisk.strings[name].len)
+    isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[s.module].fromDisk.strings[name].len)
 
 const
   EmptyItemId = ItemId(module: -1'i32, item: -1'i32)
 
 proc search(c: var NavContext; tree: PackedTree): ItemId =
   # We use the linear representation here directly:
-  for i in 0..high(tree.nodes):
-    case tree.nodes[i].kind
+  for i in 0..<len(tree):
+    let i = NodePos(i)
+    case tree[i].kind
     of nkSym:
-      let item = tree.nodes[i].operand
-      if searchLocalSym(c, c.g.packed[c.thisModule].fromDisk.syms[item], tree.nodes[i].info):
+      let item = tree[i].operand
+      if searchLocalSym(c, c.g.packed[c.thisModule].fromDisk.syms[item], tree[i].info):
         return ItemId(module: c.thisModule, item: item)
     of nkModuleRef:
-      if tree.nodes[i].info.line == c.trackPos.line and tree.nodes[i].info.file == c.trackPos.file:
-        let (n1, n2) = sons2(tree, NodePos i)
+      let (currentFile, currentLine, currentCol) = c.g.packed[c.thisModule].fromDisk.man.unpack(tree[i].info)
+      if currentLine == c.trackPos.line and currentFile == c.trackPos.file:
+        let (n1, n2) = sons2(tree, i)
         assert n1.kind == nkInt32Lit
         assert n2.kind == nkInt32Lit
-        let pId = PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)
+        let pId = PackedItemId(module: n1.litId, item: tree[n2].operand)
         let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config)
-        if searchForeignSym(c, itemId, tree.nodes[i].info):
+        if searchForeignSym(c, itemId, tree[i].info):
           return itemId
     else: discard
   return EmptyItemId
@@ -77,32 +84,34 @@ proc isDecl(tree: PackedTree; n: NodePos): bool =
   const declarativeNodes = procDefs + {nkMacroDef, nkTemplateDef,
     nkLetSection, nkVarSection, nkUsingStmt, nkConstSection, nkTypeSection,
     nkIdentDefs, nkEnumTy, nkVarTuple}
-  result = n.int >= 0 and tree[n.int].kind in declarativeNodes
+  result = n.int >= 0 and tree[n].kind in declarativeNodes
 
 proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) =
+  let (fileId, line, col) = unpack(c.g.packed[c.thisModule].fromDisk.man, info)
   var m = ""
-  var file = c.g.packed[c.thisModule].fromDisk.strings[info.file]
+  var file = c.g.packed[c.thisModule].fromDisk.strings[fileId]
   if c.outputSep == ' ':
     file = os.extractFilename file
-  toLocation(m, file, info.line.int, info.col.int + ColOffset)
+  toLocation(m, file, line, col + ColOffset)
   if not c.alreadyEmitted.containsOrIncl(m):
     msgWriteln c.g.config, (if isDecl: "def" else: "usage") & c.outputSep & m
 
 proc list(c: var NavContext; tree: PackedTree; sym: ItemId) =
-  for i in 0..high(tree.nodes):
-    case tree.nodes[i].kind
+  for i in 0..<len(tree):
+    let i = NodePos(i)
+    case tree[i].kind
     of nkSym:
-      let item = tree.nodes[i].operand
+      let item = tree[i].operand
       if sym.item == item and sym.module == c.thisModule:
-        usage(c, tree.nodes[i].info, isDecl(tree, parent(NodePos i)))
+        usage(c, tree[i].info, isDecl(tree, parent(i)))
     of nkModuleRef:
-      let (n1, n2) = sons2(tree, NodePos i)
-      assert n1.kind == nkInt32Lit
-      assert n2.kind == nkInt32Lit
-      let pId = PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)
+      let (n1, n2) = sons2(tree, i)
+      assert n1.kind == nkNone
+      assert n2.kind == nkNone
+      let pId = PackedItemId(module: n1.litId, item: tree[n2].operand)
       let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config)
       if itemId.item == sym.item and sym.module == itemId.module:
-        usage(c, tree.nodes[i].info, isDecl(tree, parent(NodePos i)))
+        usage(c, tree[i].info, isDecl(tree, parent(i)))
     else: discard
 
 proc searchForIncludeFile(g: ModuleGraph; fullPath: string): int =
@@ -138,7 +147,7 @@ proc nav(g: ModuleGraph) =
   var c = NavContext(
     g: g,
     thisModule: int32 mid,
-    trackPos: PackedLineInfo(line: unpacked.line, col: unpacked.col, file: fileId),
+    trackPos: UnpackedLineInfo(line: unpacked.line.int, col: unpacked.col.int, file: fileId),
     outputSep: if isDefined(g.config, "nimIcNavigatorTests"): ' ' else: '\t'
   )
   var symId = search(c, g.packed[mid].fromDisk.topLevel)
diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim
index e7443c3c7..ea7c7f967 100644
--- a/compiler/ic/packed_ast.nim
+++ b/compiler/ic/packed_ast.nim
@@ -13,9 +13,11 @@
 ## it is superior.
 
 import std/[hashes, tables, strtabs]
-import bitabs
+import bitabs, rodfiles
 import ".." / [ast, options]
 
+import ".." / nir / nirlineinfos
+
 when defined(nimPreviewSlimSystem):
   import std/assertions
 
@@ -37,11 +39,6 @@ const
   emptyNodeId* = NodeId(-1)
 
 type
-  PackedLineInfo* = object
-    line*: uint16
-    col*: int16
-    file*: LitId
-
   PackedLib* = object
     kind*: TLibKind
     generated*: bool
@@ -95,13 +92,15 @@ type
     flags*: TNodeFlags
     operand*: int32  # for kind in {nkSym, nkSymDef}: SymId
                      # for kind in {nkStrLit, nkIdent, nkNumberLit}: LitId
-                     # for kind in nkInt32Lit: direct value
+                     # for kind in nkNone: direct value
                      # for non-atom kinds: the number of nodes (for easy skipping)
     typeId*: PackedItemId
     info*: PackedLineInfo
 
   PackedTree* = object ## usually represents a full Nim module
-    nodes*: seq[PackedNode]
+    nodes: seq[PackedNode]
+    #withFlags: seq[(int, TNodeFlags)]
+    #withTypes: seq[(int, PackedItemId)]
 
   PackedInstantiation* = object
     key*, sym*: PackedItemId
@@ -118,25 +117,6 @@ proc newTreeFrom*(old: PackedTree): PackedTree =
   result.nodes = @[]
   when false: result.sh = old.sh
 
-when false:
-  proc declareSym*(tree: var PackedTree; kind: TSymKind;
-                  name: LitId; info: PackedLineInfo): SymId =
-    result = SymId(tree.sh.syms.len)
-    tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info)
-
-  proc litIdFromName*(tree: PackedTree; name: string): LitId =
-    result = tree.sh.strings.getOrIncl(name)
-
-  proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) =
-    tree.nodes.add PackedNode(kind: kind, info: info,
-                              operand: int32 getOrIncl(tree.sh.strings, token))
-
-  proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) =
-    tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
-
-proc throwAwayLastNode*(tree: var PackedTree) =
-  tree.nodes.setLen(tree.nodes.len-1)
-
 proc addIdent*(tree: var PackedTree; s: LitId; info: PackedLineInfo) =
   tree.nodes.add PackedNode(kind: nkIdent, operand: int32(s), info: info)
 
@@ -144,42 +124,25 @@ proc addSym*(tree: var PackedTree; s: int32; info: PackedLineInfo) =
   tree.nodes.add PackedNode(kind: nkSym, operand: s, info: info)
 
 proc addModuleId*(tree: var PackedTree; s: ModuleId; info: PackedLineInfo) =
-  tree.nodes.add PackedNode(kind: nkInt32Lit, operand: int32(s), info: info)
+  tree.nodes.add PackedNode(kind: nkNone, operand: int32(s), info: info)
 
 proc addSymDef*(tree: var PackedTree; s: SymId; info: PackedLineInfo) =
   tree.nodes.add PackedNode(kind: nkSym, operand: int32(s), info: info)
 
 proc isAtom*(tree: PackedTree; pos: int): bool {.inline.} = tree.nodes[pos].kind <= nkNilLit
 
-proc copyTree*(dest: var PackedTree; tree: PackedTree; n: NodePos) =
-  # and this is why the IR is superior. We can copy subtrees
-  # via a linear scan.
-  let pos = n.int
-  let L = if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
-  let d = dest.nodes.len
-  dest.nodes.setLen(d + L)
-  for i in 0..<L:
-    dest.nodes[d+i] = tree.nodes[pos+i]
-
-when false:
-  proc copySym*(dest: var PackedTree; tree: PackedTree; s: SymId): SymId =
-    result = SymId(dest.sh.syms.len)
-    assert int(s) < tree.sh.syms.len
-    let oldSym = tree.sh.syms[s.int]
-    dest.sh.syms.add oldSym
-
 type
   PatchPos = distinct int
 
-when false:
-  proc prepare*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo): PatchPos =
-    result = PatchPos tree.nodes.len
-    tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
+proc addNode*(t: var PackedTree; kind: TNodeKind; operand: int32;
+              typeId: PackedItemId = nilItemId; info: PackedLineInfo;
+              flags: TNodeFlags = {}) =
+  t.nodes.add PackedNode(kind: kind, flags: flags, operand: operand,
+                         typeId: typeId, info: info)
 
 proc prepare*(tree: var PackedTree; kind: TNodeKind; flags: TNodeFlags; typeId: PackedItemId; info: PackedLineInfo): PatchPos =
   result = PatchPos tree.nodes.len
-  tree.nodes.add PackedNode(kind: kind, flags: flags, operand: 0, info: info,
-                            typeId: typeId)
+  tree.addNode(kind = kind, flags = flags, operand = 0, info = info, typeId = typeId)
 
 proc prepare*(dest: var PackedTree; source: PackedTree; sourcePos: NodePos): PatchPos =
   result = PatchPos dest.nodes.len
@@ -193,8 +156,8 @@ proc patch*(tree: var PackedTree; pos: PatchPos) =
 
 proc len*(tree: PackedTree): int {.inline.} = tree.nodes.len
 
-proc `[]`*(tree: PackedTree; i: int): lent PackedNode {.inline.} =
-  tree.nodes[i]
+proc `[]`*(tree: PackedTree; i: NodePos): lent PackedNode {.inline.} =
+  tree.nodes[i.int]
 
 proc nextChild(tree: PackedTree; pos: var int) {.inline.} =
   if tree.nodes[pos].kind > nkNilLit:
@@ -357,16 +320,17 @@ const
     nkIntLit,
     nkInt8Lit,
     nkInt16Lit,
+    nkInt32Lit,
     nkInt64Lit,
     nkUIntLit,
     nkUInt8Lit,
     nkUInt16Lit,
     nkUInt32Lit,
-    nkUInt64Lit} # nkInt32Lit is missing by design!
+    nkUInt64Lit}
 
-  externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt64Lit}
+  externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt32Lit, nkInt64Lit}
   externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit}
-  directIntLit* = nkInt32Lit
+  directIntLit* = nkNone
 
 when false:
   proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
@@ -420,3 +384,9 @@ iterator allNodes*(tree: PackedTree): NodePos =
 
 proc toPackedItemId*(item: int32): PackedItemId {.inline.} =
   PackedItemId(module: LitId(0), item: item)
+
+proc load*(f: var RodFile; t: var PackedTree) =
+  loadSeq f, t.nodes
+
+proc store*(f: var RodFile; t: PackedTree) =
+  storeSeq f, t.nodes
diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim
index be5095fbb..968bf255f 100644
--- a/compiler/ic/rodfiles.nim
+++ b/compiler/ic/rodfiles.nim
@@ -112,7 +112,7 @@ type
                        # better than exceptions.
 
 const
-  RodVersion = 1
+  RodVersion = 2
   defaultCookie = [byte(0), byte('R'), byte('O'), byte('D'),
             byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)]
 
diff --git a/compiler/nir/nirlineinfos.nim b/compiler/nir/nirlineinfos.nim
index 5f5d55086..f11ef7c42 100644
--- a/compiler/nir/nirlineinfos.nim
+++ b/compiler/nir/nirlineinfos.nim
@@ -42,6 +42,9 @@ type
   LineInfoManager* = object
     aside: seq[(LitId, int32, int32)]
 
+const
+  NoLineInfo* = PackedLineInfo(0'u32)
+
 proc pack*(m: var LineInfoManager; file: LitId; line, col: int32): PackedLineInfo =
   if file.uint32 <= FileMax.uint32 and line <= LineMax and col <= ColMax:
     let col = if col < 0'i32: 0'u32 else: col.uint32