new symbol files: loading of ASTs and module graphs

2 files changed, 378 insertions, 2 deletions

diff --git a/compiler/rod.nim b/compiler/rod.nim
index 6d34e9ea0..23c816498 100644
--- a/compiler/rod.nim
+++ b/compiler/rod.nim
@@ -14,6 +14,7 @@ import ast, idgen
 when not defined(nimSymbolfiles):
   template setupModuleCache* = discard
   template storeNode*(module: PSym; n: PNode) = discard
+  template loadNode*(module: PSym; index: var int): PNode = discard
 
   template getModuleId*(fullpath: string): int = getID()
 
diff --git a/compiler/rodimpl.nim b/compiler/rodimpl.nim
index aca576b23..a6f525787 100644
--- a/compiler/rodimpl.nim
+++ b/compiler/rodimpl.nim
@@ -9,8 +9,8 @@
 
 ## This module implements the canonalization for the various caching mechanisms.
 
-import strutils, os, intsets, ropes, db_sqlite, msgs, options, types,
-  renderer, rodutils, std / sha1
+import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
+  renderer, rodutils, std / sha1, idents
 
 var db: DbConn
 
@@ -367,6 +367,381 @@ proc storeNode*(module: PSym; n: PNode) =
       break
     inc i
 
+# ---------------- decoder -----------------------------------
+type
+  TRodReader = object
+    module: PSym
+    #sstack: seq[(PSym, ptr PSym)]       # a stack of symbols to process
+    #tstack: seq[(PType, ptr PType)]     # a stack of types to process
+
+    #tmarks, smarks: IntSet
+    syms: Table[int, PSym] ## XXX make this more efficients
+    types: Table[int, PType]
+    cache: IdentCache
+
+  BlobReader = object
+    s: string
+    pos: int
+
+  PRodReader = var TRodReader
+
+proc initRodReader(cache: IdentCache): TRodReader =
+  TRodReader(module: nil,
+    syms: initTable[int, PSym](), types: initTable[int, PType](),
+    cache: cache)
+
+var gr = initRodReader(newIdentCache())
+
+using
+  r: PRodReader
+  b: var BlobReader
+
+proc loadSym(r; id: int, info: TLineInfo): PSym
+proc loadType(r; id: int, info: TLineInfo): PType
+
+proc decodeLineInfo(r; b; info: var TLineInfo) =
+  if b.s[b.pos] == '?':
+    inc(b.pos)
+    if b.s[b.pos] == ',': info.col = -1'i16
+    else: info.col = int16(decodeVInt(b.s, b.pos))
+    if b.s[b.pos] == ',':
+      inc(b.pos)
+      if b.s[b.pos] == ',': info.line = -1'i16
+      else: info.line = int16(decodeVInt(b.s, b.pos))
+      if b.s[b.pos] == ',':
+        inc(b.pos)
+        info.fileIndex = int32(decodeVInt(b.s, b.pos))
+
+proc skipNode(b) =
+  assert b.s[b.pos] == '('
+  var par = 0
+  var pos = b.pos+1
+  while true:
+    case b.s[pos]
+    of ')':
+      if par == 0: break
+      dec par
+    of '(': inc par
+    else: discard
+    inc pos
+  b.pos = pos+1 # skip ')'
+
+proc decodeNodeLazyBody(r; b; fInfo: TLineInfo,
+                        belongsTo: PSym): PNode =
+  result = nil
+  if b.s[b.pos] == '(':
+    inc(b.pos)
+    if b.s[b.pos] == ')':
+      inc(b.pos)
+      return                  # nil node
+    result = newNodeI(TNodeKind(decodeVInt(b.s, b.pos)), fInfo)
+    decodeLineInfo(r, b, result.info)
+    if b.s[b.pos] == '$':
+      inc(b.pos)
+      result.flags = cast[TNodeFlags](int32(decodeVInt(b.s, b.pos)))
+    if b.s[b.pos] == '^':
+      inc(b.pos)
+      var id = decodeVInt(b.s, b.pos)
+      result.typ = loadType(r, id, result.info)
+    case result.kind
+    of nkCharLit..nkUInt64Lit:
+      if b.s[b.pos] == '!':
+        inc(b.pos)
+        result.intVal = decodeVBiggestInt(b.s, b.pos)
+    of nkFloatLit..nkFloat64Lit:
+      if b.s[b.pos] == '!':
+        inc(b.pos)
+        var fl = decodeStr(b.s, b.pos)
+        result.floatVal = parseFloat(fl)
+    of nkStrLit..nkTripleStrLit:
+      if b.s[b.pos] == '!':
+        inc(b.pos)
+        result.strVal = decodeStr(b.s, b.pos)
+      else:
+        result.strVal = ""
+    of nkIdent:
+      if b.s[b.pos] == '!':
+        inc(b.pos)
+        var fl = decodeStr(b.s, b.pos)
+        result.ident = r.cache.getIdent(fl)
+      else:
+        internalError(result.info, "decodeNode: nkIdent")
+    of nkSym:
+      if b.s[b.pos] == '!':
+        inc(b.pos)
+        var id = decodeVInt(b.s, b.pos)
+        result.sym = loadSym(r, id, result.info)
+      else:
+        internalError(result.info, "decodeNode: nkSym")
+    else:
+      var i = 0
+      while b.s[b.pos] != ')':
+        when false:
+          if belongsTo != nil and i == bodyPos:
+            addSonNilAllowed(result, nil)
+            belongsTo.offset = b.pos
+            skipNode(b)
+          else:
+            discard
+        addSonNilAllowed(result, decodeNodeLazyBody(r, b, result.info, nil))
+        inc i
+    if b.s[b.pos] == ')': inc(b.pos)
+    else: internalError(result.info, "decodeNode: ')' missing")
+  else:
+    internalError(fInfo, "decodeNode: '(' missing " & $b.pos)
+
+proc decodeNode(r; b; fInfo: TLineInfo): PNode =
+  result = decodeNodeLazyBody(r, b, fInfo, nil)
+
+proc decodeLoc(r; b; loc: var TLoc, info: TLineInfo) =
+  if b.s[b.pos] == '<':
+    inc(b.pos)
+    if b.s[b.pos] in {'0'..'9', 'a'..'z', 'A'..'Z'}:
+      loc.k = TLocKind(decodeVInt(b.s, b.pos))
+    else:
+      loc.k = low(loc.k)
+    if b.s[b.pos] == '*':
+      inc(b.pos)
+      loc.storage = TStorageLoc(decodeVInt(b.s, b.pos))
+    else:
+      loc.storage = low(loc.storage)
+    if b.s[b.pos] == '$':
+      inc(b.pos)
+      loc.flags = cast[TLocFlags](int32(decodeVInt(b.s, b.pos)))
+    else:
+      loc.flags = {}
+    if b.s[b.pos] == '^':
+      inc(b.pos)
+      loc.lode = decodeNode(r, b, info)
+      # rrGetType(b, decodeVInt(b.s, b.pos), info)
+    else:
+      loc.lode = nil
+    if b.s[b.pos] == '!':
+      inc(b.pos)
+      loc.r = rope(decodeStr(b.s, b.pos))
+    else:
+      loc.r = nil
+    if b.s[b.pos] == '>': inc(b.pos)
+    else: internalError(info, "decodeLoc " & b.s[b.pos])
+
+proc loadBlob(query: SqlQuery; id: int): BlobReader =
+  let blob = db.getValue(query, id)
+  if blob.len == 0:
+    internalError("symbolfiles: cannot find ID " & $ id)
+  result = BlobReader(pos: 0)
+  shallowCopy(result.s, blob)
+
+proc loadType(r; id: int; info: TLineInfo): PType =
+  result = r.types.getOrDefault(id)
+  if result != nil: return result
+  var b = loadBlob(sql"select data from types where nimid = ?", id)
+
+  if b.s[b.pos] == '[':
+    inc(b.pos)
+    if b.s[b.pos] == ']':
+      inc(b.pos)
+      return                  # nil type
+  new(result)
+  result.kind = TTypeKind(decodeVInt(b.s, b.pos))
+  if b.s[b.pos] == '+':
+    inc(b.pos)
+    result.id = decodeVInt(b.s, b.pos)
+    setId(result.id)
+    #if debugIds: registerID(result)
+  else:
+    internalError(info, "decodeType: no id")
+  # here this also avoids endless recursion for recursive type
+  r.types[result.id] = result
+  if b.s[b.pos] == '(': result.n = decodeNode(r, b, unknownLineInfo())
+  if b.s[b.pos] == '$':
+    inc(b.pos)
+    result.flags = cast[TTypeFlags](int32(decodeVInt(b.s, b.pos)))
+  if b.s[b.pos] == '?':
+    inc(b.pos)
+    result.callConv = TCallingConvention(decodeVInt(b.s, b.pos))
+  if b.s[b.pos] == '*':
+    inc(b.pos)
+    result.owner = loadSym(r, decodeVInt(b.s, b.pos), info)
+  if b.s[b.pos] == '&':
+    inc(b.pos)
+    result.sym = loadSym(r, decodeVInt(b.s, b.pos), info)
+  if b.s[b.pos] == '/':
+    inc(b.pos)
+    result.size = decodeVInt(b.s, b.pos)
+  else:
+    result.size = - 1
+  if b.s[b.pos] == '=':
+    inc(b.pos)
+    result.align = decodeVInt(b.s, b.pos).int16
+  else:
+    result.align = 2
+
+  if b.s[b.pos] == '\14':
+    inc(b.pos)
+    result.lockLevel = decodeVInt(b.s, b.pos).TLockLevel
+  else:
+    result.lockLevel = UnspecifiedLockLevel
+
+  if b.s[b.pos] == '\15':
+    inc(b.pos)
+    result.destructor = loadSym(r, decodeVInt(b.s, b.pos), info)
+  if b.s[b.pos] == '\16':
+    inc(b.pos)
+    result.deepCopy = loadSym(r, decodeVInt(b.s, b.pos), info)
+  if b.s[b.pos] == '\17':
+    inc(b.pos)
+    result.assignment = loadSym(r, decodeVInt(b.s, b.pos), info)
+  if b.s[b.pos] == '\18':
+    inc(b.pos)
+    result.sink = loadSym(r, decodeVInt(b.s, b.pos), info)
+  while b.s[b.pos] == '\19':
+    inc(b.pos)
+    let x = decodeVInt(b.s, b.pos)
+    doAssert b.s[b.pos] == '\20'
+    inc(b.pos)
+    let y = loadSym(r, decodeVInt(b.s, b.pos), info)
+    result.methods.safeAdd((x, y))
+  decodeLoc(r, b, result.loc, info)
+  while b.s[b.pos] == '^':
+    inc(b.pos)
+    if b.s[b.pos] == '(':
+      inc(b.pos)
+      if b.s[b.pos] == ')': inc(b.pos)
+      else: internalError(info, "decodeType ^(" & b.s[b.pos])
+      rawAddSon(result, nil)
+    else:
+      var d = decodeVInt(b.s, b.pos)
+      rawAddSon(result, loadType(r, d, info))
+
+proc decodeLib(r; b; info: TLineInfo): PLib =
+  result = nil
+  if b.s[b.pos] == '|':
+    new(result)
+    inc(b.pos)
+    result.kind = TLibKind(decodeVInt(b.s, b.pos))
+    if b.s[b.pos] != '|': internalError("decodeLib: 1")
+    inc(b.pos)
+    result.name = rope(decodeStr(b.s, b.pos))
+    if b.s[b.pos] != '|': internalError("decodeLib: 2")
+    inc(b.pos)
+    result.path = decodeNode(r, b, info)
+
+proc decodeInstantiations(r; b; info: TLineInfo;
+                          s: var seq[PInstantiation]) =
+  while b.s[b.pos] == '\15':
+    inc(b.pos)
+    var ii: PInstantiation
+    new ii
+    ii.sym = loadSym(r, decodeVInt(b.s, b.pos), info)
+    ii.concreteTypes = @[]
+    while b.s[b.pos] == '\17':
+      inc(b.pos)
+      ii.concreteTypes.add loadType(r, decodeVInt(b.s, b.pos), info)
+    if b.s[b.pos] == '\20':
+      inc(b.pos)
+      ii.compilesId = decodeVInt(b.s, b.pos)
+    s.safeAdd ii
+
+proc loadSym(r; id: int; info: TLineInfo): PSym =
+  var
+    id: int
+    ident: PIdent
+  result = r.syms.getOrDefault(id)
+  if result != nil: return result
+  var b = loadBlob(sql"select data from syms where nimid = ?", id)
+  if b.s[b.pos] == '{':
+    inc(b.pos)
+    if b.s[b.pos] == '}':
+      inc(b.pos)
+      return                  # nil sym
+  var k = TSymKind(decodeVInt(b.s, b.pos))
+  if b.s[b.pos] == '+':
+    inc(b.pos)
+    id = decodeVInt(b.s, b.pos)
+    setId(id)
+  else:
+    internalError(info, "decodeSym: no id")
+  if b.s[b.pos] == '&':
+    inc(b.pos)
+    ident = r.cache.getIdent(decodeStr(b.s, b.pos))
+  else:
+    internalError(info, "decodeSym: no ident")
+  #echo "decoding: {", ident.s
+  new(result)
+  result.id = id
+  result.kind = k
+  result.name = ident         # read the rest of the symbol description:
+  r.syms[result.id] = result
+  if b.s[b.pos] == '^':
+    inc(b.pos)
+    result.typ = loadType(r, decodeVInt(b.s, b.pos), info)
+  decodeLineInfo(r, b, result.info)
+  if b.s[b.pos] == '*':
+    inc(b.pos)
+    result.owner = loadSym(r, decodeVInt(b.s, b.pos), result.info)
+  if b.s[b.pos] == '$':
+    inc(b.pos)
+    result.flags = cast[TSymFlags](int32(decodeVInt(b.s, b.pos)))
+  if b.s[b.pos] == '@':
+    inc(b.pos)
+    result.magic = TMagic(decodeVInt(b.s, b.pos))
+  if b.s[b.pos] == '!':
+    inc(b.pos)
+    result.options = cast[TOptions](int32(decodeVInt(b.s, b.pos)))
+  else:
+    result.options = r.module.options
+  if b.s[b.pos] == '%':
+    inc(b.pos)
+    result.position = decodeVInt(b.s, b.pos)
+  if b.s[b.pos] == '`':
+    inc(b.pos)
+    result.offset = decodeVInt(b.s, b.pos)
+  else:
+    result.offset = - 1
+  decodeLoc(r, b, result.loc, result.info)
+  result.annex = decodeLib(r, b, info)
+  if b.s[b.pos] == '#':
+    inc(b.pos)
+    result.constraint = decodeNode(r, b, unknownLineInfo())
+  case result.kind
+  of skType, skGenericParam:
+    while b.s[b.pos] == '\14':
+      inc(b.pos)
+      result.typeInstCache.safeAdd loadType(r, decodeVInt(b.s, b.pos), result.info)
+  of routineKinds:
+    decodeInstantiations(r, b, result.info, result.procInstCache)
+    if b.s[b.pos] == '\16':
+      inc(b.pos)
+      result.gcUnsafetyReason = loadSym(r, decodeVInt(b.s, b.pos), result.info)
+  of skModule, skPackage:
+    decodeInstantiations(r, b, result.info, result.usedGenerics)
+  of skLet, skVar, skField, skForVar:
+    if b.s[b.pos] == '\18':
+      inc(b.pos)
+      result.guard = loadSym(r, decodeVInt(b.s, b.pos), result.info)
+    if b.s[b.pos] == '\19':
+      inc(b.pos)
+      result.bitsize = decodeVInt(b.s, b.pos).int16
+  else: discard
+
+  if b.s[b.pos] == '(':
+    #if result.kind in routineKinds:
+    #  result.ast = decodeNodeLazyBody(b, result.info, result)
+    #else:
+    result.ast = decodeNode(r, b, result.info)
+
+proc loadNode*(module: PSym; index: var int): PNode =
+  if index == 0:
+    index = parseInt db.getValue(
+      sql"select min(id) from toplevelstmts where module = ?", abs module.id)
+  var b = BlobReader(pos: 0)
+  b.s = db.getValue(sql"select data from toplevelstmts where id = ?", index)
+  if b.s.len == 0: return nil # end marker
+  gr.module = module
+  result = decodeNode(gr, b, module.info)
+
+# --------------- Database model ---------------------------------------------
+
 proc createDb() =
   db.exec(sql"""
     create table if not exists controlblock(