# # # The Nim Compiler # (c) Copyright 2018 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ## This module implements the new compilation cache. import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types, renderer, rodutils, idents, astalgo, btrees, magicsys, cgmeth, extccomp, btrees, trees, condsyms, nversion ## Todo: ## - Dependency computation should use *signature* hashes in order to ## avoid recompiling dependent modules. ## - Patch the rest of the compiler to do lazy loading of proc bodies. ## - Patch the C codegen to cache proc bodies and maybe types. template db(): DbConn = g.incr.db proc encodeConfig(g: ModuleGraph): string = result = newStringOfCap(100) result.add RodFileVersion for d in definedSymbolNames(g.config.symbols): result.add ' ' result.add d template serialize(field) = result.add ' ' result.add($g.config.field) depConfigFields(serialize) proc needsRecompile(g: ModuleGraph; fileIdx: FileIndex; fullpath: string; cycleCheck: var IntSet): bool = let root = db.getRow(sql"select id, fullhash from filenames where fullpath = ?", fullpath) if root[0].len == 0: return true if root[1] != hashFileCached(g.config, fileIdx, fullpath): return true # cycle detection: assume "not changed" is correct. if cycleCheck.containsOrIncl(int fileIdx): return false # check dependencies (recursively): for row in db.fastRows(sql"select fullpath from filenames where id in (select dependency from deps where module = ?)", root[0]): let dep = row[0] if needsRecompile(g, g.config.fileInfoIdx(dep), dep, cycleCheck): return true return false proc getModuleId*(g: ModuleGraph; fileIdx: FileIndex; fullpath: string): int = if g.config.symbolFiles in {disabledSf, writeOnlySf} or g.incr.configChanged: return getID() let module = g.incr.db.getRow( sql"select id, fullHash, nimid from modules where fullpath = ?", fullpath) let currentFullhash = hashFileCached(g.config, fileIdx, fullpath) if module[0].len == 0: result = getID() db.exec(sql"insert into modules(fullpath, interfHash, fullHash, nimid) values (?, ?, ?, ?)", fullpath, "", currentFullhash, result) else: result = parseInt(module[2]) if currentFullhash == module[1]: # not changed, so use the cached AST: doAssert(result != 0) var cycleCheck = initIntSet() if not needsRecompile(g, fileIdx, fullpath, cycleCheck): echo "cached successfully! ", fullpath return -result db.exec(sql"update modules set fullHash = ? where id = ?", currentFullhash, module[0]) db.exec(sql"delete from deps where module = ?", module[0]) db.exec(sql"delete from types where module = ?", module[0]) db.exec(sql"delete from syms where module = ?", module[0]) db.exec(sql"delete from toplevelstmts where module = ?", module[0]) db.exec(sql"delete from statics where module = ?", module[0]) proc pushType(w: var Writer, t: PType) = if not containsOrIncl(w.tmarks, t.id): w.tstack.add(t) proc pushSym(w: var Writer, s: PSym) = if not containsOrIncl(w.smarks, s.id): w.sstack.add(s) template w: untyped = g.incr.w proc encodeNode(g: ModuleGraph; fInfo: TLineInfo, n: PNode, result: var string) = if n == nil: # nil nodes have to be stored too: result.add("()") return result.add('(') encodeVInt(ord(n.kind), result) # we do not write comments for now # Line information takes easily 20% or more of the filesize! Therefore we # omit line information if it is the same as the parent's line information: if fInfo.fileIndex != n.info.fileIndex: result.add('?') encodeVInt(n.info.col, result) result.add(',') encodeVInt(int n.info.line, result) result.add(

discard """
  output: '''int: 108'''
"""

# bug #4070

proc id(f: (proc())): auto =
  return f

proc foo(myinteger: int): (iterator(): int) =
  return iterator(): int {.closure.} =
           proc bar() =
             proc kk() =
               echo "int: ", myinteger

             kk()

           id(bar)()

discard foo(108)()

, abs(module.id), module.offset, buf) inc module.offset var i = 0 while true: if i > 10_000: doAssert false, "loop never ends!" if w.sstack.len > 0: let s = w.sstack.pop() when false: echo "popped ", s.name.s, " ", s.id storeSym(g, s) elif w.tstack.len > 0: let t = w.tstack.pop() storeType(g, t) when false: echo "popped type ", typeToString(t), " ", t.id else: break inc i proc recordStmt*(g: ModuleGraph; module: PSym; n: PNode) = storeNode(g, module, n) proc storeRemaining*(g: ModuleGraph; module: PSym) = if g.config.symbolFiles == disabledSf: return var stillForwarded: seq[PSym] = @[] for s in w.forwardedSyms: if sfForward notin s.flags: storeSym(g, s) else: stillForwarded.add s swap w.forwardedSyms, stillForwarded # ---------------- decoder ----------------------------------- type BlobReader = object s: string pos: int using b: var BlobReader g: ModuleGraph proc loadSym(g; id: int, info: TLineInfo): PSym proc loadType(g; id: int, info: TLineInfo): PType proc decodeLineInfo(g; b; info: var TLineInfo) = if b.s[b.pos] == '?': inc(b.pos) if b.s[b.pos] == ',': info.col = -1'i16 else: info.col = int16(decodeVInt(b.s, b.pos)) if b.s[b.pos] == ',': inc(b.pos) if b.s[b.pos] == ',': info.line = 0'u16 else: info.line = uint16(decodeVInt(b.s, b.pos)) if b.s[b.pos] == ',': inc(b.pos) info.fileIndex = fromDbFileId(g.incr, g.config, decodeVInt(b.s, b.pos)) proc skipNode(b) = assert b.s[b.pos] == '(' var par = 0 var pos = b.pos+1 while true: case b.s[pos] of ')': if par == 0: break dec par of '(': inc par else: discard inc pos b.pos = pos+1 # skip ')' proc decodeNodeLazyBody(g; b; fInfo: TLineInfo, belongsTo: PSym): PNode = result = nil if b.s[b.pos] == '(': inc(b.pos) if b.s[b.pos] == ')': inc(b.pos) return # nil node result = newNodeI(TNodeKind(decodeVInt(b.s, b.pos)), fInfo) decodeLineInfo(g, b, result.info) if b.s[b.pos] == '$': inc(b.pos) result.flags = cast[TNodeFlags](int32(decodeVInt(b.s, b.pos))) if b.s[b.pos] == '^': inc(b.pos) var id = decodeVInt(b.s, b.pos) result.typ = loadType(g, id, result.info) case result.kind of nkCharLit..nkUInt64Lit: if b.s[b.pos] == '!': inc(b.pos) result.intVal = decodeVBiggestInt(b.s, b.pos) of nkFloatLit..nkFloat64Lit: if b.s[b.pos] == '!': inc(b.pos) var fl = decodeStr(b.s, b.pos) result.floatVal = parseFloat(fl) of nkStrLit..nkTripleStrLit: if b.s[b.pos] == '!': inc(b.pos) result.strVal = decodeStr(b.s, b.pos) else: result.strVal = "" of nkIdent: if b.s[b.pos] == '!': inc(b.pos) var fl = decodeStr(b.s, b.pos) result.ident = g.cache.getIdent(fl) else: internalError(g.config, result.info, "decodeNode: nkIdent") of nkSym: if b.s[b.pos] == '!': inc(b.pos) var id = decodeVInt(b.s, b.pos) result.sym = loadSym(g, id, result.info) else: internalError(g.config, result.info, "decodeNode: nkSym") else: var i = 0 while b.s[b.pos] != ')': when false: if belongsTo != nil and i == bodyPos: addSonNilAllowed(result, nil) belongsTo.offset = b.pos skipNode(b) else: discard addSonNilAllowed(result, decodeNodeLazyBody(g, b, result.info, nil)) inc i if b.s[b.pos] == ')': inc(b.pos) else: internalError(g.config, result.info, "decodeNode: ')' missing") else: internalError(g.config, fInfo, "decodeNode: '(' missing " & $b.pos) proc decodeNode(g; b; fInfo: TLineInfo): PNode = result = decodeNodeLazyBody(g, b, fInfo, nil) proc decodeLoc(g; b; loc: var TLoc, info: TLineInfo) = if b.s[b.pos] == '<': inc(b.pos) if b.s[b.pos] in {'0'..'9', 'a'..'z', 'A'..'Z'}: loc.k = TLocKind(decodeVInt(b.s, b.pos)) else: loc.k = low(loc.k) if b.s[b.pos] == '*': inc(b.pos) loc.storage = TStorageLoc(decodeVInt(b.s, b.pos)) else: loc.storage = low(loc.storage) if b.s[b.pos] == '$': inc(b.pos) loc.flags = cast[TLocFlags](int32(decodeVInt(b.s, b.pos))) else: loc.flags = {} if b.s[b.pos] == '^': inc(b.pos) loc.lode = decodeNode(g, b, info) # rrGetType(b, decodeVInt(b.s, b.pos), info) else: loc.lode = nil if b.s[b.pos] == '!': inc(b.pos) loc.r = rope(decodeStr(b.s, b.pos)) else: loc.r = nil if b.s[b.pos] == '>': inc(b.pos) else: internalError(g.config, info, "decodeLoc " & b.s[b.pos]) proc loadBlob(g; query: SqlQuery; id: int): BlobReader = let blob = db.getValue(query, id) if blob.len == 0: internalError(g.config, "symbolfiles: cannot find ID " & $ id) result = BlobReader(pos: 0) shallowCopy(result.s, blob) # ensure we can read without index checks: result.s.add '\0' proc loadType(g; id: int; info: TLineInfo): PType = result = g.incr.r.types.getOrDefault(id) if result != nil: return result var b = loadBlob(g, sql"select data from types where nimid = ?", id) if b.s[b.pos] == '[': inc(b.pos) if b.s[b.pos] == ']': inc(b.pos) return # nil type new(result) result.kind = TTypeKind(decodeVInt(b.s, b.pos)) if b.s[b.pos] == '+': inc(b.pos) result.id = decodeVInt(b.s, b.pos) setId(result.id) #if debugIds: registerID(result) else: internalError(g.config, info, "decodeType: no id") # here this also avoids endless recursion for recursive type g.incr.r.types.add(result.id, result) if b.s[b.pos] == '(': result.n = decodeNode(g, b, unknownLineInfo()) if b.s[b.pos] == '$': inc(b.pos) result.flags = cast[TTypeFlags](int32(decodeVInt(b.s, b.pos))) if b.s[b.pos] == '?': inc(b.pos) result.callConv = TCallingConvention(decodeVInt(b.s, b.pos)) if b.s[b.pos] == '*': inc(b.pos) result.owner = loadSym(g, decodeVInt(b.s, b.pos), info) if b.s[b.pos] == '&': inc(b.pos) result.sym = loadSym(g, decodeVInt(b.s, b.pos), info) if b.s[b.pos] == '/': inc(b.pos) result.size = decodeVInt(b.s, b.pos) else: result.size = -1 if b.s[b.pos] == '=': inc(b.pos) result.align = decodeVInt(b.s, b.pos).int16 else: result.align = 2 if b.s[b.pos] == '\14': inc(b.pos) result.lockLevel = decodeVInt(b.s, b.pos).TLockLevel else: result.lockLevel = UnspecifiedLockLevel if b.s[b.pos] == '\15': inc(b.pos) result.destructor = loadSym(g, decodeVInt(b.s, b.pos), info) if b.s[b.pos] == '\16': inc(b.pos) result.deepCopy = loadSym(g, decodeVInt(b.s, b.pos), info) if b.s[b.pos] == '\17': inc(b.pos) result.assignment = loadSym(g, decodeVInt(b.s, b.pos), info) if b.s[b.pos] == '\18': inc(b.pos) result.sink = loadSym(g, decodeVInt(b.s, b.pos), info) while b.s[b.pos] == '\19': inc(b.pos) let x = decodeVInt(b.s, b.pos) doAssert b.s[b.pos] == '\20' inc(b.pos) let y = loadSym(g, decodeVInt(b.s, b.pos), info) result.methods.safeAdd((x, y)) decodeLoc(g, b, result.loc, info) while b.s[b.pos] == '^': inc(b.pos) if b.s[b.pos] == '(': inc(b.pos) if b.s[b.pos] == ')': inc(b.pos) else: internalError(g.config, info, "decodeType ^(" & b.s[b.pos]) rawAddSon(result, nil) else: let d = decodeVInt(b.s, b.pos) rawAddSon(result, loadType(g, d, info)) proc decodeLib(g; b; info: TLineInfo): PLib = result = nil if b.s[b.pos] == '|': new(result) inc(b.pos) result.kind = TLibKind(decodeVInt(b.s, b.pos)) if b.s[b.pos] != '|': internalError(g.config, "decodeLib: 1") inc(b.pos) result.name = rope(decodeStr(b.s, b.pos)) if b.s[b.pos] != '|': internalError(g.config, "decodeLib: 2") inc(b.pos) result.path = decodeNode(g, b, info) proc decodeInstantiations(g; b; info: TLineInfo; s: var seq[PInstantiation]) = while b.s[b.pos] == '\15': inc(b.pos) var ii: PInstantiation new ii ii.sym = loadSym(g, decodeVInt(b.s, b.pos), info) ii.concreteTypes = @[] while b.s[b.pos] == '\17': inc(b.pos) ii.concreteTypes.add loadType(g, decodeVInt(b.s, b.pos), info) if b.s[b.pos] == '\20': inc(b.pos) ii.compilesId = decodeVInt(b.s, b.pos) s.safeAdd ii proc loadSymFromBlob(g; b; info: TLineInfo): PSym = if b.s[b.pos] == '{': inc(b.pos) if b.s[b.pos] == '}': inc(b.pos) return # nil sym var k = TSymKind(decodeVInt(b.s, b.pos)) var id: int if b.s[b.pos] == '+': inc(b.pos) id = decodeVInt(b.s, b.pos) setId(id) else: internalError(g.config, info, "decodeSym: no id") var ident: PIdent if b.s[b.pos] == '&': inc(b.pos) ident = g.cache.getIdent(decodeStr(b.s, b.pos)) else: internalError(g.config, info, "decodeSym: no ident") #echo "decoding: {", ident.s new(result) result.id = id result.kind = k result.name = ident # read the rest of the symbol description: g.incr.r.syms.add(result.id, result) if b.s[b.pos] == '^': inc(b.pos) result.typ = loadType(g, decodeVInt(b.s, b.pos), info) decodeLineInfo(g, b, result.info) if b.s[b.pos] == '*': inc(b.pos) result.owner = loadSym(g, decodeVInt(b.s, b.pos), result.info) if b.s[b.pos] == '$': inc(b.pos) result.flags = cast[TSymFlags](int32(decodeVInt(b.s, b.pos))) if b.s[b.pos] == '@': inc(b.pos) result.magic = TMagic(decodeVInt(b.s, b.pos)) if b.s[b.pos] == '!': inc(b.pos) result.options = cast[TOptions](int32(decodeVInt(b.s, b.pos))) if b.s[b.pos] == '%': inc(b.pos) result.position = decodeVInt(b.s, b.pos) if b.s[b.pos] == '`': inc(b.pos) result.offset = decodeVInt(b.s, b.pos) else: result.offset = -1 decodeLoc(g, b, result.loc, result.info) result.annex = decodeLib(g, b, info) if b.s[b.pos] == '#': inc(b.pos) result.constraint = decodeNode(g, b, unknownLineInfo()) case result.kind of skType, skGenericParam: while b.s[b.pos] == '\14': inc(b.pos) result.typeInstCache.safeAdd loadType(g, decodeVInt(b.s, b.pos), result.info) of routineKinds: decodeInstantiations(g, b, result.info, result.procInstCache) if b.s[b.pos] == '\16': inc(b.pos) result.gcUnsafetyReason = loadSym(g, decodeVInt(b.s, b.pos), result.info) of skModule, skPackage: decodeInstantiations(g, b, result.info, result.usedGenerics) of skLet, skVar, skField, skForVar: if b.s[b.pos] == '\18': inc(b.pos) result.guard = loadSym(g, decodeVInt(b.s, b.pos), result.info) if b.s[b.pos] == '\19': inc(b.pos) result.bitsize = decodeVInt(b.s, b.pos).int16 else: discard if b.s[b.pos] == '(': #if result.kind in routineKinds: # result.ast = decodeNodeLazyBody(b, result.info, result) #else: result.ast = decodeNode(g, b, result.info) if sfCompilerProc in result.flags: registerCompilerProc(g, result) #echo "loading ", result.name.s proc loadSym(g; id: int; info: TLineInfo): PSym = result = g.incr.r.syms.getOrDefault(id) if result != nil: return result var b = loadBlob(g, sql"select data from syms where nimid = ?", id) result = loadSymFromBlob(g, b, info) doAssert id == result.id, "symbol ID is not consistent!" proc loadModuleSymTab(g; module: PSym) = ## goal: fill module.tab g.incr.r.syms.add(module.id, module) for row in db.fastRows(sql"select nimid, data from syms where module = ? and exported = 1", abs(module.id)): let id = parseInt(row[0]) var s = g.incr.r.syms.getOrDefault(id) if s == nil: var b = BlobReader(pos: 0) shallowCopy(b.s, row[1]) # ensure we can read without index checks: b.s.add '\0' s = loadSymFromBlob(g, b, module.info) assert s != nil strTableAdd(module.tab, s) if sfSystemModule in module.flags: g.systemModule = module proc replay(g: ModuleGraph; module: PSym; n: PNode) = # XXX check if we need to replay nkStaticStmt here. case n.kind #of nkStaticStmt: #evalStaticStmt(module, g, n[0], module) #of nkVarSection, nkLetSection: # nkVarSections are already covered by the vmgen which produces nkStaticStmt of nkMethodDef: methodDef(g, n[namePos].sym, fromCache=true) of nkCommentStmt: # pragmas are complex and can be user-overriden via templates. So # instead of using the original ``nkPragma`` nodes, we rely on the # fact that pragmas.nim was patched to produce specialized recorded # statements for us in the form of ``nkCommentStmt`` with (key, value) # pairs. Ordinary nkCommentStmt nodes never have children so this is # not ambiguous. # Fortunately only a tiny subset of the available pragmas need to # be replayed here. This is always a subset of ``pragmas.stmtPragmas``. if n.len >= 2: internalAssert g.config, n[0].kind == nkStrLit and n[1].kind == nkStrLit case n[0].strVal of "hint": message(g.config, n.info, hintUser, n[1].strVal) of "warning": message(g.config, n.info, warnUser, n[1].strVal) of "error": localError(g.config, n.info, errUser, n[1].strVal) of "compile": internalAssert g.config, n.len == 3 and n[2].kind == nkStrLit var cf = Cfile(cname: n[1].strVal, obj: n[2].strVal, flags: {CfileFlag.External}) extccomp.addExternalFileToCompile(g.config, cf) of "link": extccomp.addExternalFileToLink(g.config, n[1].strVal) of "passl": extccomp.addLinkOption(g.config, n[1].strVal) of "passc": extccomp.addCompileOption(g.config, n[1].strVal) of "cppdefine": options.cppDefine(g.config, n[1].strVal) of "inc": let destKey = n[1].strVal let by = n[2].intVal let v = getOrDefault(g.cacheCounters, destKey) g.cacheCounters[destKey] = v+by of "put": let destKey = n[1].strVal let key = n[2].strVal let val = n[3] if not contains(g.cacheTables, destKey): g.cacheTables[destKey] = initBTree[string, PNode]() if not contains(g.cacheTables[destKey], key): g.cacheTables[destKey].add(key, val) else: internalError(g.config, n.info, "key already exists: " & key) of "incl": let destKey = n[1].strVal let val = n[2] if not contains(g.cacheSeqs, destKey): g.cacheSeqs[destKey] = newTree(nkStmtList, val) else: block search: for existing in g.cacheSeqs[destKey]: if exprStructuralEquivalent(existing, val, strictSymEquality=true): break search g.cacheSeqs[destKey].add val of "add": let destKey = n[1].strVal let val = n[2] if not contains(g.cacheSeqs, destKey): g.cacheSeqs[destKey] = newTree(nkStmtList, val) else: g.cacheSeqs[destKey].add val else: internalAssert g.config, false of nkImportStmt: for x in n: internalAssert g.config, x.kind == nkStrLit let imported = g.importModuleCallback(g, module, fileInfoIdx(g.config, n[0].strVal)) internalAssert g.config, imported.id < 0 of nkStmtList, nkStmtListExpr: for x in n: replay(g, module, x) else: discard "nothing to do for this node" proc loadNode*(g: ModuleGraph; module: PSym): PNode = loadModuleSymTab(g, module) result = newNodeI(nkStmtList, module.info) for row in db.rows(sql"select data from toplevelstmts where module = ? order by position asc", abs module.id): var b = BlobReader(pos: 0) # ensure we can read without index checks: b.s = row[0] & '\0' result.add decodeNode(g, b, module.info) db.exec(sql"insert into controlblock(idgen) values (?)", gFrontEndId) replay(g, module, result) proc setupModuleCache*(g: ModuleGraph) = if g.config.symbolFiles == disabledSf: return g.recordStmt = recordStmt let dbfile = getNimcacheDir(g.config) / "rodfiles.db" if g.config.symbolFiles == writeOnlySf: removeFile(dbfile) if not fileExists(dbfile): db = open(connection=dbfile, user="nim", password="", database="nim") createDb(db) db.exec(sql"insert into config(config) values (?)", encodeConfig(g)) else: db = open(connection=dbfile, user="nim", password="", database="nim") let oldConfig = db.getValue(sql"select config from config") g.incr.configChanged = oldConfig != encodeConfig(g) db.exec(sql"pragma journal_mode=off") db.exec(sql"pragma SYNCHRONOUS=off") db.exec(sql"pragma LOCKING_MODE=exclusive") let lastId = db.getValue(sql"select max(idgen) from controlblock") if lastId.len > 0: idgen.setId(parseInt lastId)