From f3b8d922169bcf99d7430854d53765082f3ad5a8 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 15 Feb 2018 15:42:14 +0100 Subject: new .rod file implementation; part 1: writing of the file --- compiler/rodimpl.nim | 452 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 452 insertions(+) create mode 100644 compiler/rodimpl.nim (limited to 'compiler/rodimpl.nim') diff --git a/compiler/rodimpl.nim b/compiler/rodimpl.nim new file mode 100644 index 000000000..aca576b23 --- /dev/null +++ b/compiler/rodimpl.nim @@ -0,0 +1,452 @@ +# +# +# The Nim Compiler +# (c) Copyright 2018 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements the canonalization for the various caching mechanisms. + +import strutils, os, intsets, ropes, db_sqlite, msgs, options, types, + renderer, rodutils, std / sha1 + +var db: DbConn + +proc getModuleId*(fullpath: string): int = + if gSymbolFiles != v2Sf: return getID() + let module = db.getRow( + sql"select id, fullHash from modules where fullpath = ?", fullpath) + let currentFullhash = $secureHashFile(fullpath) + if module[0].len == 0: + result = int db.insertID(sql"insert into modules(fullpath, interfHash, fullHash) values (?, ?)", + fullpath, "", currentFullhash) + else: + result = parseInt(module[0]) + if currentFullhash == module[1]: + # not changed, so use the cached AST (even if it might be wrong + # due to its dependencies): + doAssert(result != 0) + result = -result + else: + db.exec(sql"update modules set fullHash = ? where id = ?", currentFullhash, module[0]) + db.exec(sql"delete from types where module = ?", module[0]) + db.exec(sql"delete from syms where module = ?", module[0]) + db.exec(sql"delete from toplevelstmts where module = ?", module[0]) + db.exec(sql"delete from statics where module = ?", module[0]) + +type + TRodWriter = object + module: PSym + sstack: seq[PSym] # a stack of symbols to process + tstack: seq[PType] # a stack of types to process + tmarks, smarks: IntSet + + PRodWriter = var TRodWriter + +proc initRodWriter(module: PSym): TRodWriter = + result = TRodWriter(module: module, sstack: @[], tstack: @[], + tmarks: initIntSet(), smarks: initIntSet()) + +when false: + proc getDefines(): string = + result = "" + for d in definedSymbolNames(): + if result.len != 0: add(result, " ") + add(result, d) + + proc addInclDep(w: PRodWriter, dep: string; info: TLineInfo) = + let resolved = dep.findModule(info.toFullPath) + encodeVInt(fileIdx(w, resolved), w.inclDeps) + add(w.inclDeps, " ") + encodeStr($secureHashFile(resolved), w.inclDeps) + add(w.inclDeps, rodNL) + +const + rodNL = "\L" + +proc pushType(w: PRodWriter, t: PType) = + if not containsOrIncl(w.tmarks, t.id): + w.tstack.add(t) + +proc pushSym(w: PRodWriter, s: PSym) = + if not containsOrIncl(w.smarks, s.id): + w.sstack.add(s) + +proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, + result: var string) = + if n == nil: + # nil nodes have to be stored too: + result.add("()") + return + result.add('(') + encodeVInt(ord(n.kind), result) + # we do not write comments for now + # Line information takes easily 20% or more of the filesize! Therefore we + # omit line information if it is the same as the parent's line information: + if fInfo.fileIndex != n.info.fileIndex: + result.add('?') + encodeVInt(n.info.col, result) + result.add(',') + encodeVInt(n.info.line, result) + result.add(',') + encodeVInt(n.info.fileIndex, result) + elif fInfo.line != n.info.line: + result.add('?') + encodeVInt(n.info.col, result) + result.add(',') + encodeVInt(n.info.line, result) + elif fInfo.col != n.info.col: + result.add('?') + encodeVInt(n.info.col, result) + # No need to output the file index, as this is the serialization of one + # file. + let f = n.flags * PersistentNodeFlags + if f != {}: + result.add('$') + encodeVInt(cast[int32](f), result) + if n.typ != nil: + result.add('^') + encodeVInt(n.typ.id, result) + pushType(w, n.typ) + case n.kind + of nkCharLit..nkUInt64Lit: + if n.intVal != 0: + result.add('!') + encodeVBiggestInt(n.intVal, result) + of nkFloatLit..nkFloat64Lit: + if n.floatVal != 0.0: + result.add('!') + encodeStr($n.floatVal, result) + of nkStrLit..nkTripleStrLit: + if n.strVal != "": + result.add('!') + encodeStr(n.strVal, result) + of nkIdent: + result.add('!') + encodeStr(n.ident.s, result) + of nkSym: + result.add('!') + encodeVInt(n.sym.id, result) + pushSym(w, n.sym) + else: + for i in countup(0, sonsLen(n) - 1): + encodeNode(w, n.info, n.sons[i], result) + add(result, ')') + +proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) = + var oldLen = result.len + result.add('<') + if loc.k != low(loc.k): encodeVInt(ord(loc.k), result) + if loc.storage != low(loc.storage): + add(result, '*') + encodeVInt(ord(loc.storage), result) + if loc.flags != {}: + add(result, '$') + encodeVInt(cast[int32](loc.flags), result) + if loc.lode != nil: + add(result, '^') + encodeNode(w, unknownLineInfo(), loc.lode, result) + #encodeVInt(cast[int32](loc.t.id), result) + #pushType(w, loc.t) + if loc.r != nil: + add(result, '!') + encodeStr($loc.r, result) + if oldLen + 1 == result.len: + # no data was necessary, so remove the '<' again: + setLen(result, oldLen) + else: + add(result, '>') + +proc encodeType(w: PRodWriter, t: PType, result: var string) = + if t == nil: + # nil nodes have to be stored too: + result.add("[]") + return + # we need no surrounding [] here because the type is in a line of its own + if t.kind == tyForward: internalError("encodeType: tyForward") + # for the new rodfile viewer we use a preceding [ so that the data section + # can easily be disambiguated: + add(result, '[') + encodeVInt(ord(t.kind), result) + add(result, '+') + encodeVInt(t.id, result) + if t.n != nil: + encodeNode(w, w.module.info, t.n, result) + if t.flags != {}: + add(result, '$') + encodeVInt(cast[int32](t.flags), result) + if t.callConv != low(t.callConv): + add(result, '?') + encodeVInt(ord(t.callConv), result) + if t.owner != nil: + add(result, '*') + encodeVInt(t.owner.id, result) + pushSym(w, t.owner) + if t.sym != nil: + add(result, '&') + encodeVInt(t.sym.id, result) + pushSym(w, t.sym) + if t.size != - 1: + add(result, '/') + encodeVBiggestInt(t.size, result) + if t.align != 2: + add(result, '=') + encodeVInt(t.align, result) + if t.lockLevel.ord != UnspecifiedLockLevel.ord: + add(result, '\14') + encodeVInt(t.lockLevel.int16, result) + if t.destructor != nil and t.destructor.id != 0: + add(result, '\15') + encodeVInt(t.destructor.id, result) + pushSym(w, t.destructor) + if t.deepCopy != nil: + add(result, '\16') + encodeVInt(t.deepcopy.id, result) + pushSym(w, t.deepcopy) + if t.assignment != nil: + add(result, '\17') + encodeVInt(t.assignment.id, result) + pushSym(w, t.assignment) + if t.sink != nil: + add(result, '\18') + encodeVInt(t.sink.id, result) + pushSym(w, t.sink) + for i, s in items(t.methods): + add(result, '\19') + encodeVInt(i, result) + add(result, '\20') + encodeVInt(s.id, result) + pushSym(w, s) + encodeLoc(w, t.loc, result) + for i in countup(0, sonsLen(t) - 1): + if t.sons[i] == nil: + add(result, "^()") + else: + add(result, '^') + encodeVInt(t.sons[i].id, result) + pushType(w, t.sons[i]) + +proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) = + add(result, '|') + encodeVInt(ord(lib.kind), result) + add(result, '|') + encodeStr($lib.name, result) + add(result, '|') + encodeNode(w, info, lib.path, result) + +proc encodeInstantiations(w: PRodWriter; s: seq[PInstantiation]; + result: var string) = + for t in s: + result.add('\15') + encodeVInt(t.sym.id, result) + pushSym(w, t.sym) + for tt in t.concreteTypes: + result.add('\17') + encodeVInt(tt.id, result) + pushType(w, tt) + result.add('\20') + encodeVInt(t.compilesId, result) + +proc encodeSym(w: PRodWriter, s: PSym, result: var string) = + if s == nil: + # nil nodes have to be stored too: + result.add("{}") + return + # we need no surrounding {} here because the symbol is in a line of its own + encodeVInt(ord(s.kind), result) + result.add('+') + encodeVInt(s.id, result) + result.add('&') + encodeStr(s.name.s, result) + if s.typ != nil: + result.add('^') + encodeVInt(s.typ.id, result) + pushType(w, s.typ) + result.add('?') + if s.info.col != -1'i16: encodeVInt(s.info.col, result) + result.add(',') + if s.info.line != -1'i16: encodeVInt(s.info.line, result) + result.add(',') + encodeVInt(s.info.fileIndex, result) + if s.owner != nil: + result.add('*') + encodeVInt(s.owner.id, result) + pushSym(w, s.owner) + if s.flags != {}: + result.add('$') + encodeVInt(cast[int32](s.flags), result) + if s.magic != mNone: + result.add('@') + encodeVInt(ord(s.magic), result) + result.add('!') + encodeVInt(cast[int32](s.options), result) + if s.position != 0: + result.add('%') + encodeVInt(s.position, result) + if s.offset != - 1: + result.add('`') + encodeVInt(s.offset, result) + encodeLoc(w, s.loc, result) + if s.annex != nil: encodeLib(w, s.annex, s.info, result) + if s.constraint != nil: + add(result, '#') + encodeNode(w, unknownLineInfo(), s.constraint, result) + case s.kind + of skType, skGenericParam: + for t in s.typeInstCache: + result.add('\14') + encodeVInt(t.id, result) + pushType(w, t) + of routineKinds: + encodeInstantiations(w, s.procInstCache, result) + if s.gcUnsafetyReason != nil: + result.add('\16') + encodeVInt(s.gcUnsafetyReason.id, result) + pushSym(w, s.gcUnsafetyReason) + of skModule, skPackage: + encodeInstantiations(w, s.usedGenerics, result) + # we don't serialize: + #tab*: TStrTable # interface table for modules + of skLet, skVar, skField, skForVar: + if s.guard != nil: + result.add('\18') + encodeVInt(s.guard.id, result) + pushSym(w, s.guard) + if s.bitsize != 0: + result.add('\19') + encodeVInt(s.bitsize, result) + else: discard + # lazy loading will soon reload the ast lazily, so the ast needs to be + # the last entry of a symbol: + if s.ast != nil: + # we used to attempt to save space here by only storing a dummy AST if + # it is not necessary, but Nim's heavy compile-time evaluation features + # make that unfeasible nowadays: + encodeNode(w, s.info, s.ast, result) + +proc storeSym(w: PRodWriter; s: PSym) = + var buf = newStringOfCap(160) + encodeSym(w, s, buf) + # XXX only store the name for exported symbols in order to speed up lookup + # times once we enable the skStub logic. + db.exec(sql"insert into syms(nimid, module, name, data) values (?, ?, ?, ?)", + s.id, abs(w.module.id), s.name.s, buf) + +proc storeType(w: PRodWriter; t: PType) = + var buf = newStringOfCap(160) + encodeType(w, t, buf) + db.exec(sql"insert into types(nimid, module, data) values (?, ?, ?)", + t.id, abs(w.module.id), buf) + +var w = initRodWriter(nil) + +proc storeNode*(module: PSym; n: PNode) = + if gSymbolFiles != v2Sf: return + w.module = module + var buf = newStringOfCap(160) + encodeNode(w, module.info, n, buf) + db.exec(sql"insert into toplevelstmts(module, data) values (?, ?)", + abs(module.id), buf) + var i = 0 + while true: + if i > 10_000: + quit "loop never ends!" + if w.sstack.len > 0: + let s = w.sstack.pop() + when false: + echo "popped ", s.name.s, " ", s.id + storeSym(w, s) + elif w.tstack.len > 0: + let t = w.tstack.pop() + storeType(w, t) + when false: + echo "popped type ", typeToString(t), " ", t.id + else: + break + inc i + +proc createDb() = + db.exec(sql""" + create table if not exists controlblock( + idgen integer not null + ); + """) + + db.exec(sql""" + create table if not exists modules( + id integer primary key, + fullpath varchar(8000) not null, + interfHash varchar(256) not null, + fullHash varchar(256) not null, + + created timestamp not null default (DATETIME('now')) + );""") + db.exec(sql"""create unique index if not exists SymNameIx on modules(fullpath);""") + + db.exec(sql""" + create table if not exists types( + id integer primary key, + nimid integer not null, + module integer not null, + data blob not null, + foreign key (module) references module(id) + ); + """) + db.exec sql"create index TypeByModuleIdx on types(module);" + db.exec sql"create index TypeByNimIdIdx on types(nimid);" + + db.exec(sql""" + create table if not exists syms( + id integer primary key, + nimid integer not null, + module integer not null, + name varchar(256) not null, + data blob not null, + foreign key (module) references module(id) + ); + """) + db.exec sql"create index if not exists SymNameIx on syms(name);" + db.exec sql"create index SymByNameAndModuleIdx on syms(name, module);" + db.exec sql"create index SymByModuleIdx on syms(module);" + db.exec sql"create index SymByNimIdIdx on syms(nimid);" + + + db.exec(sql""" + create table if not exists toplevelstmts( + id integer primary key, + module integer not null, + data blob not null, + foreign key (module) references module(id) + ); + """) + db.exec sql"create index TopLevelStmtByModuleIdx on toplevelstmts(module);" + + + db.exec(sql""" + create table if not exists statics( + id integer primary key, + module integer not null, + data blob not null, + foreign key (module) references module(id) + ); + """) + db.exec sql"create index StaticsByModuleIdx on toplevelstmts(module);" + db.exec sql"insert into controlblock(idgen) values (0)" + +proc setupModuleCache* = + if gSymbolFiles != v2Sf: return + let dbfile = getNimcacheDir() / "rodfiles.db" + if not fileExists(dbfile): + db = open(connection=dbfile, user="nim", password="", + database="nim") + createDb() + else: + db = open(connection=dbfile, user="nim", password="", + database="nim") + db.exec(sql"pragma journal_mode=off") + db.exec(sql"pragma SYNCHRONOUS=off") + db.exec(sql"pragma LOCKING_MODE=exclusive") + idgen.setId(parseInt db.getValue( + sql"select max(idgen) from controlblock")) -- cgit 1.4.1-2-gfad0 From 564cff729a9a945924e40e57ce5be5af79dd6497 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 17 Feb 2018 00:37:41 +0100 Subject: new symbol files: loading of ASTs and module graphs --- compiler/rod.nim | 1 + compiler/rodimpl.nim | 379 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 378 insertions(+), 2 deletions(-) (limited to 'compiler/rodimpl.nim') diff --git a/compiler/rod.nim b/compiler/rod.nim index 6d34e9ea0..23c816498 100644 --- a/compiler/rod.nim +++ b/compiler/rod.nim @@ -14,6 +14,7 @@ import ast, idgen when not defined(nimSymbolfiles): template setupModuleCache* = discard template storeNode*(module: PSym; n: PNode) = discard + template loadNode*(module: PSym; index: var int): PNode = discard template getModuleId*(fullpath: string): int = getID() diff --git a/compiler/rodimpl.nim b/compiler/rodimpl.nim index aca576b23..a6f525787 100644 --- a/compiler/rodimpl.nim +++ b/compiler/rodimpl.nim @@ -9,8 +9,8 @@ ## This module implements the canonalization for the various caching mechanisms. -import strutils, os, intsets, ropes, db_sqlite, msgs, options, types, - renderer, rodutils, std / sha1 +import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types, + renderer, rodutils, std / sha1, idents var db: DbConn @@ -367,6 +367,381 @@ proc storeNode*(module: PSym; n: PNode) = break inc i +# ---------------- decoder ----------------------------------- +type + TRodReader = object + module: PSym + #sstack: seq[(PSym, ptr PSym)] # a stack of symbols to process + #tstack: seq[(PType, ptr PType)] # a stack of types to process + + #tmarks, smarks: IntSet + syms: Table[int, PSym] ## XXX make this more efficients + types: Table[int, PType] + cache: IdentCache + + BlobReader = object + s: string + pos: int + + PRodReader = var TRodReader + +proc initRodReader(cache: IdentCache): TRodReader = + TRodReader(module: nil, + syms: initTable[int, PSym](), types: initTable[int, PType](), + cache: cache) + +var gr = initRodReader(newIdentCache()) + +using + r: PRodReader + b: var BlobReader + +proc loadSym(r; id: int, info: TLineInfo): PSym +proc loadType(r; id: int, info: TLineInfo): PType + +proc decodeLineInfo(r; b; info: var TLineInfo) = + if b.s[b.pos] == '?': + inc(b.pos) + if b.s[b.pos] == ',': info.col = -1'i16 + else: info.col = int16(decodeVInt(b.s, b.pos)) + if b.s[b.pos] == ',': + inc(b.pos) + if b.s[b.pos] == ',': info.line = -1'i16 + else: info.line = int16(decodeVInt(b.s, b.pos)) + if b.s[b.pos] == ',': + inc(b.pos) + info.fileIndex = int32(decodeVInt(b.s, b.pos)) + +proc skipNode(b) = + assert b.s[b.pos] == '(' + var par = 0 + var pos = b.pos+1 + while true: + case b.s[pos] + of ')': + if par == 0: break + dec par + of '(': inc par + else: discard + inc pos + b.pos = pos+1 # skip ')' + +proc decodeNodeLazyBody(r; b; fInfo: TLineInfo, + belongsTo: PSym): PNode = + result = nil + if b.s[b.pos] == '(': + inc(b.pos) + if b.s[b.pos] == ')': + inc(b.pos) + return # nil node + result = newNodeI(TNodeKind(decodeVInt(b.s, b.pos)), fInfo) + decodeLineInfo(r, b, result.info) + if b.s[b.pos] == '$': + inc(b.pos) + result.flags = cast[TNodeFlags](int32(decodeVInt(b.s, b.pos))) + if b.s[b.pos] == '^': + inc(b.pos) + var id = decodeVInt(b.s, b.pos) + result.typ = loadType(r, id, result.info) + case result.kind + of nkCharLit..nkUInt64Lit: + if b.s[b.pos] == '!': + inc(b.pos) + result.intVal = decodeVBiggestInt(b.s, b.pos) + of nkFloatLit..nkFloat64Lit: + if b.s[b.pos] == '!': + inc(b.pos) + var fl = decodeStr(b.s, b.pos) + result.floatVal = parseFloat(fl) + of nkStrLit..nkTripleStrLit: + if b.s[b.pos] == '!': + inc(b.pos) + result.strVal = decodeStr(b.s, b.pos) + else: + result.strVal = "" + of nkIdent: + if b.s[b.pos] == '!': + inc(b.pos) + var fl = decodeStr(b.s, b.pos) + result.ident = r.cache.getIdent(fl) + else: + internalError(result.info, "decodeNode: nkIdent") + of nkSym: + if b.s[b.pos] == '!': + inc(b.pos) + var id = decodeVInt(b.s, b.pos) + result.sym = loadSym(r, id, result.info) + else: + internalError(result.info, "decodeNode: nkSym") + else: + var i = 0 + while b.s[b.pos] != ')': + when false: + if belongsTo != nil and i == bodyPos: + addSonNilAllowed(result, nil) + belongsTo.offset = b.pos + skipNode(b) + else: + discard + addSonNilAllowed(result, decodeNodeLazyBody(r, b, result.info, nil)) + inc i + if b.s[b.pos] == ')': inc(b.pos) + else: internalError(result.info, "decodeNode: ')' missing") + else: + internalError(fInfo, "decodeNode: '(' missing " & $b.pos) + +proc decodeNode(r; b; fInfo: TLineInfo): PNode = + result = decodeNodeLazyBody(r, b, fInfo, nil) + +proc decodeLoc(r; b; loc: var TLoc, info: TLineInfo) = + if b.s[b.pos] == '<': + inc(b.pos) + if b.s[b.pos] in {'0'..'9', 'a'..'z', 'A'..'Z'}: + loc.k = TLocKind(decodeVInt(b.s, b.pos)) + else: + loc.k = low(loc.k) + if b.s[b.pos] == '*': + inc(b.pos) + loc.storage = TStorageLoc(decodeVInt(b.s, b.pos)) + else: + loc.storage = low(loc.storage) + if b.s[b.pos] == '$': + inc(b.pos) + loc.flags = cast[TLocFlags](int32(decodeVInt(b.s, b.pos))) + else: + loc.flags = {} + if b.s[b.pos] == '^': + inc(b.pos) + loc.lode = decodeNode(r, b, info) + # rrGetType(b, decodeVInt(b.s, b.pos), info) + else: + loc.lode = nil + if b.s[b.pos] == '!': + inc(b.pos) + loc.r = rope(decodeStr(b.s, b.pos)) + else: + loc.r = nil + if b.s[b.pos] == '>': inc(b.pos) + else: internalError(info, "decodeLoc " & b.s[b.pos]) + +proc loadBlob(query: SqlQuery; id: int): BlobReader = + let blob = db.getValue(query, id) + if blob.len == 0: + internalError("symbolfiles: cannot find ID " & $ id) + result = BlobReader(pos: 0) + shallowCopy(result.s, blob) + +proc loadType(r; id: int; info: TLineInfo): PType = + result = r.types.getOrDefault(id) + if result != nil: return result + var b = loadBlob(sql"select data from types where nimid = ?", id) + + if b.s[b.pos] == '[': + inc(b.pos) + if b.s[b.pos] == ']': + inc(b.pos) + return # nil type + new(result) + result.kind = TTypeKind(decodeVInt(b.s, b.pos)) + if b.s[b.pos] == '+': + inc(b.pos) + result.id = decodeVInt(b.s, b.pos) + setId(result.id) + #if debugIds: registerID(result) + else: + internalError(info, "decodeType: no id") + # here this also avoids endless recursion for recursive type + r.types[result.id] = result + if b.s[b.pos] == '(': result.n = decodeNode(r, b, unknownLineInfo()) + if b.s[b.pos] == '$': + inc(b.pos) + result.flags = cast[TTypeFlags](int32(decodeVInt(b.s, b.pos))) + if b.s[b.pos] == '?': + inc(b.pos) + result.callConv = TCallingConvention(decodeVInt(b.s, b.pos)) + if b.s[b.pos] == '*': + inc(b.pos) + result.owner = loadSym(r, decodeVInt(b.s, b.pos), info) + if b.s[b.pos] == '&': + inc(b.pos) + result.sym = loadSym(r, decodeVInt(b.s, b.pos), info) + if b.s[b.pos] == '/': + inc(b.pos) + result.size = decodeVInt(b.s, b.pos) + else: + result.size = - 1 + if b.s[b.pos] == '=': + inc(b.pos) + result.align = decodeVInt(b.s, b.pos).int16 + else: + result.align = 2 + + if b.s[b.pos] == '\14': + inc(b.pos) + result.lockLevel = decodeVInt(b.s, b.pos).TLockLevel + else: + result.lockLevel = UnspecifiedLockLevel + + if b.s[b.pos] == '\15': + inc(b.pos) + result.destructor = loadSym(r, decodeVInt(b.s, b.pos), info) + if b.s[b.pos] == '\16': + inc(b.pos) + result.deepCopy = loadSym(r, decodeVInt(b.s, b.pos), info) + if b.s[b.pos] == '\17': + inc(b.pos) + result.assignment = loadSym(r, decodeVInt(b.s, b.pos), info) + if b.s[b.pos] == '\18': + inc(b.pos) + result.sink = loadSym(r, decodeVInt(b.s, b.pos), info) + while b.s[b.pos] == '\19': + inc(b.pos) + let x = decodeVInt(b.s, b.pos) + doAssert b.s[b.pos] == '\20' + inc(b.pos) + let y = loadSym(r, decodeVInt(b.s, b.pos), info) + result.methods.safeAdd((x, y)) + decodeLoc(r, b, result.loc, info) + while b.s[b.pos] == '^': + inc(b.pos) + if b.s[b.pos] == '(': + inc(b.pos) + if b.s[b.pos] == ')': inc(b.pos) + else: internalError(info, "decodeType ^(" & b.s[b.pos]) + rawAddSon(result, nil) + else: + var d = decodeVInt(b.s, b.pos) + rawAddSon(result, loadType(r, d, info)) + +proc decodeLib(r; b; info: TLineInfo): PLib = + result = nil + if b.s[b.pos] == '|': + new(result) + inc(b.pos) + result.kind = TLibKind(decodeVInt(b.s, b.pos)) + if b.s[b.pos] != '|': internalError("decodeLib: 1") + inc(b.pos) + result.name = rope(decodeStr(b.s, b.pos)) + if b.s[b.pos] != '|': internalError("decodeLib: 2") + inc(b.pos) + result.path = decodeNode(r, b, info) + +proc decodeInstantiations(r; b; info: TLineInfo; + s: var seq[PInstantiation]) = + while b.s[b.pos] == '\15': + inc(b.pos) + var ii: PInstantiation + new ii + ii.sym = loadSym(r, decodeVInt(b.s, b.pos), info) + ii.concreteTypes = @[] + while b.s[b.pos] == '\17': + inc(b.pos) + ii.concreteTypes.add loadType(r, decodeVInt(b.s, b.pos), info) + if b.s[b.pos] == '\20': + inc(b.pos) + ii.compilesId = decodeVInt(b.s, b.pos) + s.safeAdd ii + +proc loadSym(r; id: int; info: TLineInfo): PSym = + var + id: int + ident: PIdent + result = r.syms.getOrDefault(id) + if result != nil: return result + var b = loadBlob(sql"select data from syms where nimid = ?", id) + if b.s[b.pos] == '{': + inc(b.pos) + if b.s[b.pos] == '}': + inc(b.pos) + return # nil sym + var k = TSymKind(decodeVInt(b.s, b.pos)) + if b.s[b.pos] == '+': + inc(b.pos) + id = decodeVInt(b.s, b.pos) + setId(id) + else: + internalError(info, "decodeSym: no id") + if b.s[b.pos] == '&': + inc(b.pos) + ident = r.cache.getIdent(decodeStr(b.s, b.pos)) + else: + internalError(info, "decodeSym: no ident") + #echo "decoding: {", ident.s + new(result) + result.id = id + result.kind = k + result.name = ident # read the rest of the symbol description: + r.syms[result.id] = result + if b.s[b.pos] == '^': + inc(b.pos) + result.typ = loadType(r, decodeVInt(b.s, b.pos), info) + decodeLineInfo(r, b, result.info) + if b.s[b.pos] == '*': + inc(b.pos) + result.owner = loadSym(r, decodeVInt(b.s, b.pos), result.info) + if b.s[b.pos] == '$': + inc(b.pos) + result.flags = cast[TSymFlags](int32(decodeVInt(b.s, b.pos))) + if b.s[b.pos] == '@': + inc(b.pos) + result.magic = TMagic(decodeVInt(b.s, b.pos)) + if b.s[b.pos] == '!': + inc(b.pos) + result.options = cast[TOptions](int32(decodeVInt(b.s, b.pos))) + else: + result.options = r.module.options + if b.s[b.pos] == '%': + inc(b.pos) + result.position = decodeVInt(b.s, b.pos) + if b.s[b.pos] == '`': + inc(b.pos) + result.offset = decodeVInt(b.s, b.pos) + else: + result.offset = - 1 + decodeLoc(r, b, result.loc, result.info) + result.annex = decodeLib(r, b, info) + if b.s[b.pos] == '#': + inc(b.pos) + result.constraint = decodeNode(r, b, unknownLineInfo()) + case result.kind + of skType, skGenericParam: + while b.s[b.pos] == '\14': + inc(b.pos) + result.typeInstCache.safeAdd loadType(r, decodeVInt(b.s, b.pos), result.info) + of routineKinds: + decodeInstantiations(r, b, result.info, result.procInstCache) + if b.s[b.pos] == '\16': + inc(b.pos) + result.gcUnsafetyReason = loadSym(r, decodeVInt(b.s, b.pos), result.info) + of skModule, skPackage: + decodeInstantiations(r, b, result.info, result.usedGenerics) + of skLet, skVar, skField, skForVar: + if b.s[b.pos] == '\18': + inc(b.pos) + result.guard = loadSym(r, decodeVInt(b.s, b.pos), result.info) + if b.s[b.pos] == '\19': + inc(b.pos) + result.bitsize = decodeVInt(b.s, b.pos).int16 + else: discard + + if b.s[b.pos] == '(': + #if result.kind in routineKinds: + # result.ast = decodeNodeLazyBody(b, result.info, result) + #else: + result.ast = decodeNode(r, b, result.info) + +proc loadNode*(module: PSym; index: var int): PNode = + if index == 0: + index = parseInt db.getValue( + sql"select min(id) from toplevelstmts where module = ?", abs module.id) + var b = BlobReader(pos: 0) + b.s = db.getValue(sql"select data from toplevelstmts where id = ?", index) + if b.s.len == 0: return nil # end marker + gr.module = module + result = decodeNode(gr, b, module.info) + +# --------------- Database model --------------------------------------------- + proc createDb() = db.exec(sql""" create table if not exists controlblock( -- cgit 1.4.1-2-gfad0 From 2fcc1637465df3ccc04394ee4703330cc51ee493 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 20 Feb 2018 01:15:43 +0100 Subject: symbol files: more progress --- compiler/modules.nim | 16 ++++++----- compiler/passes.nim | 42 ++++++++++++++++++++++++----- compiler/rod.nim | 2 +- compiler/rodimpl.nim | 76 ++++++++++++++++++++++++++++++++++++++++------------ compiler/sem.nim | 3 ++- 5 files changed, 107 insertions(+), 32 deletions(-) (limited to 'compiler/rodimpl.nim') diff --git a/compiler/modules.nim b/compiler/modules.nim index ab384311e..ede10a0f4 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -169,13 +169,15 @@ proc compileModule*(graph: ModuleGraph; fileIdx: int32; cache: IdentCache, flags if sfMainModule in result.flags: gMainPackageId = result.owner.id - if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}: - rd = handleSymbolFile(result, cache) - if result.id < 0: - internalError("handleSymbolFile should have set the module's ID") - return - else: - result.id = getModuleId(toFullPath(fileIdx)) + when false: + if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}: + rd = handleSymbolFile(result, cache) + if result.id < 0: + internalError("handleSymbolFile should have set the module's ID") + return + else: + discard + result.id = getModuleId(toFullPath(fileIdx)) discard processModule(graph, result, if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil, rd, cache) diff --git a/compiler/passes.nim b/compiler/passes.nim index 29b27627d..1b4cd6fec 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -7,13 +7,13 @@ # distribution, for details about the copyright. # -# This module implements the passes functionality. A pass must implement the -# `TPass` interface. +## This module implements the passes functionality. A pass must implement the +## `TPass` interface. import strutils, options, ast, astalgo, llstream, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math, magicsys, nversion, - nimsets, syntaxes, times, rodread, idgen, modulegraphs, reorder + nimsets, syntaxes, times, rodread, idgen, modulegraphs, reorder, rod type @@ -29,7 +29,8 @@ type TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.} TPass* = tuple[open: TPassOpen, openCached: TPassOpenCached, - process: TPassProcess, close: TPassClose] + process: TPassProcess, close: TPassClose, + isFrontend: bool] TPassData* = tuple[input: PNode, closeOutput: PNode] TPasses* = openArray[TPass] @@ -41,11 +42,13 @@ type proc makePass*(open: TPassOpen = nil, openCached: TPassOpenCached = nil, process: TPassProcess = nil, - close: TPassClose = nil): TPass = + close: TPassClose = nil, + isFrontend = false): TPass = result.open = open result.openCached = openCached result.close = close result.process = process + result.isFrontend = isFrontend # the semantic checker needs these: var @@ -178,7 +181,34 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream, a: TPassContextArray s: PLLStream fileIdx = module.fileIdx - if rd == nil: + if module.id < 0: + # new module caching mechanism: + for i in 0.. 0, "cannot find file name for DB ID " & $dbId + result = fileInfoIdx(fullpath) + proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, result: var string) = if n == nil: @@ -91,7 +104,7 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, result.add(',') encodeVInt(n.info.line, result) result.add(',') - encodeVInt(n.info.fileIndex, result) + encodeVInt(toDbFileId(n.info.toFullPath), result) elif fInfo.line != n.info.line: result.add('?') encodeVInt(n.info.col, result) @@ -269,7 +282,7 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) = result.add(',') if s.info.line != -1'i16: encodeVInt(s.info.line, result) result.add(',') - encodeVInt(s.info.fileIndex, result) + encodeVInt(toDbFileId(s.info.toFullPath), result) if s.owner != nil: result.add('*') encodeVInt(s.owner.id, result) @@ -331,8 +344,8 @@ proc storeSym(w: PRodWriter; s: PSym) = encodeSym(w, s, buf) # XXX only store the name for exported symbols in order to speed up lookup # times once we enable the skStub logic. - db.exec(sql"insert into syms(nimid, module, name, data) values (?, ?, ?, ?)", - s.id, abs(w.module.id), s.name.s, buf) + db.exec(sql"insert into syms(nimid, module, name, data, exported) values (?, ?, ?, ?, ?)", + s.id, abs(w.module.id), s.name.s, buf, ord(sfExported in s.flags)) proc storeType(w: PRodWriter; t: PType) = var buf = newStringOfCap(160) @@ -410,7 +423,7 @@ proc decodeLineInfo(r; b; info: var TLineInfo) = else: info.line = int16(decodeVInt(b.s, b.pos)) if b.s[b.pos] == ',': inc(b.pos) - info.fileIndex = int32(decodeVInt(b.s, b.pos)) + info.fileIndex = fromDbFileId(decodeVInt(b.s, b.pos)) proc skipNode(b) = assert b.s[b.pos] == '(' @@ -569,7 +582,7 @@ proc loadType(r; id: int; info: TLineInfo): PType = inc(b.pos) result.size = decodeVInt(b.s, b.pos) else: - result.size = - 1 + result.size = -1 if b.s[b.pos] == '=': inc(b.pos) result.align = decodeVInt(b.s, b.pos).int16 @@ -642,25 +655,21 @@ proc decodeInstantiations(r; b; info: TLineInfo; ii.compilesId = decodeVInt(b.s, b.pos) s.safeAdd ii -proc loadSym(r; id: int; info: TLineInfo): PSym = - var - id: int - ident: PIdent - result = r.syms.getOrDefault(id) - if result != nil: return result - var b = loadBlob(sql"select data from syms where nimid = ?", id) +proc loadSymFromBlob(r; b; info: TLineInfo): PSym = if b.s[b.pos] == '{': inc(b.pos) if b.s[b.pos] == '}': inc(b.pos) return # nil sym var k = TSymKind(decodeVInt(b.s, b.pos)) + var id: int if b.s[b.pos] == '+': inc(b.pos) id = decodeVInt(b.s, b.pos) setId(id) else: internalError(info, "decodeSym: no id") + var ident: PIdent if b.s[b.pos] == '&': inc(b.pos) ident = r.cache.getIdent(decodeStr(b.s, b.pos)) @@ -729,9 +738,34 @@ proc loadSym(r; id: int; info: TLineInfo): PSym = # result.ast = decodeNodeLazyBody(b, result.info, result) #else: result.ast = decodeNode(r, b, result.info) + if sfCompilerProc in result.flags: + registerCompilerProc(result) + +proc loadSym(r; id: int; info: TLineInfo): PSym = + result = r.syms.getOrDefault(id) + if result != nil: return result + var b = loadBlob(sql"select data from syms where nimid = ?", id) + result = loadSymFromBlob(r, b, info) + doAssert id == result.id, "symbol ID is not consistent!" + +proc loadModuleSymTab(r; module: PSym) = + ## goal: fill module.tab + gr.syms[module.id] = module + for row in db.fastRows(sql"select nimid, data from syms where module = ? and exported = 1", abs(module.id)): + let id = parseInt(row[0]) + var s = r.syms.getOrDefault(id) + if s == nil: + var b = BlobReader(pos: 0) + shallowCopy(b.s, row[1]) + s = loadSymFromBlob(r, b, module.info) + assert s != nil + strTableAdd(module.tab, s) + if sfSystemModule in module.flags: + magicsys.systemModule = module proc loadNode*(module: PSym; index: var int): PNode = if index == 0: + loadModuleSymTab(gr, module) index = parseInt db.getValue( sql"select min(id) from toplevelstmts where module = ?", abs module.id) var b = BlobReader(pos: 0) @@ -749,6 +783,14 @@ proc createDb() = ); """) + db.exec(sql""" + create table if not exists filenames( + id integer primary key, + fullpath varchar(8000) not null + ); + """) + db.exec sql"create index if not exists FilenameIx on filenames(fullpath);" + db.exec(sql""" create table if not exists modules( id integer primary key, @@ -779,6 +821,7 @@ proc createDb() = module integer not null, name varchar(256) not null, data blob not null, + exported int not null, foreign key (module) references module(id) ); """) @@ -798,7 +841,6 @@ proc createDb() = """) db.exec sql"create index TopLevelStmtByModuleIdx on toplevelstmts(module);" - db.exec(sql""" create table if not exists statics( id integer primary key, diff --git a/compiler/sem.nim b/compiler/sem.nim index 053e739d3..12e77affc 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -621,4 +621,5 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = popProcCon(c) if c.runnableExamples != nil: testExamples(c) -const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose) +const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose, + isFrontend = true) -- cgit 1.4.1-2-gfad0 From 046ed4ed22095d9381d7317beaf215cf67ea4996 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 21 Feb 2018 00:48:23 +0100 Subject: symbol files: implemented accurate module dependency tracking --- compiler/modulegraphs.nim | 5 ++- compiler/modules.nim | 2 +- compiler/msgs.nim | 9 ++++ compiler/passes.nim | 2 + compiler/rod.nim | 4 +- compiler/rodimpl.nim | 103 ++++++++++++++++++++++++++++++++++++++-------- compiler/rodwrite.nim | 2 +- 7 files changed, 105 insertions(+), 22 deletions(-) (limited to 'compiler/rodimpl.nim') diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index c081a099a..2c59a9097 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -25,7 +25,7 @@ ## - Its dependent module stays the same. ## -import ast, intsets, tables, options +import ast, intsets, tables, options, rod type ModuleGraph* = ref object @@ -81,6 +81,8 @@ proc getModule*(g: ModuleGraph; fileIdx: int32): PSym = proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b proc addDep*(g: ModuleGraph; m: PSym, dep: int32) = + assert m.position == m.info.fileIndex + addModuleDep(m.info.fileIndex, dep, isIncludeFile = false) if suggestMode: deps.incl m.position.dependsOn(dep) # we compute the transitive closure later when quering the graph lazily. @@ -88,6 +90,7 @@ proc addDep*(g: ModuleGraph; m: PSym, dep: int32) = #invalidTransitiveClosure = true proc addIncludeDep*(g: ModuleGraph; module, includeFile: int32) = + addModuleDep(module, includeFile, isIncludeFile = true) discard hasKeyOrPut(inclToMod, includeFile, module) proc parentModule*(g: ModuleGraph; fileIdx: int32): int32 = diff --git a/compiler/modules.nim b/compiler/modules.nim index ede10a0f4..a3be5a518 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -177,7 +177,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: int32; cache: IdentCache, flags return else: discard - result.id = getModuleId(toFullPath(fileIdx)) + result.id = getModuleId(fileIdx, toFullPath(fileIdx)) discard processModule(graph, result, if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil, rd, cache) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 6439d47ee..ac4242e67 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -491,6 +491,7 @@ type dirtyfile: string # the file that is actually read into memory # and parsed; usually 'nil' but is used # for 'nimsuggest' + hash*: string # the checksum of the file TLineInfo* = object # This is designed to be as small as possible, # because it is used @@ -719,6 +720,14 @@ proc setDirtyFile*(fileIdx: int32; filename: string) = assert fileIdx >= 0 fileInfos[fileIdx].dirtyFile = filename +proc setHash*(fileIdx: int32; hash: string) = + assert fileIdx >= 0 + shallowCopy(fileInfos[fileIdx].hash, hash) + +proc getHash*(fileIdx: int32): string = + assert fileIdx >= 0 + shallowCopy(result, fileInfos[fileIdx].hash) + proc toFullPathConsiderDirty*(fileIdx: int32): string = if fileIdx < 0: result = "???" diff --git a/compiler/passes.nim b/compiler/passes.nim index 1b4cd6fec..f079100ea 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -194,6 +194,8 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream, while doContinue: let n = loadNode(module, stmtIndex) if n == nil or graph.stopCompile(): break + #if n.kind == nkImportStmt: + # echo "yes and it's ", n inc stmtIndex var m = n for i in 0.. 0, "cannot find file name for DB ID " & $dbId result = fileInfoIdx(fullpath) @@ -104,7 +145,7 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, result.add(',') encodeVInt(n.info.line, result) result.add(',') - encodeVInt(toDbFileId(n.info.toFullPath), result) + encodeVInt(toDbFileId(n.info.fileIndex), result) elif fInfo.line != n.info.line: result.add('?') encodeVInt(n.info.col, result) @@ -282,7 +323,7 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) = result.add(',') if s.info.line != -1'i16: encodeVInt(s.info.line, result) result.add(',') - encodeVInt(toDbFileId(s.info.toFullPath), result) + encodeVInt(toDbFileId(s.info.fileIndex), result) if s.owner != nil: result.add('*') encodeVInt(s.owner.id, result) @@ -764,16 +805,29 @@ proc loadModuleSymTab(r; module: PSym) = magicsys.systemModule = module proc loadNode*(module: PSym; index: var int): PNode = + assert gSymbolFiles == v2Sf if index == 0: loadModuleSymTab(gr, module) index = parseInt db.getValue( sql"select min(id) from toplevelstmts where module = ?", abs module.id) var b = BlobReader(pos: 0) - b.s = db.getValue(sql"select data from toplevelstmts where id = ?", index) - if b.s.len == 0: return nil # end marker + b.s = db.getValue(sql"select data from toplevelstmts where id = ? and module = ?", + index, abs module.id) + if b.s.len == 0: + db.exec(sql"insert into controlblock(idgen) values (?)", gFrontEndId) + return nil # end marker gr.module = module result = decodeNode(gr, b, module.info) +proc addModuleDep*(module, fileIdx: int32; isIncludeFile: bool) = + if gSymbolFiles != v2Sf: return + + let a = toDbFileId(module) + let b = toDbFileId(fileIdx) + + db.exec(sql"insert into deps(module, dependency, isIncludeFile) values (?, ?, ?)", + a, b, ord(isIncludeFile)) + # --------------- Database model --------------------------------------------- proc createDb() = @@ -786,7 +840,8 @@ proc createDb() = db.exec(sql""" create table if not exists filenames( id integer primary key, - fullpath varchar(8000) not null + fullpath varchar(8000) not null, + fullHash varchar(256) not null ); """) db.exec sql"create index if not exists FilenameIx on filenames(fullpath);" @@ -802,6 +857,17 @@ proc createDb() = );""") db.exec(sql"""create unique index if not exists SymNameIx on modules(fullpath);""") + db.exec(sql""" + create table if not exists deps( + id integer primary key, + module integer not null, + dependency integer not null, + isIncludeFile integer not null, + foreign key (module) references filenames(id), + foreign key (dependency) references filenames(id) + );""") + db.exec(sql"""create index if not exists DepsIx on deps(module);""") + db.exec(sql""" create table if not exists types( id integer primary key, @@ -865,5 +931,6 @@ proc setupModuleCache* = db.exec(sql"pragma journal_mode=off") db.exec(sql"pragma SYNCHRONOUS=off") db.exec(sql"pragma LOCKING_MODE=exclusive") - idgen.setId(parseInt db.getValue( - sql"select max(idgen) from controlblock")) + let lastId = db.getValue(sql"select max(idgen) from controlblock") + if lastId.len > 0: + idgen.setId(parseInt lastId) diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index 59a709295..96deb1d5a 100644 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -642,7 +642,7 @@ proc process(c: PPassContext, n: PNode): PNode = proc myOpen(g: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = if module.id < 0: internalError("rodwrite: module ID not set") - var w = newRodWriter(module.fileIdx.getHash, module, cache) + var w = newRodWriter(rodread.getHash module.fileIdx, module, cache) rawAddInterfaceSym(w, module) result = w -- cgit 1.4.1-2-gfad0 From 67380f71d606f5e4c31cdf0a591ad5e882e93fe4 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 21 Feb 2018 11:42:58 +0100 Subject: symbol files: delay the emission of forwarded procs --- compiler/rod.nim | 2 ++ compiler/rodimpl.nim | 26 +++++++++++++++++--------- compiler/sem.nim | 1 + 3 files changed, 20 insertions(+), 9 deletions(-) (limited to 'compiler/rodimpl.nim') diff --git a/compiler/rod.nim b/compiler/rod.nim index bb9beff54..bc2f3931e 100644 --- a/compiler/rod.nim +++ b/compiler/rod.nim @@ -20,6 +20,8 @@ when not defined(nimSymbolfiles): template addModuleDep*(module, fileIdx: int32; isIncludeFile: bool) = discard + template storeRemaining*(module: PSym) = discard + else: include rodimpl diff --git a/compiler/rodimpl.nim b/compiler/rodimpl.nim index 767929aa5..ceed166b4 100644 --- a/compiler/rodimpl.nim +++ b/compiler/rodimpl.nim @@ -15,6 +15,8 @@ import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types, ## Todo: ## - Implement the 'import' replay logic so that the codegen runs over ## dependent modules. +## - Make conditional symbols and the configuration part of a module's +## dependencies. ## - Test multi methods. ## - Implement the limited VM support based on sets. ## - Depencency computation should use signature hashes in order to @@ -49,7 +51,7 @@ proc getModuleId*(fileIdx: int32; fullpath: string): int = if gSymbolFiles != v2Sf: return getID() let module = db.getRow( sql"select id, fullHash from modules where fullpath = ?", fullpath) - let currentFullhash = $secureHashFile(fullpath) + let currentFullhash = hashFileCached(fileIdx, fullpath) if module[0].len == 0: result = int db.insertID(sql"insert into modules(fullpath, interfHash, fullHash) values (?, ?, ?)", fullpath, "", currentFullhash) @@ -75,12 +77,13 @@ type sstack: seq[PSym] # a stack of symbols to process tstack: seq[PType] # a stack of types to process tmarks, smarks: IntSet + forwardedSyms: seq[PSym] PRodWriter = var TRodWriter proc initRodWriter(module: PSym): TRodWriter = result = TRodWriter(module: module, sstack: @[], tstack: @[], - tmarks: initIntSet(), smarks: initIntSet()) + tmarks: initIntSet(), smarks: initIntSet(), forwardedSyms: @[]) when false: proc getDefines(): string = @@ -89,13 +92,6 @@ when false: if result.len != 0: add(result, " ") add(result, d) - proc addInclDep(w: PRodWriter, dep: string; info: TLineInfo) = - let resolved = dep.findModule(info.toFullPath) - encodeVInt(fileIdx(w, resolved), w.inclDeps) - add(w.inclDeps, " ") - encodeStr($secureHashFile(resolved), w.inclDeps) - add(w.inclDeps, rodNL) - const rodNL = "\L" @@ -381,6 +377,9 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) = encodeNode(w, s.info, s.ast, result) proc storeSym(w: PRodWriter; s: PSym) = + if sfForward in s.flags and s.kind != skModule: + w.forwardedSyms.add s + return var buf = newStringOfCap(160) encodeSym(w, s, buf) # XXX only store the name for exported symbols in order to speed up lookup @@ -421,6 +420,14 @@ proc storeNode*(module: PSym; n: PNode) = break inc i +proc storeRemaining*(module: PSym) = + if gSymbolFiles != v2Sf: return + w.module = module + for s in w.forwardedSyms: + assert sfForward notin s.flags + storeSym(w, s) + w.forwardedSyms.setLen 0 + # ---------------- decoder ----------------------------------- type TRodReader = object @@ -781,6 +788,7 @@ proc loadSymFromBlob(r; b; info: TLineInfo): PSym = result.ast = decodeNode(r, b, result.info) if sfCompilerProc in result.flags: registerCompilerProc(result) + #echo "loading ", result.name.s proc loadSym(r; id: int; info: TLineInfo): PSym = result = r.syms.getOrDefault(id) diff --git a/compiler/sem.nim b/compiler/sem.nim index 12e77affc..937f1637a 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -619,6 +619,7 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = replayMethodDefs(graph, c.rd) popOwner(c) popProcCon(c) + storeRemaining(c.module) if c.runnableExamples != nil: testExamples(c) const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose, -- cgit 1.4.1-2-gfad0 From ac6d9a307909d664fd335fdd13ebbcf16779a65f Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 26 Feb 2018 17:39:19 +0100 Subject: symbol files: baby steps to a working solution --- compiler/rodimpl.nim | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'compiler/rodimpl.nim') diff --git a/compiler/rodimpl.nim b/compiler/rodimpl.nim index ceed166b4..aff4f6909 100644 --- a/compiler/rodimpl.nim +++ b/compiler/rodimpl.nim @@ -400,8 +400,9 @@ proc storeNode*(module: PSym; n: PNode) = w.module = module var buf = newStringOfCap(160) encodeNode(w, module.info, n, buf) - db.exec(sql"insert into toplevelstmts(module, data) values (?, ?)", - abs(module.id), buf) + db.exec(sql"insert into toplevelstmts(module, position, data) values (?, ?, ?)", + abs(module.id), module.offset, buf) + inc module.offset var i = 0 while true: if i > 10_000: @@ -812,14 +813,14 @@ proc loadModuleSymTab(r; module: PSym) = if sfSystemModule in module.flags: magicsys.systemModule = module -proc loadNode*(module: PSym; index: var int): PNode = +proc loadNode*(module: PSym; index: int): PNode = assert gSymbolFiles == v2Sf if index == 0: loadModuleSymTab(gr, module) - index = parseInt db.getValue( - sql"select min(id) from toplevelstmts where module = ?", abs module.id) + #index = parseInt db.getValue( + # sql"select min(id) from toplevelstmts where module = ?", abs module.id) var b = BlobReader(pos: 0) - b.s = db.getValue(sql"select data from toplevelstmts where id = ? and module = ?", + b.s = db.getValue(sql"select data from toplevelstmts where position = ? and module = ?", index, abs module.id) if b.s.len == 0: db.exec(sql"insert into controlblock(idgen) values (?)", gFrontEndId) @@ -908,12 +909,14 @@ proc createDb() = db.exec(sql""" create table if not exists toplevelstmts( id integer primary key, + position integer not null, module integer not null, data blob not null, foreign key (module) references module(id) ); """) db.exec sql"create index TopLevelStmtByModuleIdx on toplevelstmts(module);" + db.exec sql"create index TopLevelStmtByPositionIdx on toplevelstmts(position);" db.exec(sql""" create table if not exists statics( -- cgit 1.4.1-2-gfad0