diff options
author | Andreas Rumpf <rumpf_a@web.de> | 2018-05-30 23:50:34 +0200 |
---|---|---|
committer | Andreas Rumpf <rumpf_a@web.de> | 2018-05-30 23:50:34 +0200 |
commit | 61fb83ecbb4c691c03d500f6c71499e59a67cef2 (patch) | |
tree | 5976368415b899b6a1a1b4640657c25bafcd22b3 | |
parent | a36c779f398d786082dc8d53412f8a9aaebf637b (diff) | |
download | Nim-61fb83ecbb4c691c03d500f6c71499e59a67cef2.tar.gz |
baby steps for incremental compilation
-rw-r--r-- | compiler/ast.nim | 2 | ||||
-rw-r--r-- | compiler/cgen.nim | 17 | ||||
-rw-r--r-- | compiler/commands.nim | 4 | ||||
-rw-r--r-- | compiler/destroyer.nim | 2 | ||||
-rw-r--r-- | compiler/evaltempl.nim | 2 | ||||
-rw-r--r-- | compiler/importer.nim | 5 | ||||
-rw-r--r-- | compiler/jsgen.nim | 4 | ||||
-rw-r--r-- | compiler/lambdalifting.nim | 2 | ||||
-rw-r--r-- | compiler/lookups.nim | 14 | ||||
-rw-r--r-- | compiler/magicsys.nim | 11 | ||||
-rw-r--r-- | compiler/main.nim | 11 | ||||
-rw-r--r-- | compiler/modules.nim | 6 | ||||
-rw-r--r-- | compiler/options.nim | 4 | ||||
-rw-r--r-- | compiler/passes.nim | 15 | ||||
-rw-r--r-- | compiler/plugins/itersgen.nim | 2 | ||||
-rw-r--r-- | compiler/pragmas.nim | 5 | ||||
-rw-r--r-- | compiler/rodimpl.nim | 31 | ||||
-rw-r--r-- | compiler/rodread.nim | 1245 | ||||
-rw-r--r-- | compiler/rodwrite.nim | 653 | ||||
-rw-r--r-- | compiler/sem.nim | 9 | ||||
-rw-r--r-- | compiler/semdata.nim | 2 | ||||
-rw-r--r-- | compiler/transf.nim | 4 | ||||
-rw-r--r-- | compiler/vmgen.nim | 2 |
23 files changed, 70 insertions, 1982 deletions
diff --git a/compiler/ast.nim b/compiler/ast.nim index 592c8fbe9..1f5b4927e 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1724,3 +1724,5 @@ template incompleteType*(t: PType): bool = template typeCompleted*(s: PSym) = incl s.flags, sfNoForward + +template getBody*(s: PSym): PNode = s.ast[bodyPos] diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 2d4fbf174..92e4898e4 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -12,7 +12,7 @@ import ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets, nversion, nimsets, msgs, std / sha1, bitsets, idents, types, - ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth, + ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth, condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, semparallel, tables, sets, ndi, lineinfos @@ -1306,7 +1306,9 @@ proc rawNewModule(g: BModuleList; module: PSym; conf: ConfigRef): BModule = proc newModule(g: BModuleList; module: PSym; conf: ConfigRef): BModule = # we should create only one cgen module for each module sym result = rawNewModule(g, module, conf) - growCache g.modules, module.position + if module.position >= g.modules.len: + setLen(g.modules, module.position + 1) + #growCache g.modules, module.position g.modules[module.position] = result template injectG() {.dirty.} = @@ -1417,7 +1419,7 @@ proc writeModule(m: BModule, pending: bool) = # generate code for the init statements of the module: let cfile = getCFile(m) - if m.rd == nil or optForceFullMake in m.config.globalOptions: + if true or optForceFullMake in m.config.globalOptions: genInitCode(m) finishTypeDescriptions(m) if sfMainModule in m.module.flags: @@ -1476,7 +1478,6 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = # if the module is cached, we don't regenerate the main proc # nor the dispatchers? But if the dispatchers changed? # XXX emit the dispatchers into its own .c file? - if b.rd != nil: return if n != nil: m.initProc.options = initProcOptions(m) genStmts(m.initProc, n) @@ -1499,13 +1500,9 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) = if g.generatedHeader != nil: finishModule(g.generatedHeader) while g.forwardedProcsCounter > 0: for m in cgenModules(g): - if m.rd == nil: - finishModule(m) + finishModule(m) for m in cgenModules(g): - if m.rd != nil: - m.updateCachedModule - else: - m.writeModule(pending=true) + m.writeModule(pending=true) writeMapping(config, g.mapping) if g.generatedHeader != nil: writeHeader(g.generatedHeader) diff --git a/compiler/commands.nim b/compiler/commands.nim index efb7e2427..cd9ebbe7d 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -626,9 +626,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "help", "h": expectNoArg(conf, switch, arg, pass, info) helpOnError(conf, pass) - of "symbolfiles": + of "symbolfiles", "incremental": case arg.normalize - of "on": conf.symbolFiles = enabledSf + of "on": conf.symbolFiles = v2Sf of "off": conf.symbolFiles = disabledSf of "writeonly": conf.symbolFiles = writeOnlySf of "readonly": conf.symbolFiles = readOnlySf diff --git a/compiler/destroyer.nim b/compiler/destroyer.nim index cf56c6e36..0395728c2 100644 --- a/compiler/destroyer.nim +++ b/compiler/destroyer.nim @@ -116,7 +116,7 @@ Remarks: Rule 1.2 is not yet implemented because ``sink`` is currently import intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees, - strutils, options, dfa, lowerings, rodread, tables, modulegraphs, + strutils, options, dfa, lowerings, tables, modulegraphs, lineinfos const diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim index e3dd0f342..d6c630e79 100644 --- a/compiler/evaltempl.nim +++ b/compiler/evaltempl.nim @@ -11,7 +11,7 @@ import strutils, options, ast, astalgo, msgs, os, idents, wordrecg, renderer, - rodread, lineinfos + lineinfos type TemplCtx = object diff --git a/compiler/importer.nim b/compiler/importer.nim index dd3eb95d4..f30c23731 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -10,7 +10,7 @@ # This module implements the symbol importing mechanism. import - intsets, strutils, os, ast, astalgo, msgs, options, idents, rodread, lookups, + intsets, strutils, os, ast, astalgo, msgs, options, idents, lookups, semdata, passes, renderer, modulepaths, sigmatch, lineinfos proc evalImport*(c: PContext, n: PNode): PNode @@ -74,7 +74,8 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) = if s == nil: errorUndeclaredIdentifier(c, n.info, ident.s) else: - if s.kind == skStub: loadStub(s) + when false: + if s.kind == skStub: loadStub(s) if s.kind notin ExportableSymKinds: internalError(c.config, n.info, "importSymbol: 2") # for an enumeration we have to add all identifiers diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 00fe46eb6..3c57f97af 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -31,8 +31,8 @@ implements the required case distinction. import ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options, nversion, nimsets, msgs, std / sha1, bitsets, idents, types, os, tables, - times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils, - intsets, cgmeth, lowerings, sighashes, lineinfos + times, ropes, math, passes, ccgutils, wordrecg, renderer, + intsets, cgmeth, lowerings, sighashes, lineinfos, rodutils from modulegraphs import ModuleGraph diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 738d11050..6fb273164 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -11,7 +11,7 @@ import intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os, - idents, renderer, types, magicsys, rodread, lowerings, tables, + idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos discard """ diff --git a/compiler/lookups.nim b/compiler/lookups.nim index cf77f5f7a..87694988a 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -10,7 +10,7 @@ # This module implements lookup helpers. import - intsets, ast, astalgo, idents, semdata, types, msgs, options, rodread, + intsets, ast, astalgo, idents, semdata, types, msgs, options, renderer, wordrecg, idgen, nimfix.prettybase, lineinfos, strutils proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) @@ -288,7 +288,8 @@ proc lookUp*(c: PContext, n: PNode): PSym = return if contains(c.ambiguousSymbols, result.id): errorUseQualifier(c, n.info, result) - if result.kind == skStub: loadStub(result) + when false: + if result.kind == skStub: loadStub(result) type TLookupFlag* = enum @@ -343,7 +344,8 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym = result = errorSym(c, n.sons[1]) else: result = nil - if result != nil and result.kind == skStub: loadStub(result) + when false: + if result != nil and result.kind == skStub: loadStub(result) proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = case n.kind @@ -392,7 +394,8 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = o.inSymChoice = initIntSet() incl(o.inSymChoice, result.id) else: discard - if result != nil and result.kind == skStub: loadStub(result) + when false: + if result != nil and result.kind == skStub: loadStub(result) proc lastOverloadScope*(o: TOverloadIter): int = case o.mode @@ -443,7 +446,8 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = result = firstIdentExcluding(o.it, o.scope.symbols, n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config) - if result != nil and result.kind == skStub: loadStub(result) + when false: + if result != nil and result.kind == skStub: loadStub(result) proc pickSym*(c: PContext, n: PNode; kinds: set[TSymKind]; flags: TSymFlags = {}): PSym = diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index af89c6d69..d40b9d732 100644 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -10,7 +10,7 @@ # Built-in types and compilerprocs are registered here. import - ast, astalgo, hashes, msgs, platform, nversion, times, idents, rodread, + ast, astalgo, hashes, msgs, platform, nversion, times, idents, modulegraphs, lineinfos export createMagic @@ -31,7 +31,6 @@ proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym = localError(g.config, info, "system module needs: " & name) result = newSym(skError, getIdent(g.cache, name), g.systemModule, g.systemModule.info, {}) result.typ = newType(tyError, g.systemModule) - if result.kind == skStub: loadStub(result) if result.kind == skAlias: result = result.owner proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSym = @@ -39,7 +38,6 @@ proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSy let id = getIdent(g.cache, name) var r = initIdentIter(ti, g.systemModule.tab, id) while r != nil: - if r.kind == skStub: loadStub(r) if r.magic == m: # prefer the tyInt variant: if r.typ.sons[0] != nil and r.typ.sons[0].kind == tyInt: return r @@ -159,13 +157,6 @@ proc setIntLitType*(g: ModuleGraph; result: PNode) = proc getCompilerProc*(g: ModuleGraph; name: string): PSym = let ident = getIdent(g.cache, name) result = strTableGet(g.compilerprocs, ident) - when false: - if result == nil: - result = strTableGet(g.rodCompilerprocs, ident) - if result != nil: - strTableAdd(g.compilerprocs, result) - if result.kind == skStub: loadStub(result) - if result.kind == skAlias: result = result.owner proc registerCompilerProc*(g: ModuleGraph; s: PSym) = strTableAdd(g.compilerprocs, s) diff --git a/compiler/main.nim b/compiler/main.nim index 0929974bd..97e96dcff 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -14,7 +14,7 @@ when not defined(nimcore): import llstream, strutils, ast, astalgo, lexer, syntaxes, renderer, options, msgs, - os, condsyms, rodread, rodwrite, times, + os, condsyms, times, wordrecg, sem, semdata, idents, passes, docgen, extccomp, cgen, jsgen, json, nversion, platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen, @@ -23,10 +23,6 @@ import from magicsys import resetSysTypes -proc rodPass(g: ModuleGraph) = - if g.config.symbolFiles in {enabledSf, writeOnlySf}: - registerPass(g, rodwritePass) - proc codegenPass(g: ModuleGraph) = registerPass g, cgenPass @@ -47,7 +43,6 @@ proc writeDepsFile(g: ModuleGraph; project: string) = proc commandGenDepend(graph: ModuleGraph; cache: IdentCache) = semanticPasses(graph) registerPass(graph, gendependPass) - #registerPass(cleanupPass) compileProject(graph, cache) let project = graph.config.projectFull writeDepsFile(graph, project) @@ -59,7 +54,6 @@ proc commandCheck(graph: ModuleGraph; cache: IdentCache) = graph.config.errorMax = high(int) # do not stop after first error defineSymbol(graph.config.symbols, "nimcheck") semanticPasses(graph) # use an empty backend for semantic checking only - rodPass(graph) compileProject(graph, cache) proc commandDoc2(graph: ModuleGraph; cache: IdentCache; json: bool) = @@ -67,7 +61,6 @@ proc commandDoc2(graph: ModuleGraph; cache: IdentCache; json: bool) = semanticPasses(graph) if json: registerPass(graph, docgen2JsonPass) else: registerPass(graph, docgen2Pass) - #registerPass(cleanupPass()) compileProject(graph, cache) finishDoc2Pass(graph.config.projectName) @@ -76,8 +69,6 @@ proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) = extccomp.initVars(conf) semanticPasses(graph) registerPass(graph, cgenPass) - rodPass(graph) - #registerPass(cleanupPass()) compileProject(graph, cache) cgenWriteModules(graph.backend, conf) diff --git a/compiler/modules.nim b/compiler/modules.nim index 61568e67c..9b04578c0 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -10,7 +10,7 @@ ## Implements the module handling, including the caching of modules. import - ast, astalgo, magicsys, std / sha1, rodread, msgs, cgendata, sigmatch, options, + ast, astalgo, magicsys, std / sha1, msgs, cgendata, sigmatch, options, idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod, lineinfos @@ -42,7 +42,9 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = result.owner = packSym result.position = int fileIdx - growCache graph.modules, int fileIdx + if int(fileIdx) >= graph.modules.len: + setLen(graph.modules, int(fileIdx) + 1) + #growCache graph.modules, int fileIdx graph.modules[result.position] = result incl(result.flags, sfUsed) diff --git a/compiler/options.nim b/compiler/options.nim index 04444f7c9..cb4f1e885 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -121,7 +121,7 @@ type notnil SymbolFilesOption* = enum - disabledSf, enabledSf, writeOnlySf, readOnlySf, v2Sf + disabledSf, writeOnlySf, readOnlySf, v2Sf TSystemCC* = enum ccNone, ccGcc, ccLLVM_Gcc, ccCLang, ccLcc, ccBcc, ccDmc, ccWcc, ccVcc, @@ -362,7 +362,7 @@ proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.cmd in {cmdDoc, proc usesNativeGC*(conf: ConfigRef): bool {.inline.} = conf.selectedGC >= gcRefc template compilationCachePresent*(conf: ConfigRef): untyped = - conf.symbolFiles in {enabledSf, writeOnlySf} + conf.symbolFiles in {v2Sf, writeOnlySf} # {optCaasEnabled, optSymbolFiles} * gGlobalOptions != {} template optPreserveOrigSource*(conf: ConfigRef): untyped = diff --git a/compiler/passes.nim b/compiler/passes.nim index 4ba17adcc..e8fd89025 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -13,13 +13,13 @@ import strutils, options, ast, astalgo, llstream, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math, magicsys, nversion, - nimsets, syntaxes, times, rodread, idgen, modulegraphs, reorder, rod, + nimsets, syntaxes, times, idgen, modulegraphs, reorder, rod, lineinfos type + PRodReader* = ref object TPassContext* = object of RootObj # the pass's context - rd*: PRodReader # != nil if created by "openCached" PPassContext* = ref TPassContext @@ -107,8 +107,6 @@ proc openPassesCached(g: ModuleGraph; a: var TPassContextArray, module: PSym, for i in countup(0, gPassesLen - 1): if not isNil(gPasses[i].openCached): a[i] = gPasses[i].openCached(g, module, rd) - if a[i] != nil: - a[i].rd = rd else: a[i] = nil @@ -198,7 +196,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream, if not isNil(gPasses[i].close) and not gPasses[i].isFrontend: m = gPasses[i].close(graph, a[i], m) a[i] = nil - elif rd == nil: + else: openPasses(graph, a, module, cache) if stream == nil: let filename = toFullPathConsiderDirty(graph.config, fileIdx) @@ -241,11 +239,4 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream, closePasses(graph, a) # id synchronization point for more consistent code generation: idSynchronizationPoint(1000) - else: - openPassesCached(graph, a, module, rd) - var n = loadInitSection(rd) - for i in countup(0, sonsLen(n) - 1): - if graph.stopCompile(): break - processTopLevelStmtCached(n.sons[i], a) - closePassesCached(graph, a) result = true diff --git a/compiler/plugins/itersgen.nim b/compiler/plugins/itersgen.nim index 847d13ccf..440d2e081 100644 --- a/compiler/plugins/itersgen.nim +++ b/compiler/plugins/itersgen.nim @@ -11,7 +11,7 @@ import ".." / [ast, astalgo, magicsys, lookups, semdata, - lambdalifting, rodread, msgs] + lambdalifting, msgs] proc iterToProcImpl*(c: PContext, n: PNode): PNode = result = newNodeI(nkStmtList, n.info) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 77f75922c..061bbacfa 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -12,7 +12,7 @@ import os, platform, condsyms, ast, astalgo, idents, semdata, msgs, renderer, wordrecg, ropes, options, strutils, extccomp, math, magicsys, trees, - rodread, types, lookups, lineinfos + types, lookups, lineinfos const FirstCallConv* = wNimcall @@ -482,7 +482,8 @@ proc semAsmOrEmit*(con: PContext, n: PNode, marker: char): PNode = if sub != "": var e = searchInScopes(con, getIdent(con.cache, sub)) if e != nil: - if e.kind == skStub: loadStub(e) + when false: + if e.kind == skStub: loadStub(e) incl(e.flags, sfUsed) addSon(result, newSymNode(e)) else: diff --git a/compiler/rodimpl.nim b/compiler/rodimpl.nim index 0bb94bf22..360c82a6e 100644 --- a/compiler/rodimpl.nim +++ b/compiler/rodimpl.nim @@ -43,7 +43,7 @@ proc needsRecompile(g: ModuleGraph; fileIdx: FileIndex; fullpath: string; return false proc getModuleId*(g: ModuleGraph; fileIdx: FileIndex; fullpath: string): int = - if g.config.symbolFiles != v2Sf: return getID() + if g.config.symbolFiles in {disabledSf, writeOnlySf}: return getID() let module = g.incr.db.getRow( sql"select id, fullHash from modules where fullpath = ?", fullpath) let currentFullhash = hashFileCached(g.config, fileIdx, fullpath) @@ -57,6 +57,7 @@ proc getModuleId*(g: ModuleGraph; fileIdx: FileIndex; fullpath: string): int = doAssert(result != 0) var cycleCheck = initIntSet() if not needsRecompile(g, fileIdx, fullpath, cycleCheck): + echo "cached successfully! ", fullpath return -result db.exec(sql"update modules set fullHash = ? where id = ?", currentFullhash, module[0]) db.exec(sql"delete from deps where module = ?", module[0]) @@ -342,17 +343,21 @@ proc storeSym(g: ModuleGraph; s: PSym) = encodeSym(g, s, buf) # XXX only store the name for exported symbols in order to speed up lookup # times once we enable the skStub logic. + let m = getModule(s) + let mid = if m == nil: 0 else: abs(m.id) db.exec(sql"insert into syms(nimid, module, name, data, exported) values (?, ?, ?, ?, ?)", - s.id, abs(getModule(s).id), s.name.s, buf, ord(sfExported in s.flags)) + s.id, mid, s.name.s, buf, ord(sfExported in s.flags)) proc storeType(g: ModuleGraph; t: PType) = var buf = newStringOfCap(160) encodeType(g, t, buf) + let m = if t.owner != nil: getModule(t.owner) else: nil + let mid = if m == nil: 0 else: abs(m.id) db.exec(sql"insert into types(nimid, module, data) values (?, ?, ?)", - t.id, abs(getModule(t.owner).id), buf) + t.id, mid, buf) proc storeNode*(g: ModuleGraph; module: PSym; n: PNode) = - if g.config.symbolFiles != v2Sf: return + if g.config.symbolFiles == disabledSf: return var buf = newStringOfCap(160) encodeNode(g, module.info, n, buf) db.exec(sql"insert into toplevelstmts(module, position, data) values (?, ?, ?)", @@ -377,11 +382,14 @@ proc storeNode*(g: ModuleGraph; module: PSym; n: PNode) = inc i proc storeRemaining*(g: ModuleGraph; module: PSym) = - if g.config.symbolFiles != v2Sf: return + if g.config.symbolFiles == disabledSf: return + var stillForwarded: seq[PSym] = @[] for s in w.forwardedSyms: - assert sfForward notin s.flags - storeSym(g, s) - w.forwardedSyms.setLen 0 + if sfForward notin s.flags: + storeSym(g, s) + else: + stillForwarded.add s + swap w.forwardedSyms, stillForwarded # ---------------- decoder ----------------------------------- @@ -529,6 +537,8 @@ proc loadBlob(g; query: SqlQuery; id: int): BlobReader = internalError(g.config, "symbolfiles: cannot find ID " & $ id) result = BlobReader(pos: 0) shallowCopy(result.s, blob) + # ensure we can read without index checks: + result.s.add '\0' proc loadType(g; id: int; info: TLineInfo): PType = result = g.incr.r.types.getOrDefault(id) @@ -742,6 +752,8 @@ proc loadModuleSymTab(g; module: PSym) = if s == nil: var b = BlobReader(pos: 0) shallowCopy(b.s, row[1]) + # ensure we can read without index checks: + b.s.add '\0' s = loadSymFromBlob(g, b, module.info) assert s != nil strTableAdd(module.tab, s) @@ -749,7 +761,6 @@ proc loadModuleSymTab(g; module: PSym) = g.systemModule = module proc loadNode*(g: ModuleGraph; module: PSym; index: int): PNode = - assert g.config.symbolFiles == v2Sf if index == 0: loadModuleSymTab(g, module) #index = parseInt db.getValue( @@ -763,7 +774,7 @@ proc loadNode*(g: ModuleGraph; module: PSym; index: int): PNode = result = decodeNode(g, b, module.info) proc setupModuleCache*(g: ModuleGraph) = - if g.config.symbolFiles != v2Sf: return + if g.config.symbolFiles == disabledSf: return let dbfile = getNimcacheDir(g.config) / "rodfiles.db" if not fileExists(dbfile): db = open(connection=dbfile, user="nim", password="", diff --git a/compiler/rodread.nim b/compiler/rodread.nim deleted file mode 100644 index dc9b1c61d..000000000 --- a/compiler/rodread.nim +++ /dev/null @@ -1,1245 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2013 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# This module is responsible for loading of rod files. -# -# Reading and writing binary files are really hard to debug. Therefore we use -# a "creative" text/binary hybrid format. ROD-files are more efficient -# to process because symbols can be loaded on demand. -# -# A ROD file consists of: -# -# - a header: -# NIM:$fileversion\n -# - the module's id (even if the module changed, its ID will not!): -# ID:Ax3\n -# - HASH value of this module: -# HASH:HASH-val\n -# - a section containing the compiler options and defines this -# module has been compiled with: -# OPTIONS:options\n -# GOPTIONS:options\n # global options -# CMD:command\n -# DEFINES:defines\n -# - FILES( -# myfile.inc -# lib/mymodA -# ) -# - an include file dependency section: -# INCLUDES( -# <fileidx> <Hash of myfile.inc>\n # fileidx is the LINE in the file section! -# ) -# - a module dependency section: -# DEPS: <fileidx> <fileidx>\n -# - an interface section: -# INTERF( -# identifier1 id\n # id is the symbol's id -# identifier2 id\n -# ) -# - a compiler proc section: -# COMPILERPROCS( -# identifier1 id\n # id is the symbol's id -# ) -# - an index consisting of (ID, linenumber)-pairs: -# INDEX( -# id-diff idx-diff\n -# id-diff idx-diff\n -# ) -# -# Since the whole index has to be read in advance, we compress it by -# storing the integer differences to the last entry instead of using the -# real numbers. -# -# - an import index consisting of (ID, moduleID)-pairs: -# IMPORTS( -# id-diff moduleID-diff\n -# id-diff moduleID-diff\n -# ) -# - a list of all exported type converters because they are needed for correct -# semantic checking: -# CONVERTERS:id id\n # symbol ID -# -# This is a misnomer now; it's really a "load unconditionally" section as -# it is also used for pattern templates. -# -# - a list of all (private or exported) methods because they are needed for -# correct dispatcher generation: -# METHODS: id id\n # symbol ID -# - an AST section that contains the module's AST: -# INIT( -# idx\n # position of the node in the DATA section -# idx\n -# ) -# - a data section, where each type, symbol or AST is stored. -# DATA( -# type -# (node) -# sym -# ) -# -# The data section MUST be the last section of the file, because processing -# stops immediately after ``DATA(`` and the rest is only loaded on demand -# by using a mem'mapped file. -# - -import - os, options, strutils, nversion, ast, astalgo, msgs, platform, condsyms, - ropes, idents, std / sha1, idgen, types, rodutils, memfiles, tables, - lineinfos - -type - TReasonForRecompile* = enum ## all the reasons that can trigger recompilation - rrEmpty, # dependencies not yet computed - rrNone, # no need to recompile - rrRodDoesNotExist, # rod file does not exist - rrRodInvalid, # rod file is invalid - rrHashChange, # file has been edited since last recompilation - rrDefines, # defines have changed - rrOptions, # options have changed - rrInclDeps, # an include has changed - rrModDeps # a module this module depends on has been changed - -const - reasonToFrmt*: array[TReasonForRecompile, string] = ["", - "no need to recompile: $1", "symbol file for $1 does not exist", - "symbol file for $1 has the wrong version", - "file edited since last compilation: $1", - "list of conditional symbols changed for: $1", - "list of options changed for: $1", - "an include file edited: $1", - "a module $1 depends on has changed"] - -type - TIndex*{.final.} = object # an index with compression - lastIdxKey*, lastIdxVal*: int - tab*: TIITable - r*: string # writers use this - offset*: int # readers use this - - TRodReader* = object of RootObj - pos: int # position; used for parsing - s: cstring # mmap'ed file contents - options: TOptions - reason: TReasonForRecompile - modDeps: seq[FileIndex] - files: seq[FileIndex] - dataIdx: int # offset of start of data section - convertersIdx: int # offset of start of converters section - initIdx, interfIdx, compilerProcsIdx, methodsIdx: int - filename: string - index, imports: TIndex - readerIndex: int - line: int # only used for debugging, but is always in the code - moduleID: int - syms: Table[int, PSym] # already processed symbols - memfile: MemFile # unfortunately there is no point in time where we - # can close this! XXX - methods*: TSymSeq - origFile: string - inViewMode: bool - cache*: IdentCache - config: ConfigRef - - PRodReader* = ref TRodReader - -var rodCompilerprocs*: TStrTable # global because this is needed by magicsys - -proc rawLoadStub(s: PSym) - -var gTypeTable: TIdTable - -proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym - # `info` is only used for debugging purposes -proc rrGetType(r: PRodReader, id: int, info: TLineInfo): PType - -proc decodeLineInfo(r: PRodReader, info: var TLineInfo) = - if r.s[r.pos] == '?': - inc(r.pos) - if r.s[r.pos] == ',': info.col = -1'i16 - else: info.col = int16(decodeVInt(r.s, r.pos)) - if r.s[r.pos] == ',': - inc(r.pos) - if r.s[r.pos] == ',': info.line = 0'u16 - else: info.line = uint16(decodeVInt(r.s, r.pos)) - if r.s[r.pos] == ',': - inc(r.pos) - info = newLineInfo(r.files[decodeVInt(r.s, r.pos)], int info.line, info.col) - -proc skipNode(r: PRodReader) = - assert r.s[r.pos] == '(' - var par = 0 - var pos = r.pos+1 - while true: - case r.s[pos] - of ')': - if par == 0: break - dec par - of '(': inc par - else: discard - inc pos - r.pos = pos+1 # skip ')' - -proc decodeNodeLazyBody(r: PRodReader, fInfo: TLineInfo, - belongsTo: PSym): PNode = - result = nil - if r.s[r.pos] == '(': - inc(r.pos) - if r.s[r.pos] == ')': - inc(r.pos) - return # nil node - result = newNodeI(TNodeKind(decodeVInt(r.s, r.pos)), fInfo) - decodeLineInfo(r, result.info) - if r.s[r.pos] == '$': - inc(r.pos) - result.flags = cast[TNodeFlags](int32(decodeVInt(r.s, r.pos))) - if r.s[r.pos] == '^': - inc(r.pos) - var id = decodeVInt(r.s, r.pos) - result.typ = rrGetType(r, id, result.info) - case result.kind - of nkCharLit..nkUInt64Lit: - if r.s[r.pos] == '!': - inc(r.pos) - result.intVal = decodeVBiggestInt(r.s, r.pos) - of nkFloatLit..nkFloat64Lit: - if r.s[r.pos] == '!': - inc(r.pos) - var fl = decodeStr(r.s, r.pos) - result.floatVal = parseFloat(fl) - of nkStrLit..nkTripleStrLit: - if r.s[r.pos] == '!': - inc(r.pos) - result.strVal = decodeStr(r.s, r.pos) - else: - result.strVal = "" # BUGFIX - of nkIdent: - if r.s[r.pos] == '!': - inc(r.pos) - var fl = decodeStr(r.s, r.pos) - result.ident = r.cache.getIdent(fl) - else: - internalError(r.config, result.info, "decodeNode: nkIdent") - of nkSym: - if r.s[r.pos] == '!': - inc(r.pos) - var id = decodeVInt(r.s, r.pos) - result.sym = rrGetSym(r, id, result.info) - else: - internalError(r.config, result.info, "decodeNode: nkSym") - else: - var i = 0 - while r.s[r.pos] != ')': - if belongsTo != nil and i == bodyPos: - addSonNilAllowed(result, nil) - belongsTo.offset = r.pos - skipNode(r) - else: - addSonNilAllowed(result, decodeNodeLazyBody(r, result.info, nil)) - inc i - if r.s[r.pos] == ')': inc(r.pos) - else: internalError(r.config, result.info, "decodeNode: ')' missing") - else: - internalError(r.config, fInfo, "decodeNode: '(' missing " & $r.pos) - -proc decodeNode(r: PRodReader, fInfo: TLineInfo): PNode = - result = decodeNodeLazyBody(r, fInfo, nil) - -proc decodeLoc(r: PRodReader, loc: var TLoc, info: TLineInfo) = - if r.s[r.pos] == '<': - inc(r.pos) - if r.s[r.pos] in {'0'..'9', 'a'..'z', 'A'..'Z'}: - loc.k = TLocKind(decodeVInt(r.s, r.pos)) - else: - loc.k = low(loc.k) - if r.s[r.pos] == '*': - inc(r.pos) - loc.storage = TStorageLoc(decodeVInt(r.s, r.pos)) - else: - loc.storage = low(loc.storage) - if r.s[r.pos] == '$': - inc(r.pos) - loc.flags = cast[TLocFlags](int32(decodeVInt(r.s, r.pos))) - else: - loc.flags = {} - if r.s[r.pos] == '^': - inc(r.pos) - loc.lode = decodeNode(r, info) - # rrGetType(r, decodeVInt(r.s, r.pos), info) - else: - loc.lode = nil - if r.s[r.pos] == '!': - inc(r.pos) - loc.r = rope(decodeStr(r.s, r.pos)) - else: - loc.r = nil - if r.s[r.pos] == '>': inc(r.pos) - else: internalError(r.config, info, "decodeLoc " & r.s[r.pos]) - -proc decodeType(r: PRodReader, info: TLineInfo): PType = - result = nil - if r.s[r.pos] == '[': - inc(r.pos) - if r.s[r.pos] == ']': - inc(r.pos) - return # nil type - new(result) - result.kind = TTypeKind(decodeVInt(r.s, r.pos)) - if r.s[r.pos] == '+': - inc(r.pos) - result.id = decodeVInt(r.s, r.pos) - setId(result.id) - if debugIds: registerID(result) - else: - internalError(r.config, info, "decodeType: no id") - # here this also avoids endless recursion for recursive type - idTablePut(gTypeTable, result, result) - if r.s[r.pos] == '(': result.n = decodeNode(r, unknownLineInfo()) - if r.s[r.pos] == '$': - inc(r.pos) - result.flags = cast[TTypeFlags](int32(decodeVInt(r.s, r.pos))) - if r.s[r.pos] == '?': - inc(r.pos) - result.callConv = TCallingConvention(decodeVInt(r.s, r.pos)) - if r.s[r.pos] == '*': - inc(r.pos) - result.owner = rrGetSym(r, decodeVInt(r.s, r.pos), info) - if r.s[r.pos] == '&': - inc(r.pos) - result.sym = rrGetSym(r, decodeVInt(r.s, r.pos), info) - if r.s[r.pos] == '/': - inc(r.pos) - result.size = decodeVInt(r.s, r.pos) - else: - result.size = - 1 - if r.s[r.pos] == '=': - inc(r.pos) - result.align = decodeVInt(r.s, r.pos).int16 - else: - result.align = 2 - - if r.s[r.pos] == '\14': - inc(r.pos) - result.lockLevel = decodeVInt(r.s, r.pos).TLockLevel - else: - result.lockLevel = UnspecifiedLockLevel - - if r.s[r.pos] == '\15': - inc(r.pos) - result.destructor = rrGetSym(r, decodeVInt(r.s, r.pos), info) - if r.s[r.pos] == '\16': - inc(r.pos) - result.deepCopy = rrGetSym(r, decodeVInt(r.s, r.pos), info) - if r.s[r.pos] == '\17': - inc(r.pos) - result.assignment = rrGetSym(r, decodeVInt(r.s, r.pos), info) - if r.s[r.pos] == '\18': - inc(r.pos) - result.sink = rrGetSym(r, decodeVInt(r.s, r.pos), info) - while r.s[r.pos] == '\19': - inc(r.pos) - let x = decodeVInt(r.s, r.pos) - doAssert r.s[r.pos] == '\20' - inc(r.pos) - let y = rrGetSym(r, decodeVInt(r.s, r.pos), info) - result.methods.add((x, y)) - decodeLoc(r, result.loc, info) - while r.s[r.pos] == '^': - inc(r.pos) - if r.s[r.pos] == '(': - inc(r.pos) - if r.s[r.pos] == ')': inc(r.pos) - else: internalError(r.config, info, "decodeType ^(" & r.s[r.pos]) - rawAddSon(result, nil) - else: - var d = decodeVInt(r.s, r.pos) - rawAddSon(result, rrGetType(r, d, info)) - -proc decodeLib(r: PRodReader, info: TLineInfo): PLib = - result = nil - if r.s[r.pos] == '|': - new(result) - inc(r.pos) - result.kind = TLibKind(decodeVInt(r.s, r.pos)) - if r.s[r.pos] != '|': internalError(r.config, "decodeLib: 1") - inc(r.pos) - result.name = rope(decodeStr(r.s, r.pos)) - if r.s[r.pos] != '|': internalError(r.config, "decodeLib: 2") - inc(r.pos) - result.path = decodeNode(r, info) - -proc decodeInstantiations(r: PRodReader; info: TLineInfo; - s: var seq[PInstantiation]) = - while r.s[r.pos] == '\15': - inc(r.pos) - var ii: PInstantiation - new ii - ii.sym = rrGetSym(r, decodeVInt(r.s, r.pos), info) - ii.concreteTypes = @[] - while r.s[r.pos] == '\17': - inc(r.pos) - ii.concreteTypes.add rrGetType(r, decodeVInt(r.s, r.pos), info) - if r.s[r.pos] == '\20': - inc(r.pos) - ii.compilesId = decodeVInt(r.s, r.pos) - s.add ii - -proc decodeSym(r: PRodReader, info: TLineInfo): PSym = - var - id: int - ident: PIdent - result = nil - if r.s[r.pos] == '{': - inc(r.pos) - if r.s[r.pos] == '}': - inc(r.pos) - return # nil sym - var k = TSymKind(decodeVInt(r.s, r.pos)) - if r.s[r.pos] == '+': - inc(r.pos) - id = decodeVInt(r.s, r.pos) - setId(id) - else: - internalError(r.config, info, "decodeSym: no id") - if r.s[r.pos] == '&': - inc(r.pos) - ident = r.cache.getIdent(decodeStr(r.s, r.pos)) - else: - internalError(r.config, info, "decodeSym: no ident") - #echo "decoding: {", ident.s - result = r.syms.getOrDefault(id) - if result == nil: - new(result) - result.id = id - r.syms[result.id] = result - if debugIds: registerID(result) - elif result.id != id: - internalError(r.config, info, "decodeSym: wrong id") - elif result.kind != skStub and not r.inViewMode: - # we already loaded the symbol - return - else: - reset(result[]) - result.id = id - result.kind = k - result.name = ident # read the rest of the symbol description: - if r.s[r.pos] == '^': - inc(r.pos) - result.typ = rrGetType(r, decodeVInt(r.s, r.pos), info) - decodeLineInfo(r, result.info) - if r.s[r.pos] == '*': - inc(r.pos) - result.owner = rrGetSym(r, decodeVInt(r.s, r.pos), result.info) - if r.s[r.pos] == '$': - inc(r.pos) - result.flags = cast[TSymFlags](int32(decodeVInt(r.s, r.pos))) - if r.s[r.pos] == '@': - inc(r.pos) - result.magic = TMagic(decodeVInt(r.s, r.pos)) - if r.s[r.pos] == '!': - inc(r.pos) - result.options = cast[TOptions](int32(decodeVInt(r.s, r.pos))) - else: - result.options = r.options - if r.s[r.pos] == '%': - inc(r.pos) - result.position = decodeVInt(r.s, r.pos) - elif result.kind notin routineKinds + {skModule}: - result.position = 0 - # this may have been misused as reader index! But we still - # need it for routines as the body is loaded lazily. - if r.s[r.pos] == '`': - inc(r.pos) - result.offset = decodeVInt(r.s, r.pos) - else: - result.offset = - 1 - decodeLoc(r, result.loc, result.info) - result.annex = decodeLib(r, info) - if r.s[r.pos] == '#': - inc(r.pos) - result.constraint = decodeNode(r, unknownLineInfo()) - case result.kind - of skType, skGenericParam: - while r.s[r.pos] == '\14': - inc(r.pos) - result.typeInstCache.add rrGetType(r, decodeVInt(r.s, r.pos), result.info) - of routineKinds: - decodeInstantiations(r, result.info, result.procInstCache) - if r.s[r.pos] == '\16': - inc(r.pos) - result.gcUnsafetyReason = rrGetSym(r, decodeVInt(r.s, r.pos), result.info) - of skModule, skPackage: - decodeInstantiations(r, result.info, result.usedGenerics) - of skLet, skVar, skField, skForVar: - if r.s[r.pos] == '\18': - inc(r.pos) - result.guard = rrGetSym(r, decodeVInt(r.s, r.pos), result.info) - if r.s[r.pos] == '\19': - inc(r.pos) - result.bitsize = decodeVInt(r.s, r.pos).int16 - else: discard - - if r.s[r.pos] == '(': - if result.kind in routineKinds: - result.ast = decodeNodeLazyBody(r, result.info, result) - # since we load the body lazily, we need to set the reader to - # be able to reload: - result.position = r.readerIndex - else: - result.ast = decodeNode(r, result.info) - #echo "decoded: ", ident.s, "}" - -proc skipSection(r: PRodReader) = - if r.s[r.pos] == ':': - while r.s[r.pos] > '\x0A': inc(r.pos) - elif r.s[r.pos] == '(': - var c = 0 # count () pairs - inc(r.pos) - while true: - case r.s[r.pos] - of '\x0A': inc(r.line) - of '(': inc(c) - of ')': - if c == 0: - inc(r.pos) - break - elif c > 0: - dec(c) - of '\0': break # end of file - else: discard - inc(r.pos) - else: - internalError(r.config, "skipSection " & $r.line) - -proc rdWord(r: PRodReader): string = - result = "" - while r.s[r.pos] in {'A'..'Z', '_', 'a'..'z', '0'..'9'}: - add(result, r.s[r.pos]) - inc(r.pos) - -proc newStub(r: PRodReader, name: string, id: int): PSym = - new(result) - result.kind = skStub - result.id = id - result.name = r.cache.getIdent(name) - result.position = r.readerIndex - setId(id) #MessageOut(result.name.s); - if debugIds: registerID(result) - -proc processInterf(r: PRodReader, module: PSym) = - if r.interfIdx == 0: internalError(r.config, "processInterf") - r.pos = r.interfIdx - while (r.s[r.pos] > '\x0A') and (r.s[r.pos] != ')'): - var w = decodeStr(r.s, r.pos) - inc(r.pos) - var key = decodeVInt(r.s, r.pos) - inc(r.pos) # #10 - var s = newStub(r, w, key) - s.owner = module - strTableAdd(module.tab, s) - r.syms[s.id] = s - -proc processCompilerProcs(r: PRodReader, module: PSym) = - if r.compilerProcsIdx == 0: internalError(r.config, "processCompilerProcs") - r.pos = r.compilerProcsIdx - while (r.s[r.pos] > '\x0A') and (r.s[r.pos] != ')'): - var w = decodeStr(r.s, r.pos) - inc(r.pos) - var key = decodeVInt(r.s, r.pos) - inc(r.pos) # #10 - var s = r.syms.getOrDefault(key) - if s == nil: - s = newStub(r, w, key) - s.owner = module - r.syms[s.id] = s - strTableAdd(rodCompilerprocs, s) - -proc processIndex(r: PRodReader; idx: var TIndex; outf: File = nil) = - var key, val, tmp: int - inc(r.pos, 2) # skip "(\10" - inc(r.line) - while (r.s[r.pos] > '\x0A') and (r.s[r.pos] != ')'): - tmp = decodeVInt(r.s, r.pos) - if r.s[r.pos] == ' ': - inc(r.pos) - key = idx.lastIdxKey + tmp - val = decodeVInt(r.s, r.pos) + idx.lastIdxVal - else: - key = idx.lastIdxKey + 1 - val = tmp + idx.lastIdxVal - iiTablePut(idx.tab, key, val) - if not outf.isNil: outf.write(key, " ", val, "\n") - idx.lastIdxKey = key - idx.lastIdxVal = val - setId(key) # ensure that this id will not be used - if r.s[r.pos] == '\x0A': - inc(r.pos) - inc(r.line) - if r.s[r.pos] == ')': inc(r.pos) - -proc cmdChangeTriggersRecompilation(old, new: TCommands): bool = - if old == new: return false - # we use a 'case' statement without 'else' so that addition of a - # new command forces us to consider it here :-) - case old - of cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, - cmdCompileToJS, cmdCompileToLLVM: - if new in {cmdDoc, cmdCheck, cmdIdeTools, cmdPretty, cmdDef, - cmdInteractive}: - return false - of cmdNone, cmdDoc, cmdInterpret, cmdPretty, cmdGenDepend, cmdDump, - cmdCheck, cmdParse, cmdScan, cmdIdeTools, cmdDef, - cmdRst2html, cmdRst2tex, cmdInteractive, cmdRun, cmdJsonScript: - discard - # else: trigger recompilation: - result = true - -proc processRodFile(r: PRodReader, hash: SecureHash) = - var - w: string - d: int - var inclHash: SecureHash - while r.s[r.pos] != '\0': - var section = rdWord(r) - if r.reason != rrNone: - break # no need to process this file further - case section - of "HASH": - inc(r.pos) # skip ':' - if hash != parseSecureHash(decodeStr(r.s, r.pos)): - r.reason = rrHashChange - of "ID": - inc(r.pos) # skip ':' - r.moduleID = decodeVInt(r.s, r.pos) - setId(r.moduleID) - of "ORIGFILE": - inc(r.pos) - r.origFile = decodeStr(r.s, r.pos) - of "OPTIONS": - inc(r.pos) # skip ':' - r.options = cast[TOptions](int32(decodeVInt(r.s, r.pos))) - if r.config.options != r.options: r.reason = rrOptions - of "GOPTIONS": - inc(r.pos) # skip ':' - var dep = cast[TGlobalOptions](int32(decodeVInt(r.s, r.pos))) - if r.config.globalOptions-harmlessOptions != dep-harmlessOptions: - r.reason = rrOptions - of "CMD": - inc(r.pos) # skip ':' - var dep = cast[TCommands](int32(decodeVInt(r.s, r.pos))) - if cmdChangeTriggersRecompilation(dep, r.config.cmd): r.reason = rrOptions - of "DEFINES": - inc(r.pos) # skip ':' - d = 0 - while r.s[r.pos] > '\x0A': - w = decodeStr(r.s, r.pos) - inc(d) - if not isDefined(r.config, w): - r.reason = rrDefines #MessageOut('not defined, but should: ' + w); - if r.s[r.pos] == ' ': inc(r.pos) - if d != countDefinedSymbols(r.config.symbols): r.reason = rrDefines - of "FILES": - inc(r.pos, 2) # skip "(\10" - inc(r.line) - while r.s[r.pos] != ')': - let finalPath = decodeStr(r.s, r.pos) - #let resolvedPath = relativePath.findModule(r.origFile) - #let finalPath = if resolvedPath.len > 0: resolvedPath else: relativePath - r.files.add(fileInfoIdx(r.config, finalPath)) - inc(r.pos) # skip #10 - inc(r.line) - if r.s[r.pos] == ')': inc(r.pos) - of "INCLUDES": - inc(r.pos, 2) # skip "(\10" - inc(r.line) - while r.s[r.pos] != ')': - w = toFullPath(r.config, r.files[decodeVInt(r.s, r.pos)]) - inc(r.pos) # skip ' ' - inclHash = parseSecureHash(decodeStr(r.s, r.pos)) - if r.reason == rrNone: - if not existsFile(w) or (inclHash != secureHashFile(w)): - r.reason = rrInclDeps - if r.s[r.pos] == '\x0A': - inc(r.pos) - inc(r.line) - if r.s[r.pos] == ')': inc(r.pos) - of "DEPS": - inc(r.pos) # skip ':' - while r.s[r.pos] > '\x0A': - r.modDeps.add(r.files[int32(decodeVInt(r.s, r.pos))]) - if r.s[r.pos] == ' ': inc(r.pos) - of "INTERF": - r.interfIdx = r.pos + 2 - skipSection(r) - of "COMPILERPROCS": - r.compilerProcsIdx = r.pos + 2 - skipSection(r) - of "INDEX": - processIndex(r, r.index) - of "IMPORTS": - processIndex(r, r.imports) - of "CONVERTERS": - r.convertersIdx = r.pos + 1 - skipSection(r) - of "METHODS": - r.methodsIdx = r.pos + 1 - skipSection(r) - of "DATA": - r.dataIdx = r.pos + 2 # "(\10" - # We do not read the DATA section here! We read the needed objects on - # demand. And the DATA section comes last in the file, so we stop here: - break - of "INIT": - r.initIdx = r.pos + 2 # "(\10" - skipSection(r) - else: - internalError(r.config, "invalid section: '" & section & - "' at " & $r.line & " in " & r.filename) - #MsgWriteln("skipping section: " & section & - # " at " & $r.line & " in " & r.filename) - skipSection(r) - if r.s[r.pos] == '\x0A': - inc(r.pos) - inc(r.line) - - -proc startsWith(buf: cstring, token: string, pos = 0): bool = - var s = 0 - while s < token.len and buf[pos+s] == token[s]: inc s - result = s == token.len - -proc newRodReader(modfilename: string, hash: SecureHash, - readerIndex: int; cache: IdentCache; - config: ConfigRef): PRodReader = - new(result) - result.cache = cache - result.config = config - try: - result.memfile = memfiles.open(modfilename) - except OSError: - return nil - result.files = @[] - result.modDeps = @[] - result.methods = @[] - var r = result - r.reason = rrNone - r.pos = 0 - r.line = 1 - r.readerIndex = readerIndex - r.filename = modfilename - r.syms = initTable[int, PSym]() - # we terminate the file explicitly with ``\0``, so the cast to `cstring` - # is safe: - r.s = cast[cstring](r.memfile.mem) - if startsWith(r.s, "NIM:"): - initIiTable(r.index.tab) - initIiTable(r.imports.tab) # looks like a ROD file - inc(r.pos, 4) - var version = "" - while r.s[r.pos] notin {'\0', '\x0A'}: - add(version, r.s[r.pos]) - inc(r.pos) - if r.s[r.pos] == '\x0A': inc(r.pos) - if version != RodFileVersion: - # since ROD files are only for caching, no backwards compatibility is - # needed - #echo "expected version ", version, " ", RodFileVersion - result.memfile.close - result = nil - else: - result.memfile.close - result = nil - -proc rrGetType(r: PRodReader, id: int, info: TLineInfo): PType = - result = PType(idTableGet(gTypeTable, id)) - if result == nil: - # load the type: - var oldPos = r.pos - var d = iiTableGet(r.index.tab, id) - if d == InvalidKey: internalError(r.config, info, "rrGetType") - r.pos = d + r.dataIdx - result = decodeType(r, info) - r.pos = oldPos - -type - TFileModuleRec = object - filename*: string - reason*: TReasonForRecompile - rd*: PRodReader - hash*: SecureHash - hashDone*: bool - - TFileModuleMap = seq[TFileModuleRec] - -var gMods*: TFileModuleMap = @[] - -proc decodeSymSafePos(rd: PRodReader, offset: int, info: TLineInfo): PSym = - # all compiled modules - if rd.dataIdx == 0: internalError(rd.config, info, "dataIdx == 0") - var oldPos = rd.pos - rd.pos = offset + rd.dataIdx - result = decodeSym(rd, info) - rd.pos = oldPos - -proc findSomeWhere(id: int) = - for i in countup(0, high(gMods)): - var rd = gMods[i].rd - if rd != nil: - var d = iiTableGet(rd.index.tab, id) - if d != InvalidKey: - when declared(echo): - echo "found id ", id, " in ", gMods[i].filename - -proc getReader(moduleId: int): PRodReader = - # we can't index 'gMods' here as it's indexed by a *file index* which is not - # the module ID! We could introduce a mapping ID->PRodReader but I'll leave - # this for later versions if benchmarking shows the linear search causes - # problems: - for i in 0 ..< gMods.len: - result = gMods[i].rd - if result != nil and result.moduleID == moduleId: return result - return nil - -proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym = - result = r.syms.getOrDefault(id) - if result == nil: - # load the symbol: - var d = iiTableGet(r.index.tab, id) - if d == InvalidKey: - # import from other module: - var moduleID = iiTableGet(r.imports.tab, id) - if moduleID < 0: - var x = "" - encodeVInt(id, x) - internalError(r.config, info, "missing from both indexes: +" & x) - var rd = getReader(moduleID) - doAssert rd != nil - d = iiTableGet(rd.index.tab, id) - if d != InvalidKey: - result = decodeSymSafePos(rd, d, info) - else: - var x = "" - encodeVInt(id, x) - when false: findSomeWhere(id) - internalError(r.config, info, "rrGetSym: no reader found: +" & x) - else: - # own symbol: - result = decodeSymSafePos(r, d, info) - if result != nil and result.kind == skStub: rawLoadStub(result) - -proc loadInitSection*(r: PRodReader): PNode = - if r.initIdx == 0 or r.dataIdx == 0: internalError(r.config, "loadInitSection") - var oldPos = r.pos - r.pos = r.initIdx - result = newNode(nkStmtList) - while r.s[r.pos] > '\x0A' and r.s[r.pos] != ')': - var d = decodeVInt(r.s, r.pos) - inc(r.pos) # #10 - var p = r.pos - r.pos = d + r.dataIdx - addSon(result, decodeNode(r, unknownLineInfo())) - r.pos = p - r.pos = oldPos - -proc loadConverters(r: PRodReader) = - # We have to ensure that no exported converter is a stub anymore, and the - # import mechanism takes care of the rest. - if r.convertersIdx == 0 or r.dataIdx == 0: - internalError(r.config, "importConverters") - r.pos = r.convertersIdx - while r.s[r.pos] > '\x0A': - var d = decodeVInt(r.s, r.pos) - discard rrGetSym(r, d, unknownLineInfo()) - if r.s[r.pos] == ' ': inc(r.pos) - -proc loadMethods(r: PRodReader) = - if r.methodsIdx == 0 or r.dataIdx == 0: - internalError(r.config, "loadMethods") - r.pos = r.methodsIdx - while r.s[r.pos] > '\x0A': - var d = decodeVInt(r.s, r.pos) - r.methods.add(rrGetSym(r, d, unknownLineInfo())) - if r.s[r.pos] == ' ': inc(r.pos) - -proc getHash*(conf: ConfigRef; fileIdx: FileIndex): SecureHash = - if fileIdx.int32 <% gMods.len and gMods[fileIdx.int32].hashDone: - return gMods[fileIdx.int32].hash - - result = secureHashFile(toFullPath(conf, fileIdx)) - if fileIdx.int32 >= gMods.len: setLen(gMods, fileIdx.int32+1) - gMods[fileIdx.int32].hash = result - -template growCache*(cache, pos) = - if cache.len <= pos: cache.setLen(pos+1) - -proc checkDep(fileIdx: FileIndex; cache: IdentCache; conf: ConfigRef): TReasonForRecompile = - assert fileIdx != InvalidFileIDX - growCache gMods, fileIdx.int32 - if gMods[fileIdx.int32].reason != rrEmpty: - # reason has already been computed for this module: - return gMods[fileIdx.int32].reason - let filename = toFilename(conf, fileIdx) - var hash = getHash(conf, fileIdx) - gMods[fileIdx.int32].reason = rrNone # we need to set it here to avoid cycles - result = rrNone - var rodfile = toGeneratedFile(conf, conf.withPackageName(filename), RodExt) - var r = newRodReader(rodfile, hash, fileIdx.int32, cache, conf) - if r == nil: - result = (if existsFile(rodfile): rrRodInvalid else: rrRodDoesNotExist) - else: - processRodFile(r, hash) - result = r.reason - if result == rrNone: - # check modules it depends on - # NOTE: we need to process the entire module graph so that no ID will - # be used twice! However, compilation speed does not suffer much from - # this, since results are cached. - var res = checkDep(conf.m.systemFileIdx, cache, conf) - if res != rrNone: result = rrModDeps - for i in countup(0, high(r.modDeps)): - res = checkDep(r.modDeps[i], cache, conf) - if res != rrNone: - result = rrModDeps - # we cannot break here, because of side-effects of `checkDep` - if result != rrNone: - rawMessage(conf, hintProcessing, reasonToFrmt[result] % filename) - if result != rrNone or optForceFullMake in conf.globalOptions: - # recompilation is necessary: - if r != nil: memfiles.close(r.memfile) - r = nil - gMods[fileIdx.int32].rd = r - gMods[fileIdx.int32].reason = result # now we know better - -proc handleSymbolFile*(module: PSym; cache: IdentCache; conf: ConfigRef): PRodReader = - if conf.symbolFiles in {disabledSf, writeOnlySf, v2Sf}: - module.id = getID() - return nil - idgen.loadMaxIds(conf, conf.projectPath / conf.projectName) - let fileIdx = module.fileIdx - discard checkDep(fileIdx, cache, conf) - #if gMods[fileIdx.int32].reason == rrEmpty: internalError("handleSymbolFile") - result = gMods[fileIdx.int32].rd - if result != nil: - module.id = result.moduleID - result.syms[module.id] = module - processInterf(result, module) - processCompilerProcs(result, module) - loadConverters(result) - loadMethods(result) - else: - module.id = getID() - -proc rawLoadStub(s: PSym) = - assert s.kind == skStub - #if s.kind != skStub: internalError("loadStub") - var rd = gMods[s.position].rd - var theId = s.id # used for later check - var d = iiTableGet(rd.index.tab, s.id) - #if d == InvalidKey: internalError("loadStub: invalid key") - var rs = decodeSymSafePos(rd, d, unknownLineInfo()) - when false: - if rs != s: - #echo "rs: ", toHex(cast[int](rs.position), int.sizeof * 2), - # "\ns: ", toHex(cast[int](s.position), int.sizeof * 2) - internalError(rs.info, "loadStub: wrong symbol") - elif rs.id != theId: - internalError(rs.info, "loadStub: wrong ID") - -proc loadStub*(s: PSym) = - ## loads the stub symbol `s`. - - # deactivate the GC here because we do a deep recursion and generate no - # garbage when restoring parts of the object graph anyway. - # Since we die with internal errors if this fails, no try-finally is - # necessary. - GC_disable() - rawLoadStub(s) - GC_enable() - -proc getBody*(s: PSym): PNode = - ## retrieves the AST's body of `s`. If `s` has been loaded from a rod-file - ## it may perform an expensive reload operation. Otherwise it's a simple - ## accessor. - assert s.kind in routineKinds - # prevent crashes due to incorrect macro transformations (bug #2377) - if s.ast.isNil or bodyPos >= s.ast.len: return newNodeI(nkEmpty, s.info) - result = s.ast.sons[bodyPos] - if result == nil: - assert s.offset != 0 - var r = gMods[s.position].rd - var oldPos = r.pos - r.pos = s.offset - result = decodeNode(r, s.info) - r.pos = oldPos - s.ast.sons[bodyPos] = result - s.offset = 0 - -initIdTable(gTypeTable) -initStrTable(rodCompilerprocs) - -# viewer: -proc writeNode(f: File; n: PNode) = - f.write("(") - if n != nil: - f.write($n.kind) - if n.typ != nil: - f.write('^') - f.write(n.typ.id) - case n.kind - of nkCharLit..nkUInt64Lit: - if n.intVal != 0: - f.write('!') - f.write(n.intVal) - of nkFloatLit..nkFloat64Lit: - if n.floatVal != 0.0: - f.write('!') - f.write($n.floatVal) - of nkStrLit..nkTripleStrLit: - if n.strVal != "": - f.write('!') - f.write(n.strVal.escape) - of nkIdent: - f.write('!') - f.write(n.ident.s) - of nkSym: - f.write('!') - f.write(n.sym.id) - else: - for i in countup(0, sonsLen(n) - 1): - writeNode(f, n.sons[i]) - f.write(")") - -proc writeSym(f: File; s: PSym) = - if s == nil: - f.write("{}\n") - return - f.write("{") - f.write($s.kind) - f.write('+') - f.write(s.id) - f.write('&') - f.write(s.name.s) - if s.typ != nil: - f.write('^') - f.write(s.typ.id) - if s.owner != nil: - f.write('*') - f.write(s.owner.id) - if s.flags != {}: - f.write('$') - f.write($s.flags) - if s.magic != mNone: - f.write('@') - f.write($s.magic) - f.write('!') - f.write($s.options) - if s.position != 0: - f.write('%') - f.write($s.position) - if s.offset != -1: - f.write('`') - f.write($s.offset) - if s.constraint != nil: - f.write('#') - f.writeNode(s.constraint) - if s.ast != nil: - f.writeNode(s.ast) - f.write("}\n") - -proc writeType(f: File; t: PType) = - if t == nil: - f.write("[]\n") - return - f.write('[') - f.write($t.kind) - f.write('+') - f.write($t.id) - if t.n != nil: - f.writeNode(t.n) - if t.flags != {}: - f.write('$') - f.write($t.flags) - if t.callConv != low(t.callConv): - f.write('?') - f.write($t.callConv) - if t.owner != nil: - f.write('*') - f.write($t.owner.id) - if t.sym != nil: - f.write('&') - f.write(t.sym.id) - if t.size != -1: - f.write('/') - f.write($t.size) - if t.align != 2: - f.write('=') - f.write($t.align) - for i in countup(0, sonsLen(t) - 1): - if t.sons[i] == nil: - f.write("^()") - else: - f.write('^') - f.write($t.sons[i].id) - f.write("]\n") - -proc viewFile(rodfile: string) = - let conf = newConfigRef() - var r = newRodReader(rodfile, secureHash(""), 0, newIdentCache(), conf) - if r == nil: - rawMessage(conf, errGenerated, "cannot open file (or maybe wrong version):" & - rodfile) - return - r.inViewMode = true - var outf = system.open(rodfile.changeFileExt(".rod.txt"), fmWrite) - while r.s[r.pos] != '\0': - let section = rdWord(r) - case section - of "HASH": - inc(r.pos) # skip ':' - outf.writeLine("HASH:", $decodeVInt(r.s, r.pos)) - of "ID": - inc(r.pos) # skip ':' - r.moduleID = decodeVInt(r.s, r.pos) - setId(r.moduleID) - outf.writeLine("ID:", $r.moduleID) - of "ORIGFILE": - inc(r.pos) - r.origFile = decodeStr(r.s, r.pos) - outf.writeLine("ORIGFILE:", r.origFile) - of "OPTIONS": - inc(r.pos) # skip ':' - r.options = cast[TOptions](int32(decodeVInt(r.s, r.pos))) - outf.writeLine("OPTIONS:", $r.options) - of "GOPTIONS": - inc(r.pos) # skip ':' - let dep = cast[TGlobalOptions](int32(decodeVInt(r.s, r.pos))) - outf.writeLine("GOPTIONS:", $dep) - of "CMD": - inc(r.pos) # skip ':' - let dep = cast[TCommands](int32(decodeVInt(r.s, r.pos))) - outf.writeLine("CMD:", $dep) - of "DEFINES": - inc(r.pos) # skip ':' - var d = 0 - outf.write("DEFINES:") - while r.s[r.pos] > '\x0A': - let w = decodeStr(r.s, r.pos) - inc(d) - outf.write(" ", w) - if r.s[r.pos] == ' ': inc(r.pos) - outf.write("\n") - of "FILES": - inc(r.pos, 2) # skip "(\10" - inc(r.line) - outf.write("FILES(\n") - while r.s[r.pos] != ')': - let relativePath = decodeStr(r.s, r.pos) - let resolvedPath = findModule(conf, relativePath, r.origFile) - let finalPath = if resolvedPath.len > 0: resolvedPath else: relativePath - r.files.add(fileInfoIdx(conf, finalPath)) - inc(r.pos) # skip #10 - inc(r.line) - outf.writeLine finalPath - if r.s[r.pos] == ')': inc(r.pos) - outf.write(")\n") - of "INCLUDES": - inc(r.pos, 2) # skip "(\10" - inc(r.line) - outf.write("INCLUDES(\n") - while r.s[r.pos] != ')': - let w = r.files[decodeVInt(r.s, r.pos)] - inc(r.pos) # skip ' ' - let inclHash = decodeVInt(r.s, r.pos) - if r.s[r.pos] == '\x0A': - inc(r.pos) - inc(r.line) - outf.write(w.int32, " ", inclHash, "\n") - if r.s[r.pos] == ')': inc(r.pos) - outf.write(")\n") - of "DEPS": - inc(r.pos) # skip ':' - outf.write("DEPS:") - while r.s[r.pos] > '\x0A': - let v = int32(decodeVInt(r.s, r.pos)) - r.modDeps.add(r.files[v]) - if r.s[r.pos] == ' ': inc(r.pos) - outf.write(" ", r.files[v].int32) - outf.write("\n") - of "INTERF", "COMPILERPROCS": - inc r.pos, 2 - if section == "INTERF": r.interfIdx = r.pos - else: r.compilerProcsIdx = r.pos - outf.write(section, "(\n") - while (r.s[r.pos] > '\x0A') and (r.s[r.pos] != ')'): - let w = decodeStr(r.s, r.pos) - inc(r.pos) - let key = decodeVInt(r.s, r.pos) - inc(r.pos) # #10 - outf.write(w, " ", key, "\n") - if r.s[r.pos] == ')': inc r.pos - outf.write(")\n") - of "INDEX": - outf.write(section, "(\n") - processIndex(r, r.index, outf) - outf.write(")\n") - of "IMPORTS": - outf.write(section, "(\n") - processIndex(r, r.imports, outf) - outf.write(")\n") - of "CONVERTERS", "METHODS": - inc r.pos - if section == "METHODS": r.methodsIdx = r.pos - else: r.convertersIdx = r.pos - outf.write(section, ":") - while r.s[r.pos] > '\x0A': - let d = decodeVInt(r.s, r.pos) - outf.write(" ", $d) - if r.s[r.pos] == ' ': inc(r.pos) - outf.write("\n") - of "DATA": - inc(r.pos, 2) - r.dataIdx = r.pos - outf.write("DATA(\n") - while r.s[r.pos] != ')': - if r.s[r.pos] == '(': - outf.writeNode decodeNode(r, unknownLineInfo()) - outf.write("\n") - elif r.s[r.pos] == '[': - outf.writeType decodeType(r, unknownLineInfo()) - else: - outf.writeSym decodeSym(r, unknownLineInfo()) - if r.s[r.pos] == '\x0A': - inc(r.pos) - inc(r.line) - if r.s[r.pos] == ')': inc r.pos - outf.write(")\n") - of "INIT": - outf.write("INIT(\n") - inc r.pos, 2 - r.initIdx = r.pos - while r.s[r.pos] > '\x0A' and r.s[r.pos] != ')': - let d = decodeVInt(r.s, r.pos) - inc(r.pos) # #10 - #let p = r.pos - #r.pos = d + r.dataIdx - #outf.writeNode decodeNode(r, UnknownLineInfo()) - #outf.write("\n") - #r.pos = p - if r.s[r.pos] == ')': inc r.pos - outf.write("<not supported by viewer>)\n") - else: - internalError(r.config, "invalid section: '" & section & - "' at " & $r.line & " in " & r.filename) - skipSection(r) - if r.s[r.pos] == '\x0A': - inc(r.pos) - inc(r.line) - outf.close - -when isMainModule: - viewFile(paramStr(1).addFileExt(RodExt)) diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim deleted file mode 100644 index 968cb9e7f..000000000 --- a/compiler/rodwrite.nim +++ /dev/null @@ -1,653 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# This module is responsible for writing of rod files. Note that writing of -# rod files is a pass, reading of rod files is not! This is why reading and -# writing of rod files is split into two different modules. - -import - intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform, - condsyms, ropes, idents, std / sha1, rodread, passes, idgen, - rodutils, modulepaths, lineinfos - -from modulegraphs import ModuleGraph - -type - TRodWriter = object of TPassContext - module: PSym - hash: SecureHash - options: TOptions - defines: string - inclDeps: string - modDeps: string - interf: string - compilerProcs: string - index, imports: TIndex - converters, methods: string - init: string - data: string - sstack: TSymSeq # a stack of symbols to process - tstack: TTypeSeq # a stack of types to process - files: TStringSeq - origFile: string - cache: IdentCache - config: ConfigRef - - PRodWriter = ref TRodWriter - -proc getDefines(conf: ConfigRef): string = - result = "" - for d in definedSymbolNames(conf.symbols): - if result.len != 0: add(result, " ") - add(result, d) - -proc fileIdx(w: PRodWriter, filename: string): int = - for i in countup(0, high(w.files)): - if w.files[i] == filename: - return i - result = len(w.files) - setLen(w.files, result + 1) - w.files[result] = filename - -template filename*(w: PRodWriter): string = - toFilename(w.config, FileIndex w.module.position) - -proc newRodWriter(hash: SecureHash, module: PSym; cache: IdentCache; - config: ConfigRef): PRodWriter = - new(result) - result.config = config - result.sstack = @[] - result.tstack = @[] - initIiTable(result.index.tab) - initIiTable(result.imports.tab) - result.index.r = "" - result.imports.r = "" - result.hash = hash - result.module = module - result.defines = getDefines(config) - result.options = config.options - result.files = @[] - result.inclDeps = "" - result.modDeps = "" - result.interf = newStringOfCap(2_000) - result.compilerProcs = "" - result.converters = "" - result.methods = "" - result.init = "" - result.origFile = toFullPath(config, module.info) - result.data = newStringOfCap(12_000) - result.cache = cache - -proc addModDep(w: PRodWriter, dep: string; info: TLineInfo) = - if w.modDeps.len != 0: add(w.modDeps, ' ') - let resolved = findModule(w.config, dep, toFullPath(w.config, info)) - encodeVInt(fileIdx(w, resolved), w.modDeps) - -const - rodNL = "\x0A" - -proc addInclDep(w: PRodWriter, dep: string; info: TLineInfo) = - let resolved = findModule(w.config, dep, toFullPath(w.config, info)) - encodeVInt(fileIdx(w, resolved), w.inclDeps) - add(w.inclDeps, " ") - encodeStr($secureHashFile(resolved), w.inclDeps) - add(w.inclDeps, rodNL) - -proc pushType(w: PRodWriter, t: PType) = - # check so that the stack does not grow too large: - if iiTableGet(w.index.tab, t.id) == InvalidKey: - w.tstack.add(t) - -proc pushSym(w: PRodWriter, s: PSym) = - # check so that the stack does not grow too large: - if iiTableGet(w.index.tab, s.id) == InvalidKey: - when false: - if s.kind == skMethod: - echo "encoding ", s.id, " ", s.name.s - writeStackTrace() - w.sstack.add(s) - -proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, - result: var string) = - if n == nil: - # nil nodes have to be stored too: - result.add("()") - return - result.add('(') - encodeVInt(ord(n.kind), result) - # we do not write comments for now - # Line information takes easily 20% or more of the filesize! Therefore we - # omit line information if it is the same as the father's line information: - if fInfo.fileIndex != n.info.fileIndex: - result.add('?') - encodeVInt(n.info.col, result) - result.add(',') - encodeVInt(int n.info.line, result) - result.add(',') - encodeVInt(fileIdx(w, toFullPath(w.config, n.info)), result) - elif fInfo.line != n.info.line: - result.add('?') - encodeVInt(n.info.col, result) - result.add(',') - encodeVInt(int n.info.line, result) - elif fInfo.col != n.info.col: - result.add('?') - encodeVInt(n.info.col, result) - # No need to output the file index, as this is the serialization of one - # file. - var f = n.flags * PersistentNodeFlags - if f != {}: - result.add('$') - encodeVInt(cast[int32](f), result) - if n.typ != nil: - result.add('^') - encodeVInt(n.typ.id, result) - pushType(w, n.typ) - case n.kind - of nkCharLit..nkUInt64Lit: - if n.intVal != 0: - result.add('!') - encodeVBiggestInt(n.intVal, result) - of nkFloatLit..nkFloat64Lit: - if n.floatVal != 0.0: - result.add('!') - encodeStr($n.floatVal, result) - of nkStrLit..nkTripleStrLit: - if n.strVal != "": - result.add('!') - encodeStr(n.strVal, result) - of nkIdent: - result.add('!') - encodeStr(n.ident.s, result) - of nkSym: - result.add('!') - encodeVInt(n.sym.id, result) - pushSym(w, n.sym) - else: - for i in countup(0, sonsLen(n) - 1): - encodeNode(w, n.info, n.sons[i], result) - add(result, ')') - -proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) = - var oldLen = result.len - result.add('<') - if loc.k != low(loc.k): encodeVInt(ord(loc.k), result) - if loc.storage != low(loc.storage): - add(result, '*') - encodeVInt(ord(loc.storage), result) - if loc.flags != {}: - add(result, '$') - encodeVInt(cast[int32](loc.flags), result) - if loc.lode != nil: - add(result, '^') - encodeNode(w, unknownLineInfo(), loc.lode, result) - #encodeVInt(cast[int32](loc.t.id), result) - #pushType(w, loc.t) - if loc.r != nil: - add(result, '!') - encodeStr($loc.r, result) - if oldLen + 1 == result.len: - # no data was necessary, so remove the '<' again: - setLen(result, oldLen) - else: - add(result, '>') - -proc encodeType(w: PRodWriter, t: PType, result: var string) = - if t == nil: - # nil nodes have to be stored too: - result.add("[]") - return - # we need no surrounding [] here because the type is in a line of its own - if t.kind == tyForward: internalError(w.config, "encodeType: tyForward") - # for the new rodfile viewer we use a preceding [ so that the data section - # can easily be disambiguated: - add(result, '[') - encodeVInt(ord(t.kind), result) - add(result, '+') - encodeVInt(t.id, result) - if t.n != nil: - encodeNode(w, unknownLineInfo(), t.n, result) - if t.flags != {}: - add(result, '$') - encodeVInt(cast[int32](t.flags), result) - if t.callConv != low(t.callConv): - add(result, '?') - encodeVInt(ord(t.callConv), result) - if t.owner != nil: - add(result, '*') - encodeVInt(t.owner.id, result) - pushSym(w, t.owner) - if t.sym != nil: - add(result, '&') - encodeVInt(t.sym.id, result) - pushSym(w, t.sym) - if t.size != - 1: - add(result, '/') - encodeVBiggestInt(t.size, result) - if t.align != 2: - add(result, '=') - encodeVInt(t.align, result) - if t.lockLevel.ord != UnspecifiedLockLevel.ord: - add(result, '\14') - encodeVInt(t.lockLevel.int16, result) - if t.destructor != nil and t.destructor.id != 0: - add(result, '\15') - encodeVInt(t.destructor.id, result) - pushSym(w, t.destructor) - if t.deepCopy != nil: - add(result, '\16') - encodeVInt(t.deepcopy.id, result) - pushSym(w, t.deepcopy) - if t.assignment != nil: - add(result, '\17') - encodeVInt(t.assignment.id, result) - pushSym(w, t.assignment) - if t.sink != nil: - add(result, '\18') - encodeVInt(t.sink.id, result) - pushSym(w, t.sink) - for i, s in items(t.methods): - add(result, '\19') - encodeVInt(i, result) - add(result, '\20') - encodeVInt(s.id, result) - pushSym(w, s) - encodeLoc(w, t.loc, result) - for i in countup(0, sonsLen(t) - 1): - if t.sons[i] == nil: - add(result, "^()") - else: - add(result, '^') - encodeVInt(t.sons[i].id, result) - pushType(w, t.sons[i]) - -proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) = - add(result, '|') - encodeVInt(ord(lib.kind), result) - add(result, '|') - encodeStr($lib.name, result) - add(result, '|') - encodeNode(w, info, lib.path, result) - -proc encodeInstantiations(w: PRodWriter; s: seq[PInstantiation]; - result: var string) = - for t in s: - result.add('\15') - encodeVInt(t.sym.id, result) - pushSym(w, t.sym) - for tt in t.concreteTypes: - result.add('\17') - encodeVInt(tt.id, result) - pushType(w, tt) - result.add('\20') - encodeVInt(t.compilesId, result) - -proc encodeSym(w: PRodWriter, s: PSym, result: var string) = - if s == nil: - # nil nodes have to be stored too: - result.add("{}") - return - # we need no surrounding {} here because the symbol is in a line of its own - encodeVInt(ord(s.kind), result) - result.add('+') - encodeVInt(s.id, result) - result.add('&') - encodeStr(s.name.s, result) - if s.typ != nil: - result.add('^') - encodeVInt(s.typ.id, result) - pushType(w, s.typ) - result.add('?') - if s.info.col != -1'i16: encodeVInt(s.info.col, result) - result.add(',') - if s.info.line != 0'u16: encodeVInt(int s.info.line, result) - result.add(',') - encodeVInt(fileIdx(w, toFullPath(w.config, s.info)), result) - if s.owner != nil: - result.add('*') - encodeVInt(s.owner.id, result) - pushSym(w, s.owner) - if s.flags != {}: - result.add('$') - encodeVInt(cast[int32](s.flags), result) - if s.magic != mNone: - result.add('@') - encodeVInt(ord(s.magic), result) - if s.options != w.options: - result.add('!') - encodeVInt(cast[int32](s.options), result) - if s.position != 0: - result.add('%') - encodeVInt(s.position, result) - if s.offset != - 1: - result.add('`') - encodeVInt(s.offset, result) - encodeLoc(w, s.loc, result) - if s.annex != nil: encodeLib(w, s.annex, s.info, result) - if s.constraint != nil: - add(result, '#') - encodeNode(w, unknownLineInfo(), s.constraint, result) - case s.kind - of skType, skGenericParam: - for t in s.typeInstCache: - result.add('\14') - encodeVInt(t.id, result) - pushType(w, t) - of routineKinds: - encodeInstantiations(w, s.procInstCache, result) - if s.gcUnsafetyReason != nil: - result.add('\16') - encodeVInt(s.gcUnsafetyReason.id, result) - pushSym(w, s.gcUnsafetyReason) - of skModule, skPackage: - encodeInstantiations(w, s.usedGenerics, result) - # we don't serialize: - #tab*: TStrTable # interface table for modules - of skLet, skVar, skField, skForVar: - if s.guard != nil: - result.add('\18') - encodeVInt(s.guard.id, result) - pushSym(w, s.guard) - if s.bitsize != 0: - result.add('\19') - encodeVInt(s.bitsize, result) - else: discard - # lazy loading will soon reload the ast lazily, so the ast needs to be - # the last entry of a symbol: - if s.ast != nil: - # we used to attempt to save space here by only storing a dummy AST if - # it is not necessary, but Nim's heavy compile-time evaluation features - # make that unfeasible nowadays: - encodeNode(w, s.info, s.ast, result) - -proc addToIndex(w: var TIndex, key, val: int) = - if key - w.lastIdxKey == 1: - # we do not store a key-diff of 1 to safe space - encodeVInt(val - w.lastIdxVal, w.r) - else: - encodeVInt(key - w.lastIdxKey, w.r) - add(w.r, ' ') - encodeVInt(val - w.lastIdxVal, w.r) - add(w.r, rodNL) - w.lastIdxKey = key - w.lastIdxVal = val - iiTablePut(w.tab, key, val) - -const debugWrittenIds = false - -when debugWrittenIds: - var debugWritten = initIntSet() - -proc symStack(w: PRodWriter): int = - var i = 0 - while i < len(w.sstack): - var s = w.sstack[i] - if sfForward in s.flags: - w.sstack[result] = s - inc result - elif iiTableGet(w.index.tab, s.id) == InvalidKey: - var m = getModule(s) - #if m == nil and s.kind != skPackage and sfGenSym notin s.flags: - # internalError("symStack: module nil: " & s.name.s & " " & $s.kind & " ID " & $s.id) - if m == nil or s.kind == skPackage or {sfFromGeneric, sfGenSym} * s.flags != {} or m.id == w.module.id: - # put definition in here - var L = w.data.len - addToIndex(w.index, s.id, L) - when debugWrittenIds: incl(debugWritten, s.id) - encodeSym(w, s, w.data) - add(w.data, rodNL) - # put into interface section if appropriate: - if {sfExported, sfFromGeneric} * s.flags == {sfExported} and - s.kind in ExportableSymKinds: - encodeStr(s.name.s, w.interf) - add(w.interf, ' ') - encodeVInt(s.id, w.interf) - add(w.interf, rodNL) - if sfCompilerProc in s.flags: - encodeStr(s.name.s, w.compilerProcs) - add(w.compilerProcs, ' ') - encodeVInt(s.id, w.compilerProcs) - add(w.compilerProcs, rodNL) - if s.kind == skConverter or (s.ast != nil and hasPattern(s)): - if w.converters.len != 0: add(w.converters, ' ') - encodeVInt(s.id, w.converters) - if s.kind == skMethod and sfDispatcher notin s.flags: - if w.methods.len != 0: add(w.methods, ' ') - encodeVInt(s.id, w.methods) - elif iiTableGet(w.imports.tab, s.id) == InvalidKey: - addToIndex(w.imports, s.id, m.id) - when debugWrittenIds: - if not contains(debugWritten, s.id): - echo(w.filename) - debug(s) - debug(s.owner) - debug(m) - internalError("Symbol referred to but never written") - inc(i) - setLen(w.sstack, result) - -proc typeStack(w: PRodWriter): int = - var i = 0 - while i < len(w.tstack): - var t = w.tstack[i] - if t.kind == tyForward: - w.tstack[result] = t - inc result - elif iiTableGet(w.index.tab, t.id) == InvalidKey: - var L = w.data.len - addToIndex(w.index, t.id, L) - encodeType(w, t, w.data) - add(w.data, rodNL) - inc(i) - setLen(w.tstack, result) - -proc processStacks(w: PRodWriter, finalPass: bool) = - var oldS = 0 - var oldT = 0 - while true: - var slen = symStack(w) - var tlen = typeStack(w) - if slen == oldS and tlen == oldT: break - oldS = slen - oldT = tlen - if finalPass and (oldS != 0 or oldT != 0): - internalError(w.config, "could not serialize some forwarded symbols/types") - -proc rawAddInterfaceSym(w: PRodWriter, s: PSym) = - pushSym(w, s) - processStacks(w, false) - -proc addInterfaceSym(w: PRodWriter, s: PSym) = - if w == nil: return - if s.kind in ExportableSymKinds and - {sfExported, sfCompilerProc} * s.flags != {}: - rawAddInterfaceSym(w, s) - -proc addStmt(w: PRodWriter, n: PNode) = - encodeVInt(w.data.len, w.init) - add(w.init, rodNL) - encodeNode(w, unknownLineInfo(), n, w.data) - add(w.data, rodNL) - processStacks(w, false) - -proc writeRod(w: PRodWriter) = - processStacks(w, true) - var f: File - if not open(f, completeGeneratedFilePath(w.config, changeFileExt( - withPackageName(w.config, w.filename), RodExt)), - fmWrite): - #echo "couldn't write rod file for: ", w.filename - return - # write header: - f.write("NIM:") - f.write(RodFileVersion) - f.write(rodNL) - var id = "ID:" - encodeVInt(w.module.id, id) - f.write(id) - f.write(rodNL) - - var orig = "ORIGFILE:" - encodeStr(w.origFile, orig) - f.write(orig) - f.write(rodNL) - - var hash = "HASH:" - encodeStr($w.hash, hash) - f.write(hash) - f.write(rodNL) - - var options = "OPTIONS:" - encodeVInt(cast[int32](w.options), options) - f.write(options) - f.write(rodNL) - - var goptions = "GOPTIONS:" - encodeVInt(cast[int32](w.config.globalOptions), goptions) - f.write(goptions) - f.write(rodNL) - - var cmd = "CMD:" - encodeVInt(cast[int32](w.config.cmd), cmd) - f.write(cmd) - f.write(rodNL) - - f.write("DEFINES:") - f.write(w.defines) - f.write(rodNL) - - var files = "FILES(" & rodNL - for i in countup(0, high(w.files)): - encodeStr(w.files[i], files) - files.add(rodNL) - f.write(files) - f.write(')' & rodNL) - - f.write("INCLUDES(" & rodNL) - f.write(w.inclDeps) - f.write(')' & rodNL) - - f.write("DEPS:") - f.write(w.modDeps) - f.write(rodNL) - - f.write("INTERF(" & rodNL) - f.write(w.interf) - f.write(')' & rodNL) - - f.write("COMPILERPROCS(" & rodNL) - f.write(w.compilerProcs) - f.write(')' & rodNL) - - f.write("INDEX(" & rodNL) - f.write(w.index.r) - f.write(')' & rodNL) - - f.write("IMPORTS(" & rodNL) - f.write(w.imports.r) - f.write(')' & rodNL) - - f.write("CONVERTERS:") - f.write(w.converters) - f.write(rodNL) - - f.write("METHODS:") - f.write(w.methods) - f.write(rodNL) - - f.write("INIT(" & rodNL) - f.write(w.init) - f.write(')' & rodNL) - - f.write("DATA(" & rodNL) - f.write(w.data) - f.write(')' & rodNL) - # write trailing zero which is necessary because we use memory mapped files - # for reading: - f.write("\0") - f.close() - - #echo "interf: ", w.interf.len - #echo "index: ", w.index.r.len - #echo "init: ", w.init.len - #echo "data: ", w.data.len - -proc process(c: PPassContext, n: PNode): PNode = - result = n - if c == nil: return - var w = PRodWriter(c) - case n.kind - of nkStmtList: - for i in countup(0, sonsLen(n) - 1): discard process(c, n.sons[i]) - #var s = n.sons[namePos].sym - #addInterfaceSym(w, s) - of nkProcDef, nkFuncDef, nkIteratorDef, nkConverterDef, - nkTemplateDef, nkMacroDef: - let s = n.sons[namePos].sym - if s == nil: internalError(w.config, n.info, "rodwrite.process") - if n.sons[bodyPos] == nil: - internalError(w.config, n.info, "rodwrite.process: body is nil") - if n.sons[bodyPos].kind != nkEmpty or s.magic != mNone or - sfForward notin s.flags: - addInterfaceSym(w, s) - of nkMethodDef: - let s = n.sons[namePos].sym - if s == nil: internalError(w.config, n.info, "rodwrite.process") - if n.sons[bodyPos] == nil: - internalError(w.config, n.info, "rodwrite.process: body is nil") - if n.sons[bodyPos].kind != nkEmpty or s.magic != mNone or - sfForward notin s.flags: - pushSym(w, s) - processStacks(w, false) - - of nkVarSection, nkLetSection, nkConstSection: - for i in countup(0, sonsLen(n) - 1): - var a = n.sons[i] - if a.kind == nkCommentStmt: continue - addInterfaceSym(w, a.sons[0].sym) - of nkTypeSection: - for i in countup(0, sonsLen(n) - 1): - var a = n.sons[i] - if a.kind == nkCommentStmt: continue - if a.sons[0].kind != nkSym: internalError(w.config, a.info, "rodwrite.process") - var s = a.sons[0].sym - addInterfaceSym(w, s) - # this takes care of enum fields too - # Note: The check for ``s.typ.kind = tyEnum`` is wrong for enum - # type aliasing! Otherwise the same enum symbol would be included - # several times! - of nkImportStmt: - for i in countup(0, sonsLen(n) - 1): - addModDep(w, getModuleName(w.config, n.sons[i]), n.info) - addStmt(w, n) - of nkFromStmt, nkImportExceptStmt: - addModDep(w, getModuleName(w.config, n.sons[0]), n.info) - addStmt(w, n) - of nkIncludeStmt: - for i in countup(0, sonsLen(n) - 1): - addInclDep(w, getModuleName(w.config, n.sons[i]), n.info) - of nkPragma: - addStmt(w, n) - else: - discard - -proc myOpen(g: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = - if module.id < 0: internalError(g.config, "rodwrite: module ID not set") - var w = newRodWriter(rodread.getHash(g.config, FileIndex module.position), module, cache, g.config) - rawAddInterfaceSym(w, module) - result = w - -proc myClose(graph: ModuleGraph; c: PPassContext, n: PNode): PNode = - result = process(c, n) - var w = PRodWriter(c) - writeRod(w) - idgen.saveMaxIds(graph.config, graph.config.projectPath / graph.config.projectName) - -const rodwritePass* = makePass(open = myOpen, close = myClose, process = process) - diff --git a/compiler/sem.nim b/compiler/sem.nim index 01e816f5c..7872302fd 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -13,7 +13,7 @@ import ast, strutils, hashes, options, lexer, astalgo, trees, treetab, wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math, magicsys, parser, nversion, nimsets, semfold, modulepaths, importer, - procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch, + procfind, lookups, pragmas, passes, semdata, semtypinst, sigmatch, intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting, evaltempl, patterns, parampatterns, sempass2, nimfix.pretty, semmacrosanity, semparallel, lowerings, pluginsupport, plugins.active, rod, lineinfos @@ -513,10 +513,7 @@ proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = result = c proc myOpenCached(graph: ModuleGraph; module: PSym; rd: PRodReader): PPassContext = - result = myOpen(graph, module, rd.cache) - -proc replayMethodDefs(graph: ModuleGraph; rd: PRodReader) = - for m in items(rd.methods): methodDef(graph, m, true) + result = myOpen(graph, module, graph.cache) proc isImportSystemStmt(g: ModuleGraph; n: PNode): bool = if g.systemModule == nil: return false @@ -623,8 +620,6 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = addCodeForGenerics(c, result) if c.module.ast != nil: result.add(c.module.ast) - if c.rd != nil: - replayMethodDefs(graph, c.rd) popOwner(c) popProcCon(c) storeRemaining(c.graph, c.module) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 02e8be85c..c858b6839 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -13,7 +13,7 @@ import strutils, intsets, options, lexer, ast, astalgo, trees, treetab, wordrecg, ropes, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math, - magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef, + magicsys, nversion, nimsets, parser, times, passes, vmdef, modulegraphs, lineinfos type diff --git a/compiler/transf.nim b/compiler/transf.nim index 7aae2e4b5..e71afcec1 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -20,7 +20,7 @@ import intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os, - idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread, + idents, renderer, types, passes, semfold, magicsys, cgmeth, lambdalifting, sempass2, lowerings, lookups, destroyer, liftlocals, modulegraphs, lineinfos @@ -916,7 +916,7 @@ proc processTransf(c: PTransf, n: PNode, owner: PSym): PNode = # Note: For interactive mode we cannot call 'passes.skipCodegen' and skip # this step! We have to rely that the semantic pass transforms too errornous # nodes into an empty node. - if c.rd != nil or nfTransf in n.flags: return n + if nfTransf in n.flags: return n pushTransCon(c, newTransCon(owner)) result = PNode(transform(c, n)) popTransCon(c) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 7d8a1b45f..c641cc844 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -29,7 +29,7 @@ import strutils, ast, astalgo, types, msgs, renderer, vmdef, - trees, intsets, rodread, magicsys, options, lowerings, lineinfos + trees, intsets, magicsys, options, lowerings, lineinfos import platform from os import splitFile |