diff options
Diffstat (limited to 'compiler/ic')
| -rw-r--r-- | compiler/ic/bitabs.nim | 32 | ||||
| -rw-r--r-- | compiler/ic/cbackend.nim | 87 | ||||
| -rw-r--r-- | compiler/ic/dce.nim | 40 | ||||
| -rw-r--r-- | compiler/ic/design.rst | 8 | ||||
| -rw-r--r-- | compiler/ic/ic.nim | 811 | ||||
| -rw-r--r-- | compiler/ic/iclineinfos.nim | 84 | ||||
| -rw-r--r-- | compiler/ic/integrity.nim | 155 | ||||
| -rw-r--r-- | compiler/ic/navigator.nim | 183 | ||||
| -rw-r--r-- | compiler/ic/packed_ast.nim | 298 | ||||
| -rw-r--r-- | compiler/ic/replayer.nim | 40 | ||||
| -rw-r--r-- | compiler/ic/rodfiles.nim | 130 |
11 files changed, 1286 insertions, 582 deletions
diff --git a/compiler/ic/bitabs.nim b/compiler/ic/bitabs.nim index 1f75b7759..0c9994c83 100644 --- a/compiler/ic/bitabs.nim +++ b/compiler/ic/bitabs.nim @@ -1,7 +1,11 @@ ## A BiTable is a table that can be seen as an optimized pair -## of (Table[LitId, Val], Table[Val, LitId]). +## of `(Table[LitId, Val], Table[Val, LitId])`. -import hashes, rodfiles +import std/hashes +import rodfiles + +when defined(nimPreviewSlimSystem): + import std/assertions type LitId* = distinct uint32 @@ -10,6 +14,8 @@ type vals: seq[T] # indexed by LitId keys: seq[LitId] # indexed by hash(val) +proc initBiTable*[T](): BiTable[T] = BiTable[T](vals: @[], keys: @[]) + proc nextTry(h, maxHash: Hash): Hash {.inline.} = result = (h + 1) and maxHash @@ -30,12 +36,14 @@ proc mustRehash(length, counter: int): bool {.inline.} = result = (length * 2 < counter * 3) or (length - counter < 4) const - idStart = 256 ## - ## Ids do not start with 0 but with this value. The IR needs it. - ## TODO: explain why + idStart = 1 template idToIdx(x: LitId): int = x.int - idStart +proc hasLitId*[T](t: BiTable[T]; x: LitId): bool = + let idx = idToIdx(x) + result = idx >= 0 and idx < t.vals.len + proc enlarge[T](t: var BiTable[T]) = var n: seq[LitId] newSeq(n, len(t.keys) * 2) @@ -86,13 +94,13 @@ proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId = t.vals.add v -proc `[]`*[T](t: var BiTable[T]; LitId: LitId): var T {.inline.} = - let idx = idToIdx LitId +proc `[]`*[T](t: var BiTable[T]; litId: LitId): var T {.inline.} = + let idx = idToIdx litId assert idx < t.vals.len result = t.vals[idx] -proc `[]`*[T](t: BiTable[T]; LitId: LitId): lent T {.inline.} = - let idx = idToIdx LitId +proc `[]`*[T](t: BiTable[T]; litId: LitId): lent T {.inline.} = + let idx = idToIdx litId assert idx < t.vals.len result = t.vals[idx] @@ -111,6 +119,12 @@ proc load*[T](f: var RodFile; t: var BiTable[T]) = loadSeq(f, t.vals) loadSeq(f, t.keys) +proc sizeOnDisc*(t: BiTable[string]): int = + result = 4 + for x in t.vals: + result += x.len + 4 + result += t.keys.len * sizeof(LitId) + when isMainModule: var t: BiTable[string] diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index 34ee59d52..83f1b4cc7 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -19,9 +19,12 @@ ## anymore. DCE is now done as prepass over the entire packed module graph. import std/[packedsets, algorithm, tables] - # std/intsets would give `UnusedImport`, pending https://github.com/nim-lang/Nim/issues/14246 + +when defined(nimPreviewSlimSystem): + import std/assertions + import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen, - pathutils, extccomp, msgs] + pathutils, extccomp, msgs, modulepaths] import packed_ast, ic, dce, rodfiles @@ -30,12 +33,16 @@ proc unpackTree(g: ModuleGraph; thisModule: int; var decoder = initPackedDecoder(g.config, g.cache) result = loadNodes(decoder, g.packed, thisModule, tree, n) -proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) = +proc setupBackendModule(g: ModuleGraph; m: var LoadedModule) = if g.backend == nil: g.backend = cgendata.newModuleList(g) - + assert g.backend != nil var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config) bmod.idgen = idgenFromLoadedModule(m) + +proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) = + var bmod = BModuleList(g.backend).modules[m.module.position] + assert bmod != nil bmod.flags.incl useAliveDataFromDce bmod.alive = move alive[m.module.position] @@ -44,9 +51,13 @@ proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var Alive cgen.genTopLevelStmt(bmod, n) finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info)) + for disp in getDispatchers(g): + genProcAux(bmod, disp) + m.fromDisk.backendFlags = cgen.whichInitProcs(bmod) proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) = for x in mitems(m.fromDisk.emittedTypeInfo): + #echo "found type ", x, " for file ", int(origin) g.emittedTypeInfo[x] = origin proc addFileToLink(config: ConfigRef; m: PSym) = @@ -55,7 +66,8 @@ proc addFileToLink(config: ConfigRef; m: PSym) = if config.backend == backendCpp: ".nim.cpp" elif config.backend == backendObjc: ".nim.m" else: ".nim.c" - let cfile = changeFileExt(completeCfilePath(config, withPackageName(config, filename)), ext) + let cfile = changeFileExt(completeCfilePath(config, + mangleModuleName(config, filename).AbsoluteFile), ext) let objFile = completeCfilePath(config, toObjFile(config, cfile)) if fileExists(objFile): var cf = Cfile(nimname: m.name.s, cname: cfile, @@ -64,7 +76,7 @@ proc addFileToLink(config: ConfigRef; m: PSym) = addFileToCompile(config, cf) when defined(debugDce): - import std / [os, packedsets] + import os, std/packedsets proc storeAliveSymsImpl(asymFile: AbsoluteFile; s: seq[int32]) = var f = rodfiles.create(asymFile.string) @@ -88,7 +100,7 @@ proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool var f2 = rodfiles.open(asymFile.string) f2.loadHeader() f2.loadSection aliveSymsSection - var oldData: seq[int32] + var oldData: seq[int32] = @[] f2.loadSeq(oldData) f2.close if f2.err == ok and oldData == s: @@ -98,38 +110,71 @@ proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool let oldAsSet = toPackedSet[int32](oldData) let newAsSet = toPackedSet[int32](s) echo "set of live symbols changed ", asymFile.changeFileExt("rod"), " ", position, " ", f2.err - echo "in old but not in new ", oldAsSet.difference(newAsSet) - echo "in new but not in old ", newAsSet.difference(oldAsSet) - - if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0: - echo "command failed" + echo "in old but not in new ", oldAsSet.difference(newAsSet), " number of entries in old ", oldAsSet.len + echo "in new but not in old ", newAsSet.difference(oldAsSet), " number of entries in new ", newAsSet.len + #if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0: + # echo "command failed" result = true storeAliveSymsImpl(asymFile, s) +proc genPackedModule(g: ModuleGraph, i: int; alive: var AliveSyms) = + # case statement here to enforce exhaustive checks. + case g.packed[i].status + of undefined: + discard "nothing to do" + of loading, stored: + assert false + of storing, outdated: + storeAliveSyms(g.config, g.packed[i].module.position, alive) + generateCodeForModule(g, g.packed[i], alive) + closeRodFile(g, g.packed[i].module) + of loaded: + if g.packed[i].loadedButAliveSetChanged: + generateCodeForModule(g, g.packed[i], alive) + else: + addFileToLink(g.config, g.packed[i].module) + replayTypeInfo(g, g.packed[i], FileIndex(i)) + + if g.backend == nil: + g.backend = cgendata.newModuleList(g) + registerInitProcs(BModuleList(g.backend), g.packed[i].module, g.packed[i].fromDisk.backendFlags) + proc generateCode*(g: ModuleGraph) = ## The single entry point, generate C(++) code for the entire ## Nim program aka `ModuleGraph`. resetForBackend(g) var alive = computeAliveSyms(g.packed, g.config) - for i in 0..high(g.packed): + when false: + for i in 0..<len(g.packed): + echo i, " is of status ", g.packed[i].status, " ", toFullPath(g.config, FileIndex(i)) + + # First pass: Setup all the backend modules for all the modules that have + # changed: + for i in 0..<len(g.packed): # case statement here to enforce exhaustive checks. case g.packed[i].status of undefined: discard "nothing to do" - of loading: + of loading, stored: assert false of storing, outdated: - generateCodeForModule(g, g.packed[i], alive) - closeRodFile(g, g.packed[i].module) - storeAliveSyms(g.config, g.packed[i].module.position, alive) + setupBackendModule(g, g.packed[i]) of loaded: # Even though this module didn't change, DCE might trigger a change. # Consider this case: Module A uses symbol S from B and B does not use # S itself. A is then edited not to use S either. Thus we have to # recompile B in order to remove S from the final result. if aliveSymsChanged(g.config, g.packed[i].module.position, alive): - generateCodeForModule(g, g.packed[i], alive) - else: - addFileToLink(g.config, g.packed[i].module) - replayTypeInfo(g, g.packed[i], FileIndex(i)) + g.packed[i].loadedButAliveSetChanged = true + setupBackendModule(g, g.packed[i]) + + # Second pass: Code generation. + let mainModuleIdx = g.config.projectMainIdx2.int + # We need to generate the main module last, because only then + # all init procs have been registered: + for i in 0..<len(g.packed): + if i != mainModuleIdx: + genPackedModule(g, i, alive) + if mainModuleIdx >= 0: + genPackedModule(g, mainModuleIdx, alive) diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim index 0918fc379..6eb36431e 100644 --- a/compiler/ic/dce.nim +++ b/compiler/ic/dce.nim @@ -9,7 +9,11 @@ ## Dead code elimination (=DCE) for IC. -import std / [intsets, tables] +import std/[intsets, tables] + +when defined(nimPreviewSlimSystem): + import std/assertions + import ".." / [ast, options, lineinfos, types] import packed_ast, ic, bitabs @@ -27,7 +31,7 @@ type proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): bool = ## "Exported to C" procs are special (these are marked with '.exportc') because these ## must not be optimized away! - let symPtr = addr g[c.thisModule].fromDisk.sh.syms[symId] + let symPtr = unsafeAddr g[c.thisModule].fromDisk.syms[symId] let flags = symPtr.flags # due to a bug/limitation in the lambda lifting, unused inner procs # are not transformed correctly; issue (#411). However, the whole purpose here @@ -36,10 +40,14 @@ proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): boo if ({sfExportc, sfCompilerProc} * flags != {}) or (symPtr.kind == skMethod): result = true + else: + result = false # XXX: This used to be a condition to: # (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or if sfCompilerProc in flags: - c.compilerProcs[g[c.thisModule].fromDisk.sh.strings[symPtr.name]] = (c.thisModule, symId) + c.compilerProcs[g[c.thisModule].fromDisk.strings[symPtr.name]] = (c.thisModule, symId) + else: + result = false template isNotGeneric(n: NodePos): bool = ithSon(tree, n, genericParamsPos).kind == nkEmpty @@ -47,17 +55,17 @@ proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: i ## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will ## be analysed. if not c.alive[module].containsOrIncl(item): - var body = g[module].fromDisk.sh.syms[item].ast + var body = g[module].fromDisk.syms[item].ast if body != emptyNodeId: - let opt = g[module].fromDisk.sh.syms[item].options - if g[module].fromDisk.sh.syms[item].kind in routineKinds: + let opt = g[module].fromDisk.syms[item].options + if g[module].fromDisk.syms[item].kind in routineKinds: body = NodeId ithSon(g[module].fromDisk.bodies, NodePos body, bodyPos) c.stack.add((module, opt, NodePos(body))) when false: - let nid = g[module].fromDisk.sh.syms[item].name + let nid = g[module].fromDisk.syms[item].name if nid != LitId(0): - let name = g[module].fromDisk.sh.strings[nid] + let name = g[module].fromDisk.strings[nid] if name in ["nimFrame", "callDepthLimitReached"]: echo "I was called! ", name, " body exists: ", body != emptyNodeId, " ", module, " ", item @@ -66,12 +74,12 @@ proc requestCompilerProc(c: var AliveContext; g: PackedModuleGraph; name: string followLater(c, g, module, item) proc loadTypeKind(t: PackedItemId; c: AliveContext; g: PackedModuleGraph; toSkip: set[TTypeKind]): TTypeKind = - template kind(t: ItemId): TTypeKind = g[t.module].fromDisk.sh.types[t.item].kind + template kind(t: ItemId): TTypeKind = g[t.module].fromDisk.types[t.item].kind var t2 = translateId(t, g, c.thisModule, c.decoder.config) result = t2.kind while result in toSkip: - t2 = translateId(g[t2.module].fromDisk.sh.types[t2.item].types[^1], g, t2.module, c.decoder.config) + t2 = translateId(g[t2.module].fromDisk.types[t2.item].types[^1], g, t2.module, c.decoder.config) result = t2.kind proc rangeCheckAnalysis(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) = @@ -101,13 +109,13 @@ proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: N discard "ignore non-sym atoms" of nkSym: # This symbol is alive and everything its body references. - followLater(c, g, c.thisModule, n.operand) + followLater(c, g, c.thisModule, tree[n].soperand) of nkModuleRef: let (n1, n2) = sons2(tree, n) - assert n1.kind == nkInt32Lit - assert n2.kind == nkInt32Lit + assert n1.kind == nkNone + assert n2.kind == nkNone let m = n1.litId - let item = n2.operand + let item = tree[n2].soperand let otherModule = toFileIndexCached(c.decoder, g, c.thisModule, m).int followLater(c, g, otherModule, item) of nkMacroDef, nkTemplateDef, nkTypeSection, nkTypeOfExpr, @@ -123,7 +131,7 @@ proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: N rangeCheckAnalysis(c, g, tree, n) of nkProcDef, nkConverterDef, nkMethodDef, nkFuncDef, nkIteratorDef: if n.firstSon.kind == nkSym and isNotGeneric(n): - let item = n.firstSon.operand + let item = tree[n.firstSon].soperand if isExportedToC(c, g, item): # This symbol is alive and everything its body references. followLater(c, g, c.thisModule, item) @@ -145,7 +153,7 @@ proc computeAliveSyms*(g: PackedModuleGraph; conf: ConfigRef): AliveSyms = var c = AliveContext(stack: @[], decoder: PackedDecoder(config: conf), thisModule: -1, alive: newSeq[IntSet](g.len), options: conf.options) - for i in countdown(high(g), 0): + for i in countdown(len(g)-1, 0): if g[i].status != undefined: c.thisModule = i for p in allNodes(g[i].fromDisk.topLevel): diff --git a/compiler/ic/design.rst b/compiler/ic/design.rst index d8e1315b1..b096e3103 100644 --- a/compiler/ic/design.rst +++ b/compiler/ic/design.rst @@ -7,12 +7,8 @@ The frontend produces a set of `.rod` files. Every `.nim` module produces its own `.rod` file. - The IR must be a faithful representation of the AST in memory. -- The backend can do its own caching but doesn't have to. -- We know by comparing 'nim check compiler/nim' against 'nim c compiler/nim' - that 2/3 of the compiler's runtime is spent in the frontend. Hence we - implement IC for the frontend first and only later for the backend. The - backend will recompile everything until we implement its own caching - mechanisms. +- The backend can do its own caching but doesn't have to. In the + current implementation the backend also caches its results. Advantage of the "set of files" vs the previous global database: - By construction, we either read from the `.rod` file or from the diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 230b4d087..8e81633ef 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -7,12 +7,19 @@ # distribution, for details about the copyright. # -import std / [hashes, tables, intsets, sha1] +import std/[hashes, tables, intsets, monotimes] import packed_ast, bitabs, rodfiles import ".." / [ast, idents, lineinfos, msgs, ropes, options, - pathutils, condsyms] + pathutils, condsyms, packages, modulepaths] #import ".." / [renderer, astalgo] -from std / os import removeFile, isAbsolute +from std/os import removeFile, isAbsolute + +import ../../dist/checksums/src/checksums/sha1 + +import iclineinfos + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, formatfloat] type PackedConfig* = object @@ -22,29 +29,43 @@ type options: TOptions globalOptions: TGlobalOptions + ModuleBackendFlag* = enum + HasDatInitProc + HasModuleInitProc + PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file definedSymbols: string - includes: seq[(LitId, string)] # first entry is the module filename itself + moduleFlags: TSymFlags + includes*: seq[(LitId, string)] # first entry is the module filename itself imports: seq[LitId] # the modules this module depends on - toReplay: PackedTree # pragmas and VM specific state to replay. + toReplay*: PackedTree # pragmas and VM specific state to replay. topLevel*: PackedTree # top level statements bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position. #producedGenerics*: Table[GenericKey, SymId] exports*: seq[(LitId, int32)] - reexports*: seq[(LitId, PackedItemId)] + hidden: seq[(LitId, int32)] + reexports: seq[(LitId, PackedItemId)] compilerProcs*: seq[(LitId, int32)] converters*, methods*, trmacros*, pureEnums*: seq[int32] - macroUsages*: seq[(PackedItemId, PackedLineInfo)] typeInstCache*: seq[(PackedItemId, PackedItemId)] procInstCache*: seq[PackedInstantiation] - attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)] - methodsPerType*: seq[(PackedItemId, int, PackedItemId)] + attachedOps*: seq[(PackedItemId, TTypeAttachedOp, PackedItemId)] + methodsPerGenericType*: seq[(PackedItemId, int, PackedItemId)] enumToStringProcs*: seq[(PackedItemId, PackedItemId)] + methodsPerType*: seq[(PackedItemId, PackedItemId)] + dispatchers*: seq[PackedItemId] emittedTypeInfo*: seq[string] + backendFlags*: set[ModuleBackendFlag] + + syms*: OrderedTable[int32, PackedSym] + types*: OrderedTable[int32, PackedType] + strings*: BiTable[string] # we could share these between modules. + numbers*: BiTable[BiggestInt] # we also store floats in here so + # that we can assure that every bit is kept + man*: LineInfoManager - sh*: Shared cfg: PackedConfig PackedEncoder* = object @@ -59,6 +80,49 @@ type symMarker*: IntSet #Table[ItemId, SymId] # ItemId.item -> SymId config*: ConfigRef +proc toString*(tree: PackedTree; pos: NodePos; m: PackedModule; nesting: int; + result: var string) = + if result.len > 0 and result[^1] notin {' ', '\n'}: + result.add ' ' + + result.add $tree[pos].kind + case tree[pos].kind + of nkEmpty, nkNilLit, nkType: discard + of nkIdent, nkStrLit..nkTripleStrLit: + result.add " " + result.add m.strings[LitId tree[pos].uoperand] + of nkSym: + result.add " " + result.add m.strings[m.syms[tree[pos].soperand].name] + of directIntLit: + result.add " " + result.addInt tree[pos].soperand + of externSIntLit: + result.add " " + result.addInt m.numbers[LitId tree[pos].uoperand] + of externUIntLit: + result.add " " + result.addInt cast[uint64](m.numbers[LitId tree[pos].uoperand]) + of nkFloatLit..nkFloat128Lit: + result.add " " + result.addFloat cast[BiggestFloat](m.numbers[LitId tree[pos].uoperand]) + else: + result.add "(\n" + for i in 1..(nesting+1)*2: result.add ' ' + for child in sonsReadonly(tree, pos): + toString(tree, child, m, nesting + 1, result) + result.add "\n" + for i in 1..nesting*2: result.add ' ' + result.add ")" + #for i in 1..nesting*2: result.add ' ' + +proc toString*(tree: PackedTree; n: NodePos; m: PackedModule): string = + result = "" + toString(tree, n, m, 0, result) + +proc debug*(tree: PackedTree; m: PackedModule) = + stdout.write toString(tree, NodePos 0, m) + proc isActive*(e: PackedEncoder): bool = e.config != nil proc disable(e: var PackedEncoder) = e.config = nil @@ -83,14 +147,27 @@ proc rememberConfig(c: var PackedEncoder; m: var PackedModule; config: ConfigRef #primConfigFields rem m.cfg = pc +const + debugConfigDiff = defined(debugConfigDiff) + +when debugConfigDiff: + import hashes, tables, intsets, sha1, strutils, sets + proc configIdentical(m: PackedModule; config: ConfigRef): bool = result = m.definedSymbols == definedSymbolsAsString(config) - #if not result: - # echo "A ", m.definedSymbols, " ", definedSymbolsAsString(config) + when debugConfigDiff: + if not result: + var wordsA = m.definedSymbols.split(Whitespace).toHashSet() + var wordsB = definedSymbolsAsString(config).split(Whitespace).toHashSet() + for c in wordsA - wordsB: + echo "in A but not in B ", c + for c in wordsB - wordsA: + echo "in B but not in A ", c template eq(x) = result = result and m.cfg.x == config.x - #if not result: - # echo "B ", m.cfg.x, " ", config.x + when debugConfigDiff: + if m.cfg.x != config.x: + echo "B ", m.cfg.x, " ", config.x primConfigFields eq proc rememberStartupConfig*(dest: var PackedConfig, config: ConfigRef) = @@ -114,14 +191,14 @@ proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId = result = c.filenames.getOrDefault(x) if result == LitId(0): let p = msgs.toFullPath(c.config, x) - result = getOrIncl(m.sh.strings, p) + result = getOrIncl(m.strings, p) c.filenames[x] = result c.lastFile = x c.lastLit = result - assert result != LitId(0) + assert result != LitId(0) proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex = - result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x]) + result = msgs.fileInfoIdx(config, AbsoluteFile m.strings[x]) proc includesIdentical(m: var PackedModule; config: ConfigRef): bool = for it in mitems(m.includes): @@ -131,12 +208,14 @@ proc includesIdentical(m: var PackedModule; config: ConfigRef): bool = proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; config: ConfigRef; pc: PackedConfig) = ## setup a context for serializing to packed ast - m.sh = Shared() c.thisModule = moduleSym.itemId.module c.config = config + m.moduleFlags = moduleSym.flags m.bodies = newTreeFrom(m.topLevel) m.toReplay = newTreeFrom(m.topLevel) + c.lastFile = FileIndex(-10) + let thisNimFile = FileIndex c.thisModule var h = msgs.getHash(config, thisNimFile) if h.len == 0: @@ -155,11 +234,20 @@ proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) = m.imports.add toLitId(f, c, m) +proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + assert s.kind != skUnknown + let nameId = getOrIncl(m.strings, s.name.s) + m.hidden.add((nameId, s.itemId.item)) + assert s.itemId.module == c.thisModule + proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - let nameId = getOrIncl(m.sh.strings, s.name.s) + assert s.kind != skUnknown + assert s.itemId.module == c.thisModule + let nameId = getOrIncl(m.strings, s.name.s) m.exports.add((nameId, s.itemId.item)) proc addConverter*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + assert c.thisModule == s.itemId.module m.converters.add(s.itemId.item) proc addTrmacro*(c: var PackedEncoder; m: var PackedModule; s: PSym) = @@ -173,12 +261,14 @@ proc addMethod*(c: var PackedEncoder; m: var PackedModule; s: PSym) = m.methods.add s.itemId.item proc addReexport*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - let nameId = getOrIncl(m.sh.strings, s.name.s) + assert s.kind != skUnknown + if s.kind == skModule: return + let nameId = getOrIncl(m.strings, s.name.s) m.reexports.add((nameId, PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item))) proc addCompilerProc*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - let nameId = getOrIncl(m.sh.strings, s.name.s) + let nameId = getOrIncl(m.strings, s.name.s) m.compilerProcs.add((nameId, s.itemId.item)) proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) @@ -197,14 +287,15 @@ proc flush(c: var PackedEncoder; m: var PackedModule) = proc toLitId(x: string; m: var PackedModule): LitId = ## store a string as a literal - result = getOrIncl(m.sh.strings, x) + result = getOrIncl(m.strings, x) proc toLitId(x: BiggestInt; m: var PackedModule): LitId = ## store an integer as a literal - result = getOrIncl(m.sh.integers, x) + result = getOrIncl(m.numbers, x) proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo = - PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m)) + pack(m.man, toLitId(x.fileIndex, c, m), x.line.int32, x.col.int32) + #PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m)) proc safeItemId(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId {.inline.} = ## given a symbol, produce an ItemId with the correct properties @@ -246,81 +337,55 @@ proc storeTypeLater(t: PType; c: var PackedEncoder; m: var PackedModule): Packed # we only write one tree into m.bodies after the other. if t.isNil: return nilItemId - if t.uniqueId.module != c.thisModule: - # XXX Assert here that it already was serialized in the foreign module! - # it is a foreign type: - assert t.uniqueId.module >= 0 - assert t.uniqueId.item > 0 - return PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) - assert t.itemId.module >= 0 + assert t.uniqueId.module >= 0 assert t.uniqueId.item > 0 - result = PackedItemId(module: toLitId(t.itemId.module.FileIndex, c, m), item: t.uniqueId.item) - addMissing(c, t) + result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) + if t.uniqueId.module == c.thisModule: + # the type belongs to this module, so serialize it here, eventually. + addMissing(c, t) proc storeSymLater(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId = if s.isNil: return nilItemId assert s.itemId.module >= 0 - if s.itemId.module != c.thisModule: - # XXX Assert here that it already was serialized in the foreign module! - # it is a foreign symbol: - assert s.itemId.module >= 0 - return PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - assert s.itemId.module >= 0 + assert s.itemId.item >= 0 result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - addMissing(c, s) + if s.itemId.module == c.thisModule: + # the sym belongs to this module, so serialize it here, eventually. + addMissing(c, s) proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId = ## serialize a ptype if t.isNil: return nilItemId - if t.uniqueId.module != c.thisModule: - # XXX Assert here that it already was serialized in the foreign module! - # it is a foreign type: - assert t.uniqueId.module >= 0 - assert t.uniqueId.item > 0 - return PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) + assert t.uniqueId.module >= 0 + assert t.uniqueId.item > 0 + result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) - if not c.typeMarker.containsOrIncl(t.uniqueId.item): - if t.uniqueId.item >= m.sh.types.len: - setLen m.sh.types, t.uniqueId.item+1 + if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item): + #if t.uniqueId.item >= m.types.len: + # setLen m.types, t.uniqueId.item+1 - var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv, + var p = PackedType(id: t.uniqueId.item, kind: t.kind, flags: t.flags, callConv: t.callConv, size: t.size, align: t.align, nonUniqueId: t.itemId.item, - paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel) + paddingAtEnd: t.paddingAtEnd) storeNode(p, t, n) - - when false: - for op, s in pairs t.attachedOps: - c.addMissing s - p.attachedOps[op] = s.safeItemId(c, m) - p.typeInst = t.typeInst.storeType(c, m) - for kid in items t.sons: + for kid in kids t: p.types.add kid.storeType(c, m) - - when false: - for i, s in items t.methods: - c.addMissing s - p.methods.add (i, s.safeItemId(c, m)) c.addMissing t.sym p.sym = t.sym.safeItemId(c, m) c.addMissing t.owner p.owner = t.owner.safeItemId(c, m) # fill the reserved slot, nothing else: - m.sh.types[t.uniqueId.item] = p - - assert t.itemId.module >= 0 - assert t.uniqueId.item > 0 - result = PackedItemId(module: toLitId(t.itemId.module.FileIndex, c, m), item: t.uniqueId.item) + m.types[t.uniqueId.item] = p proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib = ## the plib hangs off the psym via the .annex field if l.isNil: return - result.kind = l.kind - result.generated = l.generated - result.isOverriden = l.isOverriden - result.name = toLitId($l.name, m) + result = PackedLib(kind: l.kind, generated: l.generated, + isOverridden: l.isOverridden, name: toLitId($l.name, m) + ) storeNode(result, l, path) proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId = @@ -328,21 +393,16 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId if s.isNil: return nilItemId assert s.itemId.module >= 0 + result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - if s.itemId.module != c.thisModule: - # XXX Assert here that it already was serialized in the foreign module! - # it is a foreign symbol: - assert s.itemId.module >= 0 - return PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - - if not c.symMarker.containsOrIncl(s.itemId.item): - if s.itemId.item >= m.sh.syms.len: - setLen m.sh.syms, s.itemId.item+1 + if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item): + #if s.itemId.item >= m.syms.len: + # setLen m.syms, s.itemId.item+1 assert sfForward notin s.flags - var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic, - position: s.position, offset: s.offset, options: s.options, + var p = PackedSym(id: s.itemId.item, kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic, + position: s.position, offset: s.offset, disamb: s.disamb, options: s.options, name: s.name.s.toLitId(m)) storeNode(p, s, ast) @@ -354,7 +414,7 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId p.bitsize = s.bitsize p.alignment = s.alignment - p.externalName = toLitId(if s.loc.r.isNil: "" else: $s.loc.r, m) + p.externalName = toLitId(s.loc.snippet, m) p.locFlags = s.loc.flags c.addMissing s.typ p.typ = s.typ.storeType(c, m) @@ -363,63 +423,67 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId p.annex = toPackedLib(s.annex, c, m) when hasFFI: p.cname = toLitId(s.cname, m) + p.instantiatedFrom = s.instantiatedFrom.safeItemId(c, m) # fill the reserved slot, nothing else: - m.sh.syms[s.itemId.item] = p - - assert s.itemId.module >= 0 - result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) + m.syms[s.itemId.item] = p proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = ## add a remote symbol reference to the tree let info = n.info.toPackedInfo(c, m) - ir.nodes.add PackedNode(kind: nkModuleRef, operand: 3.int32, # spans 3 nodes in total - typeId: storeTypeLater(n.typ, c, m), info: info) - ir.nodes.add PackedNode(kind: nkInt32Lit, info: info, - operand: toLitId(n.sym.itemId.module.FileIndex, c, m).int32) - ir.nodes.add PackedNode(kind: nkInt32Lit, info: info, - operand: n.sym.itemId.item) + if n.typ != n.sym.typ: + ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total + info = info, flags = n.flags, + typeId = storeTypeLater(n.typ, c, m)) + else: + ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total + info = info, flags = n.flags) + ir.addNode(kind = nkNone, info = info, + operand = toLitId(n.sym.itemId.module.FileIndex, c, m).int32) + ir.addNode(kind = nkNone, info = info, + operand = n.sym.itemId.item) proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = ## serialize a node into the tree if n == nil: - ir.nodes.add PackedNode(kind: nkNilRodNode, flags: {}, operand: 1) + ir.addNode(kind = nkNilRodNode, operand = 1, info = NoLineInfo) return let info = toPackedInfo(n.info, c, m) case n.kind of nkNone, nkEmpty, nkNilLit, nkType: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, operand: 0, - typeId: storeTypeLater(n.typ, c, m), info: info) + ir.addNode(kind = n.kind, flags = n.flags, operand = 0, + typeId = storeTypeLater(n.typ, c, m), info = info) of nkIdent: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.sh.strings, n.ident.s), - typeId: storeTypeLater(n.typ, c, m), info: info) + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.strings, n.ident.s), + typeId = storeTypeLater(n.typ, c, m), info = info) of nkSym: if n.sym.itemId.module == c.thisModule: # it is a symbol that belongs to the module we're currently # packing: let id = n.sym.storeSymLater(c, m).item - ir.nodes.add PackedNode(kind: nkSym, flags: n.flags, operand: id, - typeId: storeTypeLater(n.typ, c, m), info: info) + if n.typ != n.sym.typ: + ir.addNode(kind = nkSym, flags = n.flags, operand = id, + info = info, + typeId = storeTypeLater(n.typ, c, m)) + else: + ir.addNode(kind = nkSym, flags = n.flags, operand = id, + info = info) else: # store it as an external module reference: addModuleRef(n, ir, c, m) - of directIntLit: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32(n.intVal), - typeId: storeTypeLater(n.typ, c, m), info: info) of externIntLit: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.sh.integers, n.intVal), - typeId: storeTypeLater(n.typ, c, m), info: info) + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.numbers, n.intVal), + typeId = storeTypeLater(n.typ, c, m), info = info) of nkStrLit..nkTripleStrLit: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.sh.strings, n.strVal), - typeId: storeTypeLater(n.typ, c, m), info: info) + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.strings, n.strVal), + typeId = storeTypeLater(n.typ, c, m), info = info) of nkFloatLit..nkFloat128Lit: - ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.sh.floats, n.floatVal), - typeId: storeTypeLater(n.typ, c, m), info: info) + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)), + typeId = storeTypeLater(n.typ, c, m), info = info) else: let patchPos = ir.prepare(n.kind, n.flags, storeTypeLater(n.typ, c, m), info) @@ -444,8 +508,8 @@ proc toPackedProcDef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var # do not serialize the body of the proc, it's unnecessary since # n[0].sym.ast has the sem'checked variant of it which is what # everybody should use instead. - ir.nodes.add PackedNode(kind: nkEmpty, flags: {}, operand: 0, - typeId: nilItemId, info: info) + ir.addNode(kind = nkEmpty, flags = {}, operand = 0, + typeId = nilItemId, info = info) ir.patch patchPos proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var PackedModule) = @@ -464,6 +528,9 @@ proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var Pac of nkStmtList, nkStmtListExpr: for it in n: toPackedNodeIgnoreProcDefs(it, encoder, m) + of nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt, + nkFromStmt, nkIncludeStmt: + discard "nothing to do" else: toPackedNode(n, m.topLevel, encoder, m) @@ -480,6 +547,15 @@ proc toPackedGeneratedProcDef*(s: PSym, encoder: var PackedEncoder; m: var Packe toPackedProcDef(s.ast, m.topLevel, encoder, m) #flush encoder, m +proc storeAttachedProcDef*(t: PType; op: TTypeAttachedOp; s: PSym, + encoder: var PackedEncoder; m: var PackedModule) = + assert s.kind in routineKinds + assert isActive(encoder) + let tid = storeTypeLater(t, encoder, m) + let sid = storeSymLater(s, encoder, m) + m.attachedOps.add (tid, op, sid) + toPackedGeneratedProcDef(s, encoder, m) + proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) = var t = newSeq[PackedItemId](i.concreteTypes.len) for j in 0..high(i.concreteTypes): @@ -489,6 +565,9 @@ proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: concreteTypes: t) toPackedGeneratedProcDef(i.sym, c, m) +proc storeExpansion*(c: var PackedEncoder; m: var PackedModule; info: TLineInfo; s: PSym) = + toPackedNode(newSymNode(s, info), m.bodies, c, m) + proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) = case err of cannotOpen: @@ -496,16 +575,35 @@ proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) = of includeFileChanged: rawMessage(config, warnFileChanged, filename.string) else: - echo "Error: ", $err, " loading file: ", filename.string + rawMessage(config, warnCannotOpenFile, filename.string & " reason: " & $err) + #echo "Error: ", $err, " loading file: ", filename.string + +proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile = + result = changeFileExt(completeGeneratedFilePath(conf, + mangleModuleName(conf, f).AbsoluteFile), ext) + +const + BenchIC* = false + +when BenchIC: + var gloadBodies: MonoTime + + template bench(x, body) = + let start = getMonoTime() + body + x = x + (getMonoTime() - start) + +else: + template bench(x, body) = body proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef; ignoreConfig = false): RodFileError = - m.sh = Shared() var f = rodfiles.open(filename.string) f.loadHeader() f.loadSection configSection f.loadPrim m.definedSymbols + f.loadPrim m.moduleFlags f.loadPrim m.cfg if f.err == ok and not configIdentical(m, config) and not ignoreConfig: @@ -515,46 +613,59 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef f.loadSection section f.loadSeq data + template loadTableSection(section, data) {.dirty.} = + f.loadSection section + f.loadOrderedTable data + template loadTabSection(section, data) {.dirty.} = f.loadSection section f.load data - loadTabSection stringsSection, m.sh.strings + loadTabSection stringsSection, m.strings loadSeqSection checkSumsSection, m.includes - if not includesIdentical(m, config): + if config.cmd != cmdM and not includesIdentical(m, config): f.err = includeFileChanged loadSeqSection depsSection, m.imports - loadTabSection integersSection, m.sh.integers - loadTabSection floatsSection, m.sh.floats + bench gloadBodies: + + loadTabSection numbersSection, m.numbers - loadSeqSection exportsSection, m.exports + loadSeqSection exportsSection, m.exports + loadSeqSection hiddenSection, m.hidden + loadSeqSection reexportsSection, m.reexports - loadSeqSection reexportsSection, m.reexports + loadSeqSection compilerProcsSection, m.compilerProcs - loadSeqSection compilerProcsSection, m.compilerProcs + loadSeqSection trmacrosSection, m.trmacros - loadSeqSection trmacrosSection, m.trmacros + loadSeqSection convertersSection, m.converters + loadSeqSection methodsSection, m.methods + loadSeqSection pureEnumsSection, m.pureEnums - loadSeqSection convertersSection, m.converters - loadSeqSection methodsSection, m.methods - loadSeqSection pureEnumsSection, m.pureEnums - loadSeqSection macroUsagesSection, m.macroUsages + loadTabSection toReplaySection, m.toReplay + loadTabSection topLevelSection, m.topLevel - loadSeqSection toReplaySection, m.toReplay.nodes - loadSeqSection topLevelSection, m.topLevel.nodes - loadSeqSection bodiesSection, m.bodies.nodes - loadSeqSection symsSection, m.sh.syms - loadSeqSection typesSection, m.sh.types + loadTabSection bodiesSection, m.bodies + loadTableSection symsSection, m.syms + loadTableSection typesSection, m.types - loadSeqSection typeInstCacheSection, m.typeInstCache - loadSeqSection procInstCacheSection, m.procInstCache - loadSeqSection attachedOpsSection, m.attachedOps - loadSeqSection methodsPerTypeSection, m.methodsPerType - loadSeqSection enumToStringProcsSection, m.enumToStringProcs - loadSeqSection typeInfoSection, m.emittedTypeInfo + loadSeqSection typeInstCacheSection, m.typeInstCache + loadSeqSection procInstCacheSection, m.procInstCache + loadSeqSection attachedOpsSection, m.attachedOps + loadSeqSection methodsPerGenericTypeSection, m.methodsPerGenericType + loadSeqSection enumToStringProcsSection, m.enumToStringProcs + loadSeqSection methodsPerTypeSection, m.methodsPerType + loadSeqSection dispatchersSection, m.dispatchers + loadSeqSection typeInfoSection, m.emittedTypeInfo + + f.loadSection backendFlagsSection + f.loadPrim m.backendFlags + + f.loadSection sideChannelSection + f.load m.man close(f) result = f.err @@ -573,6 +684,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac f.storeHeader() f.storeSection configSection f.storePrim m.definedSymbols + f.storePrim m.moduleFlags f.storePrim m.cfg template storeSeqSection(section, data) {.dirty.} = @@ -583,17 +695,20 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac f.storeSection section f.store data - storeTabSection stringsSection, m.sh.strings + template storeTableSection(section, data) {.dirty.} = + f.storeSection section + f.storeOrderedTable data + + storeTabSection stringsSection, m.strings storeSeqSection checkSumsSection, m.includes storeSeqSection depsSection, m.imports - storeTabSection integersSection, m.sh.integers - storeTabSection floatsSection, m.sh.floats + storeTabSection numbersSection, m.numbers storeSeqSection exportsSection, m.exports - + storeSeqSection hiddenSection, m.hidden storeSeqSection reexportsSection, m.reexports storeSeqSection compilerProcsSection, m.compilerProcs @@ -602,23 +717,30 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac storeSeqSection convertersSection, m.converters storeSeqSection methodsSection, m.methods storeSeqSection pureEnumsSection, m.pureEnums - storeSeqSection macroUsagesSection, m.macroUsages - storeSeqSection toReplaySection, m.toReplay.nodes - storeSeqSection topLevelSection, m.topLevel.nodes + storeTabSection toReplaySection, m.toReplay + storeTabSection topLevelSection, m.topLevel - storeSeqSection bodiesSection, m.bodies.nodes - storeSeqSection symsSection, m.sh.syms + storeTabSection bodiesSection, m.bodies + storeTableSection symsSection, m.syms - storeSeqSection typesSection, m.sh.types + storeTableSection typesSection, m.types storeSeqSection typeInstCacheSection, m.typeInstCache storeSeqSection procInstCacheSection, m.procInstCache storeSeqSection attachedOpsSection, m.attachedOps - storeSeqSection methodsPerTypeSection, m.methodsPerType + storeSeqSection methodsPerGenericTypeSection, m.methodsPerGenericType storeSeqSection enumToStringProcsSection, m.enumToStringProcs + storeSeqSection methodsPerTypeSection, m.methodsPerType + storeSeqSection dispatchersSection, m.dispatchers storeSeqSection typeInfoSection, m.emittedTypeInfo + f.storeSection backendFlagsSection + f.storePrim m.backendFlags + + f.storeSection sideChannelSection + f.store m.man + close(f) encoder.disable() if f.err != ok: @@ -646,24 +768,43 @@ type storing, # state is strictly for stress-testing purposes loading, loaded, - outdated + outdated, + stored # store is complete, no further additions possible LoadedModule* = object status*: ModuleStatus - symsInit, typesInit: bool + symsInit, typesInit, loadedButAliveSetChanged*: bool fromDisk*: PackedModule - syms: seq[PSym] # indexed by itemId - types: seq[PType] + syms: OrderedTable[int32, PSym] # indexed by itemId + types: OrderedTable[int32, PType] module*: PSym # the one true module symbol. - iface: Table[PIdent, seq[PackedItemId]] # PackedItemId so that it works with reexported symbols too + iface, ifaceHidden: Table[PIdent, seq[PackedItemId]] + # PackedItemId so that it works with reexported symbols too + # ifaceHidden includes private symbols - PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex +type + PackedModuleGraph* = object + pm*: seq[LoadedModule] # indexed by FileIndex + when BenchIC: + depAnalysis: MonoTime + loadBody: MonoTime + loadSym, loadType, loadBodies: MonoTime + +when BenchIC: + proc echoTimes*(m: PackedModuleGraph) = + echo "analysis: ", m.depAnalysis, " loadBody: ", m.loadBody, " loadSym: ", + m.loadSym, " loadType: ", m.loadType, " all bodies: ", gloadBodies + +template `[]`*(m: PackedModuleGraph; i: int): LoadedModule = m.pm[i] +template len*(m: PackedModuleGraph): int = m.pm.len proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t: PackedItemId): PType proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s: PackedItemId): PSym proc toFileIndexCached*(c: var PackedDecoder; g: PackedModuleGraph; thisModule: int; f: LitId): FileIndex = - if c.lastLit == f and c.lastModule == thisModule: + if f == LitId(0): + result = InvalidFileIdx + elif c.lastLit == f and c.lastModule == thisModule: result = c.lastFile else: result = toFileIndex(f, g[thisModule].fromDisk, c.config) @@ -673,9 +814,10 @@ proc toFileIndexCached*(c: var PackedDecoder; g: PackedModuleGraph; thisModule: proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; x: PackedLineInfo): TLineInfo = - assert g[thisModule].status in {loaded, storing} - result = TLineInfo(line: x.line, col: x.col, - fileIndex: toFileIndexCached(c, g, thisModule, x.file)) + assert g[thisModule].status in {loaded, storing, stored} + let (fileId, line, col) = unpack(g[thisModule].fromDisk.man, x) + result = TLineInfo(line: line.uint16, col: col.int16, + fileIndex: toFileIndexCached(c, g, thisModule, fileId)) proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; tree: PackedTree; n: NodePos): PNode = @@ -689,26 +831,28 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; result.flags = n.flags case k - of nkEmpty, nkNilLit, nkType: + of nkNone, nkEmpty, nkNilLit, nkType: discard of nkIdent: - result.ident = getIdent(c.cache, g[thisModule].fromDisk.sh.strings[n.litId]) + result.ident = getIdent(c.cache, g[thisModule].fromDisk.strings[n.litId]) of nkSym: - result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand)) - of directIntLit: - result.intVal = tree.nodes[n.int].operand + result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree[n].soperand)) + if result.typ == nil: + result.typ = result.sym.typ of externIntLit: - result.intVal = g[thisModule].fromDisk.sh.integers[n.litId] + result.intVal = g[thisModule].fromDisk.numbers[n.litId] of nkStrLit..nkTripleStrLit: - result.strVal = g[thisModule].fromDisk.sh.strings[n.litId] + result.strVal = g[thisModule].fromDisk.strings[n.litId] of nkFloatLit..nkFloat128Lit: - result.floatVal = g[thisModule].fromDisk.sh.floats[n.litId] + result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.numbers[n.litId]) of nkModuleRef: let (n1, n2) = sons2(tree, n) - assert n1.kind == nkInt32Lit - assert n2.kind == nkInt32Lit + assert n1.kind == nkNone + assert n2.kind == nkNone transitionNoneToSym(result) - result.sym = loadSym(c, g, thisModule, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)) + result.sym = loadSym(c, g, thisModule, PackedItemId(module: n1.litId, item: tree[n2].soperand)) + if result.typ == nil: + result.typ = result.sym.typ else: for n0 in sonsReadonly(tree, n): result.addAllowNil loadNodes(c, g, thisModule, tree, n0) @@ -740,6 +884,7 @@ proc loadProcHeader(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: proc loadProcBody(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; tree: PackedTree; n: NodePos): PNode = + result = nil var i = 0 for n0 in sonsReadonly(tree, n): if i == bodyPos: @@ -757,8 +902,10 @@ proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; kind: s.kind, magic: s.magic, flags: s.flags, info: translateLineInfo(c, g, si, s.info), options: s.options, - position: s.position, - name: getIdent(c.cache, g[si].fromDisk.sh.strings[s.name]) + position: if s.kind in {skForVar, skVar, skLet, skTemp}: 0 else: s.position, + offset: if s.kind in routineKinds: defaultOffset else: s.offset, + disamb: s.disamb, + name: getIdent(c.cache, g[si].fromDisk.strings[s.name]) ) template loadAstBody(p, field) = @@ -775,8 +922,8 @@ proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph; if l.name.int == 0: result = nil else: - result = PLib(generated: l.generated, isOverriden: l.isOverriden, - kind: l.kind, name: rope g[si].fromDisk.sh.strings[l.name]) + result = PLib(generated: l.generated, isOverridden: l.isOverridden, + kind: l.kind, name: rope g[si].fromDisk.strings[l.name]) loadAstBody(l, path) proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; @@ -789,37 +936,53 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; loadAstBody(s, ast) result.annex = loadLib(c, g, si, item, s.annex) when hasFFI: - result.cname = g[si].fromDisk.sh.strings[s.cname] + result.cname = g[si].fromDisk.strings[s.cname] if s.kind in {skLet, skVar, skField, skForVar}: result.guard = loadSym(c, g, si, s.guard) result.bitsize = s.bitsize result.alignment = s.alignment result.owner = loadSym(c, g, si, s.owner) - let externalName = g[si].fromDisk.sh.strings[s.externalName] + let externalName = g[si].fromDisk.strings[s.externalName] if externalName != "": - result.loc.r = rope externalName + result.loc.snippet = externalName result.loc.flags = s.locFlags + result.instantiatedFrom = loadSym(c, g, si, s.instantiatedFrom) + +proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; + fileIdx: FileIndex; cachedModules: var seq[FileIndex]): bool +proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; + fileIdx: FileIndex; m: var LoadedModule) proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s: PackedItemId): PSym = if s == nilItemId: result = nil else: let si = moduleIndex(c, g, thisModule, s) - assert g[si].status in {loaded, storing} - if not g[si].symsInit: - g[si].symsInit = true - setLen g[si].syms, g[si].fromDisk.sh.syms.len - - if g[si].syms[s.item] == nil: - if g[si].fromDisk.sh.syms[s.item].kind != skModule: - result = symHeaderFromPacked(c, g, g[si].fromDisk.sh.syms[s.item], si, s.item) + if si >= g.len: + g.pm.setLen(si+1) + + if g[si].status == undefined and c.config.cmd == cmdM: + var cachedModules: seq[FileIndex] = @[] + discard needsRecompile(g, c.config, c.cache, FileIndex(si), cachedModules) + for m in cachedModules: + loadToReplayNodes(g, c.config, c.cache, m, g[int m]) + + assert g[si].status in {loaded, storing, stored} + #if not g[si].symsInit: + # g[si].symsInit = true + # setLen g[si].syms, g[si].fromDisk.syms.len + + if g[si].syms.getOrDefault(s.item) == nil: + if g[si].fromDisk.syms[s.item].kind != skModule: + result = symHeaderFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item) # store it here early on, so that recursions work properly: g[si].syms[s.item] = result - symBodyFromPacked(c, g, g[si].fromDisk.sh.syms[s.item], si, s.item, result) + symBodyFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item, result) else: result = g[si].module assert result != nil + g[si].syms[s.item] = result else: result = g[si].syms[s.item] @@ -828,7 +991,7 @@ proc typeHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedType; si, item: int32): PType = result = PType(itemId: ItemId(module: si, item: t.nonUniqueId), kind: t.kind, flags: t.flags, size: t.size, align: t.align, - paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel, + paddingAtEnd: t.paddingAtEnd, uniqueId: ItemId(module: si, item: item), callConv: t.callConv) @@ -840,8 +1003,10 @@ proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; for op, item in pairs t.attachedOps: result.attachedOps[op] = loadSym(c, g, si, item) result.typeInst = loadType(c, g, si, t.typeInst) + var sons = newSeq[PType]() for son in items t.types: - result.sons.add loadType(c, g, si, son) + sons.add loadType(c, g, si, son) + result.setSons(sons) loadAstBody(t, n) when false: for gen, id in items t.methods: @@ -852,42 +1017,43 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t result = nil else: let si = moduleIndex(c, g, thisModule, t) - assert g[si].status in {loaded, storing} + assert g[si].status in {loaded, storing, stored} assert t.item > 0 - if not g[si].typesInit: - g[si].typesInit = true - setLen g[si].types, g[si].fromDisk.sh.types.len + #if not g[si].typesInit: + # g[si].typesInit = true + # setLen g[si].types, g[si].fromDisk.types.len - if g[si].types[t.item] == nil: - result = typeHeaderFromPacked(c, g, g[si].fromDisk.sh.types[t.item], si, t.item) + if g[si].types.getOrDefault(t.item) == nil: + result = typeHeaderFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item) # store it here early on, so that recursions work properly: g[si].types[t.item] = result - typeBodyFromPacked(c, g, g[si].fromDisk.sh.types[t.item], si, t.item, result) + typeBodyFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item, result) + #assert result.itemId.item == t.item, $(result.itemId.item, t.item) + assert result.itemId.item > 0, $(result.itemId.item, t.item) else: result = g[si].types[t.item] - assert result.itemId.item > 0 - -proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym = - let filename = AbsoluteFile toFullPath(config, fileIdx) - let name = getIdent(cache, splitFile(filename).name) - let info = newLineInfo(fileIdx, 1, 1) - let - pck = getPackageName(config, filename.string) - pck2 = if pck.len > 0: pck else: "unknown" - pack = getIdent(cache, pck2) - result = newSym(skPackage, getIdent(cache, pck2), - ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info) + assert result.itemId.item > 0, "2" proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex; m: var LoadedModule) = m.iface = initTable[PIdent, seq[PackedItemId]]() - for e in m.fromDisk.exports: + m.ifaceHidden = initTable[PIdent, seq[PackedItemId]]() + template impl(iface, e) = let nameLit = e[0] - m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(PackedItemId(module: LitId(0), item: e[1])) - for re in m.fromDisk.reexports: - let nameLit = re[0] - m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(re[1]) + let e2 = + when e[1] is PackedItemId: e[1] + else: PackedItemId(module: LitId(0), item: e[1]) + iface.mgetOrPut(cache.getIdent(m.fromDisk.strings[nameLit]), @[]).add(e2) + + for e in m.fromDisk.exports: + m.iface.impl(e) + m.ifaceHidden.impl(e) + for e in m.fromDisk.reexports: + m.iface.impl(e) + m.ifaceHidden.impl(e) + for e in m.fromDisk.hidden: + m.ifaceHidden.impl(e) let filename = AbsoluteFile toFullPath(conf, fileIdx) # We cannot call ``newSym`` here, because we have to circumvent the ID @@ -896,9 +1062,8 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa name: getIdent(cache, splitFile(filename).name), info: newLineInfo(fileIdx, 1, 1), position: int(fileIdx)) - m.module.owner = newPackage(conf, cache, fileIdx) - if fileIdx == conf.projectMainIdx2: - m.module.flags.incl sfMainModule + m.module.owner = getPackage(conf, cache, fileIdx) + m.module.flags = m.fromDisk.moduleFlags proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex; m: var LoadedModule) = @@ -918,30 +1083,37 @@ proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache # Does the file belong to the fileIdx need to be recompiled? let m = int(fileIdx) if m >= g.len: - g.setLen(m+1) + g.pm.setLen(m+1) case g[m].status of undefined: g[m].status = loading let fullpath = msgs.toFullPath(conf, fileIdx) let rod = toRodFile(conf, AbsoluteFile fullpath) - let err = loadRodFile(rod, g[m].fromDisk, conf) + let err = loadRodFile(rod, g[m].fromDisk, conf, ignoreConfig = conf.cmd == cmdM) if err == ok: - result = optForceFullMake in conf.globalOptions - # check its dependencies: - for dep in g[m].fromDisk.imports: - let fid = toFileIndex(dep, g[m].fromDisk, conf) - # Warning: we need to traverse the full graph, so - # do **not use break here**! - if needsRecompile(g, conf, cache, fid, cachedModules): - result = true - - if not result: + if conf.cmd == cmdM: setupLookupTables(g, conf, cache, fileIdx, g[m]) cachedModules.add fileIdx g[m].status = loaded + result = false else: - g[m] = LoadedModule(status: outdated, module: g[m].module) + result = optForceFullMake in conf.globalOptions + # check its dependencies: + let imp = g[m].fromDisk.imports + for dep in imp: + let fid = toFileIndex(dep, g[m].fromDisk, conf) + # Warning: we need to traverse the full graph, so + # do **not use break here**! + if needsRecompile(g, conf, cache, fid, cachedModules): + result = true + + if not result: + setupLookupTables(g, conf, cache, fileIdx, g[m]) + cachedModules.add fileIdx + g[m].status = loaded + else: + g.pm[m] = LoadedModule(status: outdated, module: g[m].module) else: loadError(err, rod, conf) g[m].status = outdated @@ -950,18 +1122,19 @@ proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache of loading, loaded: # For loading: Assume no recompile is required. result = false - of outdated, storing: + of outdated, storing, stored: result = true proc moduleFromRodFile*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex; cachedModules: var seq[FileIndex]): PSym = ## Returns 'nil' if the module needs to be recompiled. - if needsRecompile(g, conf, cache, fileIdx, cachedModules): - result = nil - else: - result = g[int fileIdx].module - assert result != nil - assert result.position == int(fileIdx) + bench g.depAnalysis: + if needsRecompile(g, conf, cache, fileIdx, cachedModules): + result = nil + else: + result = g[int fileIdx].module + assert result != nil + assert result.position == int(fileIdx) for m in cachedModules: loadToReplayNodes(g, conf, cache, m, g[int m]) @@ -975,46 +1148,43 @@ template setupDecoder() {.dirty.} = proc loadProcBody*(config: ConfigRef, cache: IdentCache; g: var PackedModuleGraph; s: PSym): PNode = - let mId = s.itemId.module - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - let pos = g[mId].fromDisk.sh.syms[s.itemId.item].ast - assert pos != emptyNodeId - result = loadProcBody(decoder, g, mId, g[mId].fromDisk.bodies, NodePos pos) - -proc loadTypeFromId*(config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: int; id: PackedItemId): PType = - if id.item < g[module].types.len: - result = g[module].types[id.item] - else: - result = nil - if result == nil: + bench g.loadBody: + let mId = s.itemId.module var decoder = PackedDecoder( lastModule: int32(-1), lastLit: LitId(0), lastFile: FileIndex(-1), config: config, cache: cache) - result = loadType(decoder, g, module, id) + let pos = g[mId].fromDisk.syms[s.itemId.item].ast + assert pos != emptyNodeId + result = loadProcBody(decoder, g, mId, g[mId].fromDisk.bodies, NodePos pos) + +proc loadTypeFromId*(config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; module: int; id: PackedItemId): PType = + bench g.loadType: + result = g[module].types.getOrDefault(id.item) + if result == nil: + var decoder = PackedDecoder( + lastModule: int32(-1), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + result = loadType(decoder, g, module, id) proc loadSymFromId*(config: ConfigRef, cache: IdentCache; g: var PackedModuleGraph; module: int; id: PackedItemId): PSym = - if id.item < g[module].syms.len: - result = g[module].syms[id.item] - else: - result = nil - if result == nil: - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - result = loadSym(decoder, g, module, id) + bench g.loadSym: + result = g[module].syms.getOrDefault(id.item) + if result == nil: + var decoder = PackedDecoder( + lastModule: int32(-1), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + result = loadSym(decoder, g, module, id) proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; config: ConfigRef): ItemId = if id.module == LitId(0): @@ -1022,19 +1192,6 @@ proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; confi else: ItemId(module: toFileIndex(id.module, g[thisModule].fromDisk, config).int32, item: id.item) -proc checkForHoles(m: PackedModule; config: ConfigRef; moduleId: int) = - var bugs = 0 - for i in 1 .. high(m.sh.syms): - if m.sh.syms[i].kind == skUnknown: - echo "EMPTY ID ", i, " module ", moduleId, " ", toFullPath(config, FileIndex(moduleId)) - inc bugs - assert bugs == 0 - when false: - var nones = 0 - for i in 1 .. high(m.sh.types): - inc nones, m.sh.types[i].kind == tyNone - assert nones < 1 - proc simulateLoadedModule*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; moduleSym: PSym; m: PackedModule) = # For now only used for heavy debugging. In the future we could use this to reduce the @@ -1054,24 +1211,31 @@ type values: seq[PackedItemId] i, module: int +template interfSelect(a: LoadedModule, importHidden: bool): auto = + var ret = a.iface.addr + if importHidden: ret = a.ifaceHidden.addr + ret[] + proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache; g: var PackedModuleGraph; module: FileIndex; - name: PIdent): PSym = + name: PIdent, importHidden: bool): PSym = it.decoder = PackedDecoder( lastModule: int32(-1), lastLit: LitId(0), lastFile: FileIndex(-1), config: config, cache: cache) - it.values = g[int module].iface.getOrDefault(name) + it.values = g[int module].interfSelect(importHidden).getOrDefault(name) it.i = 0 it.module = int(module) if it.i < it.values.len: result = loadSym(it.decoder, g, int(module), it.values[it.i]) inc it.i + else: + result = nil proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: FileIndex): PSym = + g: var PackedModuleGraph; module: FileIndex; importHidden: bool): PSym = it.decoder = PackedDecoder( lastModule: int32(-1), lastLit: LitId(0), @@ -1080,23 +1244,27 @@ proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache; cache: cache) it.values = @[] it.module = int(module) - for v in g[int module].iface.values: + for v in g[int module].interfSelect(importHidden).values: it.values.add v it.i = 0 if it.i < it.values.len: result = loadSym(it.decoder, g, int(module), it.values[it.i]) inc it.i + else: + result = nil proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym = if it.i < it.values.len: result = loadSym(it.decoder, g, it.module, it.values[it.i]) inc it.i + else: + result = nil iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache; g: var PackedModuleGraph; module: FileIndex; - name: PIdent): PSym = + name: PIdent, importHidden: bool): PSym = setupDecoder() - let values = g[int module].iface.getOrDefault(name) + let values = g[int module].interfSelect(importHidden).getOrDefault(name) for pid in values: let s = loadSym(decoder, g, int(module), pid) assert s != nil @@ -1104,51 +1272,72 @@ iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache; proc interfaceSymbol*(config: ConfigRef, cache: IdentCache; g: var PackedModuleGraph; module: FileIndex; - name: PIdent): PSym = + name: PIdent, importHidden: bool): PSym = setupDecoder() - let values = g[int module].iface.getOrDefault(name) + let values = g[int module].interfSelect(importHidden).getOrDefault(name) result = loadSym(decoder, g, int(module), values[0]) proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator = - IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.sh.syms.len, - typeId: int32 m.fromDisk.sh.types.len) + IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.syms.len, + typeId: int32 m.fromDisk.types.len) proc searchForCompilerproc*(m: LoadedModule; name: string): int32 = # slow, linear search, but the results are cached: for it in items(m.fromDisk.compilerProcs): - if m.fromDisk.sh.strings[it[0]] == name: + if m.fromDisk.strings[it[0]] == name: return it[1] return -1 # ------------------------- .rod file viewer --------------------------------- proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = - var m: PackedModule + var m: PackedModule = PackedModule() let err = loadRodFile(rodfile, m, config, ignoreConfig=true) if err != ok: - echo "Error: could not load: ", rodfile.string, " reason: ", err - quit 1 + config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err - when true: + when false: echo "exports:" for ex in m.exports: - echo " ", m.sh.strings[ex[0]], " local ID: ", ex[1] - assert ex[0] == m.sh.syms[ex[1]].name + echo " ", m.strings[ex[0]], " local ID: ", ex[1] + assert ex[0] == m.syms[ex[1]].name # ex[1] int32 echo "reexports:" for ex in m.reexports: - echo " ", m.sh.strings[ex[0]] + echo " ", m.strings[ex[0]] # reexports*: seq[(LitId, PackedItemId)] - echo "all symbols" - for i in 0..high(m.sh.syms): - if m.sh.syms[i].name != LitId(0): - echo " ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i, " kind ", m.sh.syms[i].kind - else: - echo " <anon symbol?> local ID: ", i, " kind ", m.sh.syms[i].kind + echo "hidden: " & $m.hidden.len + for ex in m.hidden: + echo " ", m.strings[ex[0]], " local ID: ", ex[1] + + when false: + echo "all symbols" + for i in 0..high(m.syms): + if m.syms[i].name != LitId(0): + echo " ", m.strings[m.syms[i].name], " local ID: ", i, " kind ", m.syms[i].kind + else: + echo " <anon symbol?> local ID: ", i, " kind ", m.syms[i].kind + + echo "symbols: ", m.syms.len, " types: ", m.types.len, + " top level nodes: ", m.topLevel.len, " other nodes: ", m.bodies.len, + " strings: ", m.strings.len, " numbers: ", m.numbers.len - echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len, - " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len, - " strings: ", m.sh.strings.len, " integers: ", m.sh.integers.len, - " floats: ", m.sh.floats.len + echo "SIZES:" + echo "symbols: ", m.syms.len * sizeof(PackedSym), " types: ", m.types.len * sizeof(PackedType), + " top level nodes: ", m.topLevel.len * sizeof(PackedNode), + " other nodes: ", m.bodies.len * sizeof(PackedNode), + " strings: ", sizeOnDisc(m.strings) + when false: + var tt = 0 + var fc = 0 + for x in m.topLevel: + if x.kind == nkSym or x.typeId == nilItemId: inc tt + if x.flags == {}: inc fc + for x in m.bodies: + if x.kind == nkSym or x.typeId == nilItemId: inc tt + if x.flags == {}: inc fc + let total = float(m.topLevel.len + m.bodies.len) + echo "nodes with nil type: ", tt, " in % ", tt.float / total + echo "nodes with empty flags: ", fc.float / total diff --git a/compiler/ic/iclineinfos.nim b/compiler/ic/iclineinfos.nim new file mode 100644 index 000000000..74a7d971b --- /dev/null +++ b/compiler/ic/iclineinfos.nim @@ -0,0 +1,84 @@ +# +# +# The Nim Compiler +# (c) Copyright 2024 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# For the line information we use 32 bits. They are used as follows: +# Bit 0 (AsideBit): If we have inline line information or not. If not, the +# remaining 31 bits are used as an index into a seq[(LitId, int, int)]. +# +# We use 10 bits for the "file ID", this means a program can consist of as much +# as 1024 different files. (If it uses more files than that, the overflow bit +# would be set.) +# This means we have 21 bits left to encode the (line, col) pair. We use 7 bits for the column +# so 128 is the limit and 14 bits for the line number. +# The packed representation supports files with up to 16384 lines. +# Keep in mind that whenever any limit is reached the AsideBit is set and the real line +# information is kept in a side channel. + +import std / assertions + +const + AsideBit = 1 + FileBits = 10 + LineBits = 14 + ColBits = 7 + FileMax = (1 shl FileBits) - 1 + LineMax = (1 shl LineBits) - 1 + ColMax = (1 shl ColBits) - 1 + +static: + assert AsideBit + FileBits + LineBits + ColBits == 32 + +import .. / ic / [bitabs, rodfiles] # for LitId + +type + PackedLineInfo* = distinct uint32 + + LineInfoManager* = object + aside: seq[(LitId, int32, int32)] + +const + NoLineInfo* = PackedLineInfo(0'u32) + +proc pack*(m: var LineInfoManager; file: LitId; line, col: int32): PackedLineInfo = + if file.uint32 <= FileMax.uint32 and line <= LineMax and col <= ColMax: + let col = if col < 0'i32: 0'u32 else: col.uint32 + let line = if line < 0'i32: 0'u32 else: line.uint32 + # use inline representation: + result = PackedLineInfo((file.uint32 shl 1'u32) or (line shl uint32(AsideBit + FileBits)) or + (col shl uint32(AsideBit + FileBits + LineBits))) + else: + result = PackedLineInfo((m.aside.len shl 1) or AsideBit) + m.aside.add (file, line, col) + +proc unpack*(m: LineInfoManager; i: PackedLineInfo): (LitId, int32, int32) = + let i = i.uint32 + if (i and 1'u32) == 0'u32: + # inline representation: + result = (LitId((i shr 1'u32) and FileMax.uint32), + int32((i shr uint32(AsideBit + FileBits)) and LineMax.uint32), + int32((i shr uint32(AsideBit + FileBits + LineBits)) and ColMax.uint32)) + else: + result = m.aside[int(i shr 1'u32)] + +proc getFileId*(m: LineInfoManager; i: PackedLineInfo): LitId = + result = unpack(m, i)[0] + +proc store*(r: var RodFile; m: LineInfoManager) = storeSeq(r, m.aside) +proc load*(r: var RodFile; m: var LineInfoManager) = loadSeq(r, m.aside) + +when isMainModule: + var m = LineInfoManager(aside: @[]) + for i in 0'i32..<16388'i32: + for col in 0'i32..<100'i32: + let packed = pack(m, LitId(1023), i, col) + let u = unpack(m, packed) + assert u[0] == LitId(1023) + assert u[1] == i + assert u[2] == col + echo m.aside.len diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim new file mode 100644 index 000000000..3e8ea2503 --- /dev/null +++ b/compiler/ic/integrity.nim @@ -0,0 +1,155 @@ +# +# +# The Nim Compiler +# (c) Copyright 2021 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Integrity checking for a set of .rod files. +## The set must cover a complete Nim project. + +import std/[sets, tables] + +when defined(nimPreviewSlimSystem): + import std/assertions + +import ".." / [ast, modulegraphs] +import packed_ast, bitabs, ic + +type + CheckedContext = object + g: ModuleGraph + thisModule: int32 + checkedSyms: HashSet[ItemId] + checkedTypes: HashSet[ItemId] + +proc checkType(c: var CheckedContext; typeId: PackedItemId) +proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) +proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) + +proc checkTypeObj(c: var CheckedContext; typ: PackedType) = + for child in typ.types: + checkType(c, child) + if typ.n != emptyNodeId: + checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos typ.n) + if typ.sym != nilItemId: + checkForeignSym(c, typ.sym) + if typ.owner != nilItemId: + checkForeignSym(c, typ.owner) + checkType(c, typ.typeInst) + +proc checkType(c: var CheckedContext; typeId: PackedItemId) = + if typeId == nilItemId: return + let itemId = translateId(typeId, c.g.packed, c.thisModule, c.g.config) + if not c.checkedTypes.containsOrIncl(itemId): + let oldThisModule = c.thisModule + c.thisModule = itemId.module + checkTypeObj c, c.g.packed[itemId.module].fromDisk.types[itemId.item] + c.thisModule = oldThisModule + +proc checkSym(c: var CheckedContext; s: PackedSym) = + if s.name != LitId(0): + assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId s.name + checkType c, s.typ + if s.ast != emptyNodeId: + checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos s.ast) + if s.owner != nilItemId: + checkForeignSym(c, s.owner) + +proc checkLocalSym(c: var CheckedContext; item: int32) = + let itemId = ItemId(module: c.thisModule, item: item) + if not c.checkedSyms.containsOrIncl(itemId): + checkSym c, c.g.packed[c.thisModule].fromDisk.syms[item] + +proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) = + let itemId = translateId(symId, c.g.packed, c.thisModule, c.g.config) + if not c.checkedSyms.containsOrIncl(itemId): + let oldThisModule = c.thisModule + c.thisModule = itemId.module + checkSym c, c.g.packed[itemId.module].fromDisk.syms[itemId.item] + c.thisModule = oldThisModule + +proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) = + let t = findType(tree, n) + if t != nilItemId: + checkType(c, t) + case n.kind + of nkEmpty, nkNilLit, nkType, nkNilRodNode: + discard + of nkIdent: + assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId + of nkSym: + checkLocalSym(c, tree[n].soperand) + of directIntLit: + discard + of externIntLit, nkFloatLit..nkFloat128Lit: + assert c.g.packed[c.thisModule].fromDisk.numbers.hasLitId n.litId + of nkStrLit..nkTripleStrLit: + assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId + of nkModuleRef: + let (n1, n2) = sons2(tree, n) + assert n1.kind == nkNone + assert n2.kind == nkNone + checkForeignSym(c, PackedItemId(module: n1.litId, item: tree[n2].soperand)) + else: + for n0 in sonsReadonly(tree, n): + checkNode(c, tree, n0) + +proc checkTree(c: var CheckedContext; t: PackedTree) = + for p in allNodes(t): checkNode(c, t, p) + +proc checkLocalSymIds(c: var CheckedContext; m: PackedModule; symIds: seq[int32]) = + for symId in symIds: + assert symId >= 0 and symId < m.syms.len, $symId & " " & $m.syms.len + +proc checkModule(c: var CheckedContext; m: PackedModule) = + # We check that: + # - Every symbol references existing types and symbols. + # - Every tree node references existing types and symbols. + for _, v in pairs(m.syms): + checkLocalSym c, v.id + + checkTree c, m.toReplay + checkTree c, m.topLevel + + for e in m.exports: + #assert e[1] >= 0 and e[1] < m.syms.len + assert e[0] == m.syms[e[1]].name + + for e in m.compilerProcs: + #assert e[1] >= 0 and e[1] < m.syms.len + assert e[0] == m.syms[e[1]].name + + checkLocalSymIds c, m, m.converters + checkLocalSymIds c, m, m.methods + checkLocalSymIds c, m, m.trmacros + checkLocalSymIds c, m, m.pureEnums + #[ + To do: Check all these fields: + + reexports*: seq[(LitId, PackedItemId)] + macroUsages*: seq[(PackedItemId, PackedLineInfo)] + + typeInstCache*: seq[(PackedItemId, PackedItemId)] + procInstCache*: seq[PackedInstantiation] + attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)] + methodsPerGenericType*: seq[(PackedItemId, int, PackedItemId)] + enumToStringProcs*: seq[(PackedItemId, PackedItemId)] + methodsPerType*: seq[(PackedItemId, PackedItemId)] + dispatchers*: seq[PackedItemId] + ]# + +proc checkIntegrity*(g: ModuleGraph) = + var c = CheckedContext(g: g) + for i in 0..<len(g.packed): + # case statement here to enforce exhaustive checks. + case g.packed[i].status + of undefined: + discard "nothing to do" + of loading: + assert false, "cannot check integrity: Module still loading" + of stored, storing, outdated, loaded: + c.thisModule = int32 i + checkModule(c, g.packed[i].fromDisk) diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim new file mode 100644 index 000000000..39037b94f --- /dev/null +++ b/compiler/ic/navigator.nim @@ -0,0 +1,183 @@ +# +# +# The Nim Compiler +# (c) Copyright 2021 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Supports the "nim check --ic:on --defusages:FILE,LINE,COL" +## IDE-like features. It uses the set of .rod files to accomplish +## its task. The set must cover a complete Nim project. + +import std/[sets, tables] + +from std/os import nil +from std/private/miscdollars import toLocation + +when defined(nimPreviewSlimSystem): + import std/assertions + +import ".." / [ast, modulegraphs, msgs, options] +import iclineinfos +import packed_ast, bitabs, ic + +type + UnpackedLineInfo = object + file: LitId + line, col: int + NavContext = object + g: ModuleGraph + thisModule: int32 + trackPos: UnpackedLineInfo + alreadyEmitted: HashSet[string] + outputSep: char # for easier testing, use short filenames and spaces instead of tabs. + +proc isTracked(man: LineInfoManager; current: PackedLineInfo, trackPos: UnpackedLineInfo, tokenLen: int): bool = + let (currentFile, currentLine, currentCol) = man.unpack(current) + if currentFile == trackPos.file and currentLine == trackPos.line: + let col = trackPos.col + if col >= currentCol and col < currentCol+tokenLen: + result = true + else: + result = false + else: + result = false + +proc searchLocalSym(c: var NavContext; s: PackedSym; info: PackedLineInfo): bool = + result = s.name != LitId(0) and + isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[c.thisModule].fromDisk.strings[s.name].len) + +proc searchForeignSym(c: var NavContext; s: ItemId; info: PackedLineInfo): bool = + let name = c.g.packed[s.module].fromDisk.syms[s.item].name + result = name != LitId(0) and + isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[s.module].fromDisk.strings[name].len) + +const + EmptyItemId = ItemId(module: -1'i32, item: -1'i32) + +proc search(c: var NavContext; tree: PackedTree): ItemId = + # We use the linear representation here directly: + for i in 0..<len(tree): + let i = NodePos(i) + case tree[i].kind + of nkSym: + let item = tree[i].soperand + if searchLocalSym(c, c.g.packed[c.thisModule].fromDisk.syms[item], tree[i].info): + return ItemId(module: c.thisModule, item: item) + of nkModuleRef: + let (currentFile, currentLine, currentCol) = c.g.packed[c.thisModule].fromDisk.man.unpack(tree[i].info) + if currentLine == c.trackPos.line and currentFile == c.trackPos.file: + let (n1, n2) = sons2(tree, i) + assert n1.kind == nkInt32Lit + assert n2.kind == nkInt32Lit + let pId = PackedItemId(module: n1.litId, item: tree[n2].soperand) + let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config) + if searchForeignSym(c, itemId, tree[i].info): + return itemId + else: discard + return EmptyItemId + +proc isDecl(tree: PackedTree; n: NodePos): bool = + # XXX This is not correct yet. + const declarativeNodes = procDefs + {nkMacroDef, nkTemplateDef, + nkLetSection, nkVarSection, nkUsingStmt, nkConstSection, nkTypeSection, + nkIdentDefs, nkEnumTy, nkVarTuple} + result = n.int >= 0 and tree[n].kind in declarativeNodes + +proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) = + let (fileId, line, col) = unpack(c.g.packed[c.thisModule].fromDisk.man, info) + var m = "" + var file = c.g.packed[c.thisModule].fromDisk.strings[fileId] + if c.outputSep == ' ': + file = os.extractFilename file + toLocation(m, file, line, col + ColOffset) + if not c.alreadyEmitted.containsOrIncl(m): + msgWriteln c.g.config, (if isDecl: "def" else: "usage") & c.outputSep & m + +proc list(c: var NavContext; tree: PackedTree; sym: ItemId) = + for i in 0..<len(tree): + let i = NodePos(i) + case tree[i].kind + of nkSym: + let item = tree[i].soperand + if sym.item == item and sym.module == c.thisModule: + usage(c, tree[i].info, isDecl(tree, parent(i))) + of nkModuleRef: + let (n1, n2) = sons2(tree, i) + assert n1.kind == nkNone + assert n2.kind == nkNone + let pId = PackedItemId(module: n1.litId, item: tree[n2].soperand) + let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config) + if itemId.item == sym.item and sym.module == itemId.module: + usage(c, tree[i].info, isDecl(tree, parent(i))) + else: discard + +proc searchForIncludeFile(g: ModuleGraph; fullPath: string): int = + for i in 0..<len(g.packed): + for k in 1..high(g.packed[i].fromDisk.includes): + # we start from 1 because the first "include" file is + # the module's filename. + if os.cmpPaths(g.packed[i].fromDisk.strings[g.packed[i].fromDisk.includes[k][0]], fullPath) == 0: + return i + return -1 + +proc nav(g: ModuleGraph) = + # translate the track position to a packed position: + let unpacked = g.config.m.trackPos + var mid = unpacked.fileIndex.int + + let fullPath = toFullPath(g.config, unpacked.fileIndex) + + if g.packed[mid].status == undefined: + # check if 'mid' is an include file of some other module: + mid = searchForIncludeFile(g, fullPath) + + if mid < 0: + localError(g.config, unpacked, "unknown file name: " & fullPath) + return + + let fileId = g.packed[mid].fromDisk.strings.getKeyId(fullPath) + + if fileId == LitId(0): + internalError(g.config, unpacked, "cannot find a valid file ID") + return + + var c = NavContext( + g: g, + thisModule: int32 mid, + trackPos: UnpackedLineInfo(line: unpacked.line.int, col: unpacked.col.int, file: fileId), + outputSep: if isDefined(g.config, "nimIcNavigatorTests"): ' ' else: '\t' + ) + var symId = search(c, g.packed[mid].fromDisk.topLevel) + if symId == EmptyItemId: + symId = search(c, g.packed[mid].fromDisk.bodies) + + if symId == EmptyItemId: + localError(g.config, unpacked, "no symbol at this position") + return + + for i in 0..<len(g.packed): + # case statement here to enforce exhaustive checks. + case g.packed[i].status + of undefined: + discard "nothing to do" + of loading: + assert false, "cannot check integrity: Module still loading" + of stored, storing, outdated, loaded: + c.thisModule = int32 i + list(c, g.packed[i].fromDisk.topLevel, symId) + list(c, g.packed[i].fromDisk.bodies, symId) + +proc navDefinition*(g: ModuleGraph) = nav(g) +proc navUsages*(g: ModuleGraph) = nav(g) +proc navDefusages*(g: ModuleGraph) = nav(g) + +proc writeRodFiles*(g: ModuleGraph) = + for i in 0..<len(g.packed): + case g.packed[i].status + of undefined, loading, stored, loaded: + discard "nothing to do" + of storing, outdated: + closeRodFile(g, g.packed[i].module) diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim index 353cc3a42..a39bb7adf 100644 --- a/compiler/ic/packed_ast.nim +++ b/compiler/ic/packed_ast.nim @@ -12,10 +12,15 @@ ## use this representation directly in all the transformations, ## it is superior. -import std / [hashes, tables, strtabs] -import bitabs +import std/[hashes, tables, strtabs] +import bitabs, rodfiles import ".." / [ast, options] +import iclineinfos + +when defined(nimPreviewSlimSystem): + import std/assertions + type SymId* = distinct int32 ModuleId* = distinct int32 @@ -28,25 +33,21 @@ type item*: int32 # same as the in-memory representation const - nilItemId* = PackedItemId(module: LitId(0), item: -1.int32) + nilItemId* = PackedItemId(module: LitId(0), item: 0.int32) const emptyNodeId* = NodeId(-1) type - PackedLineInfo* = object - line*: uint16 - col*: int16 - file*: LitId - PackedLib* = object kind*: TLibKind generated*: bool - isOverriden*: bool + isOverridden*: bool name*: LitId path*: NodeId PackedSym* = object + id*: int32 kind*: TSymKind name*: LitId typ*: PackedItemId @@ -60,15 +61,18 @@ type alignment*: int # for alignment options*: TOptions position*: int - offset*: int + offset*: int32 + disamb*: int32 externalName*: LitId # instead of TLoc locFlags*: TLocFlags annex*: PackedLib when hasFFI: cname*: LitId constraint*: NodeId + instantiatedFrom*: PackedItemId PackedType* = object + id*: int32 kind*: TTypeKind callConv*: TCallingConvention #nodekind*: TNodeKind @@ -81,39 +85,39 @@ type size*: BiggestInt align*: int16 paddingAtEnd*: int16 - lockLevel*: TLockLevel # lock level as required for deadlock checking # not serialized: loc*: TLoc because it is backend-specific typeInst*: PackedItemId nonUniqueId*: int32 - PackedNode* = object # 20 bytes - kind*: TNodeKind - flags*: TNodeFlags - operand*: int32 # for kind in {nkSym, nkSymDef}: SymId - # for kind in {nkStrLit, nkIdent, nkNumberLit}: LitId - # for kind in nkInt32Lit: direct value - # for non-atom kinds: the number of nodes (for easy skipping) - typeId*: PackedItemId + PackedNode* = object # 8 bytes + x: uint32 info*: PackedLineInfo PackedTree* = object ## usually represents a full Nim module - nodes*: seq[PackedNode] - #sh*: Shared - - Shared* = ref object # shared between different versions of 'Module'. - # (though there is always exactly one valid - # version of a module) - syms*: seq[PackedSym] - types*: seq[PackedType] - strings*: BiTable[string] # we could share these between modules. - integers*: BiTable[BiggestInt] - floats*: BiTable[BiggestFloat] - #config*: ConfigRef + nodes: seq[PackedNode] + withFlags: seq[(int32, TNodeFlags)] + withTypes: seq[(int32, PackedItemId)] PackedInstantiation* = object key*, sym*: PackedItemId concreteTypes*: seq[PackedItemId] +const + NodeKindBits = 8'u32 + NodeKindMask = (1'u32 shl NodeKindBits) - 1'u32 + +template kind*(n: PackedNode): TNodeKind = TNodeKind(n.x and NodeKindMask) +template uoperand*(n: PackedNode): uint32 = (n.x shr NodeKindBits) +template soperand*(n: PackedNode): int32 = int32(uoperand(n)) + +template toX(k: TNodeKind; operand: uint32): uint32 = + uint32(k) or (operand shl NodeKindBits) + +template toX(k: TNodeKind; operand: LitId): uint32 = + uint32(k) or (operand.uint32 shl NodeKindBits) + +template typeId*(n: PackedNode): PackedItemId = n.typ + proc `==`*(a, b: SymId): bool {.borrow.} proc hash*(a: SymId): Hash {.borrow.} @@ -122,71 +126,35 @@ proc `==`*(a, b: NodePos): bool {.borrow.} proc `==`*(a, b: NodeId): bool {.borrow.} proc newTreeFrom*(old: PackedTree): PackedTree = - result.nodes = @[] + result = PackedTree(nodes: @[]) when false: result.sh = old.sh -when false: - proc declareSym*(tree: var PackedTree; kind: TSymKind; - name: LitId; info: PackedLineInfo): SymId = - result = SymId(tree.sh.syms.len) - tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info) - - proc litIdFromName*(tree: PackedTree; name: string): LitId = - result = tree.sh.strings.getOrIncl(name) - - proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) = - tree.nodes.add PackedNode(kind: kind, info: info, - operand: int32 getOrIncl(tree.sh.strings, token)) - - proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) = - tree.nodes.add PackedNode(kind: kind, operand: 0, info: info) - -proc throwAwayLastNode*(tree: var PackedTree) = - tree.nodes.setLen(tree.nodes.len-1) - proc addIdent*(tree: var PackedTree; s: LitId; info: PackedLineInfo) = - tree.nodes.add PackedNode(kind: nkIdent, operand: int32(s), info: info) + tree.nodes.add PackedNode(x: toX(nkIdent, uint32(s)), info: info) proc addSym*(tree: var PackedTree; s: int32; info: PackedLineInfo) = - tree.nodes.add PackedNode(kind: nkSym, operand: s, info: info) - -proc addModuleId*(tree: var PackedTree; s: ModuleId; info: PackedLineInfo) = - tree.nodes.add PackedNode(kind: nkInt32Lit, operand: int32(s), info: info) + tree.nodes.add PackedNode(x: toX(nkSym, cast[uint32](s)), info: info) proc addSymDef*(tree: var PackedTree; s: SymId; info: PackedLineInfo) = - tree.nodes.add PackedNode(kind: nkSym, operand: int32(s), info: info) + tree.nodes.add PackedNode(x: toX(nkSym, cast[uint32](s)), info: info) proc isAtom*(tree: PackedTree; pos: int): bool {.inline.} = tree.nodes[pos].kind <= nkNilLit -proc copyTree*(dest: var PackedTree; tree: PackedTree; n: NodePos) = - # and this is why the IR is superior. We can copy subtrees - # via a linear scan. - let pos = n.int - let L = if isAtom(tree, pos): 1 else: tree.nodes[pos].operand - let d = dest.nodes.len - dest.nodes.setLen(d + L) - for i in 0..<L: - dest.nodes[d+i] = tree.nodes[pos+i] - -when false: - proc copySym*(dest: var PackedTree; tree: PackedTree; s: SymId): SymId = - result = SymId(dest.sh.syms.len) - assert int(s) < tree.sh.syms.len - let oldSym = tree.sh.syms[s.int] - dest.sh.syms.add oldSym - type PatchPos = distinct int -when false: - proc prepare*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo): PatchPos = - result = PatchPos tree.nodes.len - tree.nodes.add PackedNode(kind: kind, operand: 0, info: info) +proc addNode*(t: var PackedTree; kind: TNodeKind; operand: int32; + typeId: PackedItemId = nilItemId; info: PackedLineInfo; + flags: TNodeFlags = {}) = + t.nodes.add PackedNode(x: toX(kind, cast[uint32](operand)), info: info) + if flags != {}: + t.withFlags.add (t.nodes.len.int32 - 1, flags) + if typeId != nilItemId: + t.withTypes.add (t.nodes.len.int32 - 1, typeId) proc prepare*(tree: var PackedTree; kind: TNodeKind; flags: TNodeFlags; typeId: PackedItemId; info: PackedLineInfo): PatchPos = result = PatchPos tree.nodes.len - tree.nodes.add PackedNode(kind: kind, flags: flags, operand: 0, info: info, - typeId: typeId) + tree.addNode(kind = kind, flags = flags, operand = 0, info = info, typeId = typeId) proc prepare*(dest: var PackedTree; source: PackedTree; sourcePos: NodePos): PatchPos = result = PatchPos dest.nodes.len @@ -194,26 +162,30 @@ proc prepare*(dest: var PackedTree; source: PackedTree; sourcePos: NodePos): Pat proc patch*(tree: var PackedTree; pos: PatchPos) = let pos = pos.int - assert tree.nodes[pos].kind > nkNilLit + let k = tree.nodes[pos].kind + assert k > nkNilLit let distance = int32(tree.nodes.len - pos) - tree.nodes[pos].operand = distance + assert distance > 0 + tree.nodes[pos].x = toX(k, cast[uint32](distance)) proc len*(tree: PackedTree): int {.inline.} = tree.nodes.len -proc `[]`*(tree: PackedTree; i: int): lent PackedNode {.inline.} = - tree.nodes[i] +proc `[]`*(tree: PackedTree; i: NodePos): lent PackedNode {.inline.} = + tree.nodes[i.int] + +template rawSpan(n: PackedNode): int = int(uoperand(n)) proc nextChild(tree: PackedTree; pos: var int) {.inline.} = if tree.nodes[pos].kind > nkNilLit: - assert tree.nodes[pos].operand > 0 - inc pos, tree.nodes[pos].operand + assert tree.nodes[pos].uoperand > 0 + inc pos, tree.nodes[pos].rawSpan else: inc pos iterator sonsReadonly*(tree: PackedTree; n: NodePos): NodePos = var pos = n.int assert tree.nodes[pos].kind > nkNilLit - let last = pos + tree.nodes[pos].operand + let last = pos + tree.nodes[pos].rawSpan inc pos while pos < last: yield NodePos pos @@ -234,7 +206,7 @@ iterator isons*(dest: var PackedTree; tree: PackedTree; iterator sonsFrom1*(tree: PackedTree; n: NodePos): NodePos = var pos = n.int assert tree.nodes[pos].kind > nkNilLit - let last = pos + tree.nodes[pos].operand + let last = pos + tree.nodes[pos].rawSpan inc pos if pos < last: nextChild tree, pos @@ -248,7 +220,7 @@ iterator sonsWithoutLast2*(tree: PackedTree; n: NodePos): NodePos = inc count var pos = n.int assert tree.nodes[pos].kind > nkNilLit - let last = pos + tree.nodes[pos].operand + let last = pos + tree.nodes[pos].rawSpan inc pos while pos < last and count > 2: yield NodePos pos @@ -258,9 +230,9 @@ iterator sonsWithoutLast2*(tree: PackedTree; n: NodePos): NodePos = proc parentImpl(tree: PackedTree; n: NodePos): NodePos = # finding the parent of a node is rather easy: var pos = n.int - 1 - while pos >= 0 and isAtom(tree, pos) or (pos + tree.nodes[pos].operand - 1 < n.int): + while pos >= 0 and (isAtom(tree, pos) or (pos + tree.nodes[pos].rawSpan - 1 < n.int)): dec pos - assert pos >= 0, "node has no parent" + #assert pos >= 0, "node has no parent" result = NodePos(pos) template parent*(n: NodePos): NodePos = parentImpl(tree, n) @@ -284,20 +256,32 @@ proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} = proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} = tree.nodes[n.int].kind proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} = - LitId tree.nodes[n.int].operand + LitId tree.nodes[n.int].uoperand proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} = tree.nodes[n.int].info +proc findType*(tree: PackedTree; n: NodePos): PackedItemId = + for x in tree.withTypes: + if x[0] == int32(n): return x[1] + if x[0] > int32(n): return nilItemId + return nilItemId + +proc findFlags*(tree: PackedTree; n: NodePos): TNodeFlags = + for x in tree.withFlags: + if x[0] == int32(n): return x[1] + if x[0] > int32(n): return {} + return {} + template typ*(n: NodePos): PackedItemId = - tree.nodes[n.int].typeId + tree.findType(n) template flags*(n: NodePos): TNodeFlags = - tree.nodes[n.int].flags + tree.findFlags(n) -template operand*(n: NodePos): int32 = - tree.nodes[n.int].operand +template uoperand*(n: NodePos): uint32 = + tree.nodes[n.int].uoperand proc span*(tree: PackedTree; pos: int): int {.inline.} = - if isAtom(tree, pos): 1 else: tree.nodes[pos].operand + if isAtom(tree, pos): 1 else: tree.nodes[pos].rawSpan proc sons2*(tree: PackedTree; n: NodePos): (NodePos, NodePos) = assert(not isAtom(tree, n.int)) @@ -313,6 +297,7 @@ proc sons3*(tree: PackedTree; n: NodePos): (NodePos, NodePos, NodePos) = result = (NodePos a, NodePos b, NodePos c) proc ithSon*(tree: PackedTree; n: NodePos; i: int): NodePos = + result = default(NodePos) if tree.nodes[n.int].kind > nkNilLit: var count = 0 for child in sonsReadonly(tree, n): @@ -326,105 +311,28 @@ when false: template kind*(n: NodePos): TNodeKind = tree.nodes[n.int].kind template info*(n: NodePos): PackedLineInfo = tree.nodes[n.int].info -template litId*(n: NodePos): LitId = LitId tree.nodes[n.int].operand +template litId*(n: NodePos): LitId = LitId tree.nodes[n.int].uoperand -template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].operand +template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].soperand proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1) -when false: - proc strLit*(tree: PackedTree; n: NodePos): lent string = - assert n.kind == nkStrLit - result = tree.sh.strings[LitId tree.nodes[n.int].operand] - - proc strVal*(tree: PackedTree; n: NodePos): string = - assert n.kind == nkStrLit - result = tree.sh.strings[LitId tree.nodes[n.int].operand] - #result = cookedStrLit(raw) - - proc filenameVal*(tree: PackedTree; n: NodePos): string = - case n.kind - of nkStrLit: - result = strVal(tree, n) - of nkIdent: - result = tree.sh.strings[n.litId] - of nkSym: - result = tree.sh.strings[tree.sh.syms[int n.symId].name] - else: - result = "" - - proc identAsStr*(tree: PackedTree; n: NodePos): lent string = - assert n.kind == nkIdent - result = tree.sh.strings[LitId tree.nodes[n.int].operand] - const externIntLit* = {nkCharLit, nkIntLit, nkInt8Lit, nkInt16Lit, + nkInt32Lit, nkInt64Lit, nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, - nkUInt64Lit} # nkInt32Lit is missing by design! + nkUInt64Lit} - externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt64Lit} + externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt32Lit, nkInt64Lit} externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit} - directIntLit* = nkInt32Lit - -proc toString*(tree: PackedTree; n: NodePos; sh: Shared; nesting: int; - result: var string) = - let pos = n.int - if result.len > 0 and result[^1] notin {' ', '\n'}: - result.add ' ' - - result.add $tree[pos].kind - case tree.nodes[pos].kind - of nkNone, nkEmpty, nkNilLit, nkType: discard - of nkIdent, nkStrLit..nkTripleStrLit: - result.add " " - result.add sh.strings[LitId tree.nodes[pos].operand] - of nkSym: - result.add " " - result.add sh.strings[sh.syms[tree.nodes[pos].operand].name] - of directIntLit: - result.add " " - result.addInt tree.nodes[pos].operand - of externSIntLit: - result.add " " - result.addInt sh.integers[LitId tree.nodes[pos].operand] - of externUIntLit: - result.add " " - result.add $cast[uint64](sh.integers[LitId tree.nodes[pos].operand]) - else: - result.add "(\n" - for i in 1..(nesting+1)*2: result.add ' ' - for child in sonsReadonly(tree, n): - toString(tree, child, sh, nesting + 1, result) - result.add "\n" - for i in 1..nesting*2: result.add ' ' - result.add ")" - #for i in 1..nesting*2: result.add ' ' - - -proc toString*(tree: PackedTree; n: NodePos; sh: Shared): string = - result = "" - toString(tree, n, sh, 0, result) - -proc debug*(tree: PackedTree; sh: Shared) = - stdout.write toString(tree, NodePos 0, sh) - -when false: - proc identIdImpl(tree: PackedTree; n: NodePos): LitId = - if n.kind == nkIdent: - result = n.litId - elif n.kind == nkSym: - result = tree.sh.syms[int n.symId].name - else: - result = LitId(0) - - template identId*(n: NodePos): LitId = identIdImpl(tree, n) + directIntLit* = nkNone template copyInto*(dest, n, body) = let patchPos = prepare(dest, tree, n) @@ -436,28 +344,8 @@ template copyIntoKind*(dest, kind, info, body) = body patch dest, patchPos -when false: - proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool = - let litId = tree.sh.strings.getKeyId(pragma) - if litId == LitId(0): - return false - assert n.kind == nkPragma - for ch0 in sonsReadonly(tree, n): - if ch0.kind == nkExprColonExpr: - if ch0.firstSon.identId == litId: - return true - elif ch0.identId == litId: - return true - proc getNodeId*(tree: PackedTree): NodeId {.inline.} = NodeId tree.nodes.len -when false: - proc produceError*(dest: var PackedTree; tree: PackedTree; n: NodePos; msg: string) = - let patchPos = prepare(dest, nkError, n.info) - dest.add nkStrLit, msg, n.info - copyTree(dest, tree, n) - patch dest, patchPos - iterator allNodes*(tree: PackedTree): NodePos = var p = 0 while p < tree.len: @@ -467,3 +355,13 @@ iterator allNodes*(tree: PackedTree): NodePos = proc toPackedItemId*(item: int32): PackedItemId {.inline.} = PackedItemId(module: LitId(0), item: item) + +proc load*(f: var RodFile; t: var PackedTree) = + loadSeq f, t.nodes + loadSeq f, t.withFlags + loadSeq f, t.withTypes + +proc store*(f: var RodFile; t: PackedTree) = + storeSeq f, t.nodes + storeSeq f, t.withFlags + storeSeq f, t.withTypes diff --git a/compiler/ic/replayer.nim b/compiler/ic/replayer.nim index 61aa0e697..b244ec885 100644 --- a/compiler/ic/replayer.nim +++ b/compiler/ic/replayer.nim @@ -14,7 +14,10 @@ import ".." / [ast, modulegraphs, trees, extccomp, btrees, msgs, lineinfos, pathutils, options, cgmeth] -import tables +import std/tables + +when defined(nimPreviewSlimSystem): + import std/assertions import packed_ast, ic, bitabs @@ -86,6 +89,31 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph) = else: internalAssert g.config, false +proc replayBackendProcs*(g: ModuleGraph; module: int) = + for it in mitems(g.packed[module].fromDisk.attachedOps): + let key = translateId(it[0], g.packed, module, g.config) + let op = it[1] + let tmp = translateId(it[2], g.packed, module, g.config) + let symId = FullId(module: tmp.module, packed: it[2]) + g.attachedOps[op][key] = LazySym(id: symId, sym: nil) + + for it in mitems(g.packed[module].fromDisk.enumToStringProcs): + let key = translateId(it[0], g.packed, module, g.config) + let tmp = translateId(it[1], g.packed, module, g.config) + let symId = FullId(module: tmp.module, packed: it[1]) + g.enumToStringProcs[key] = LazySym(id: symId, sym: nil) + + for it in mitems(g.packed[module].fromDisk.methodsPerType): + let key = translateId(it[0], g.packed, module, g.config) + let tmp = translateId(it[1], g.packed, module, g.config) + let symId = FullId(module: tmp.module, packed: it[1]) + g.methodsPerType.mgetOrPut(key, @[]).add LazySym(id: symId, sym: nil) + + for it in mitems(g.packed[module].fromDisk.dispatchers): + let tmp = translateId(it, g.packed, module, g.config) + let symId = FullId(module: tmp.module, packed: it) + g.dispatchers.add LazySym(id: symId, sym: nil) + proc replayGenericCacheInformation*(g: ModuleGraph; module: int) = ## We remember the generic instantiations a module performed ## in order to to avoid the code bloat that generic code tends @@ -110,18 +138,14 @@ proc replayGenericCacheInformation*(g: ModuleGraph; module: int) = module: module, sym: FullId(module: sym.module, packed: it.sym), concreteTypes: concreteTypes, inst: nil) - for it in mitems(g.packed[module].fromDisk.methodsPerType): + for it in mitems(g.packed[module].fromDisk.methodsPerGenericType): let key = translateId(it[0], g.packed, module, g.config) let col = it[1] let tmp = translateId(it[2], g.packed, module, g.config) let symId = FullId(module: tmp.module, packed: it[2]) - g.methodsPerType.mgetOrPut(key, @[]).add (col, LazySym(id: symId, sym: nil)) + g.methodsPerGenericType.mgetOrPut(key, @[]).add (col, LazySym(id: symId, sym: nil)) - for it in mitems(g.packed[module].fromDisk.enumToStringProcs): - let key = translateId(it[0], g.packed, module, g.config) - let tmp = translateId(it[1], g.packed, module, g.config) - let symId = FullId(module: tmp.module, packed: it[1]) - g.enumToStringProcs[key] = LazySym(id: symId, sym: nil) + replayBackendProcs(g, module) for it in mitems(g.packed[module].fromDisk.methods): let sym = loadSymFromId(g.config, g.cache, g.packed, module, diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index a518870f8..ac995dd2e 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -7,7 +7,68 @@ # distribution, for details about the copyright. # -from typetraits import supportsCopyMem +## Low level binary format used by the compiler to store and load various AST +## and related data. +## +## NB: this is incredibly low level and if you're interested in how the +## compiler works and less a storage format, you're probably looking for +## the `ic` or `packed_ast` modules to understand the logical format. + +from std/typetraits import supportsCopyMem + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + +import std / tables + +## Overview +## ======== +## `RodFile` represents a Rod File (versioned binary format), and the +## associated data for common interactions such as IO and error tracking +## (`RodFileError`). The file format broken up into sections (`RodSection`) +## and preceded by a header (see: `cookie`). The precise layout, section +## ordering and data following the section are determined by the user. See +## `ic.loadRodFile`. +## +## A basic but "wrong" example of the lifecycle: +## --------------------------------------------- +## 1. `create` or `open` - create a new one or open an existing +## 2. `storeHeader` - header info +## 3. `storePrim` or `storeSeq` - save your stuff +## 4. `close` - and we're done +## +## Now read the bits below to understand what's missing. +## +## ### Issues with the Example +## Missing Sections: +## This is a low level API, so headers and sections need to be stored and +## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` & +## `loadSection`, respectively. +## +## No Error Handling: +## The API is centered around IO and prone to error, each operation checks or +## sets the `RodFile.err` field. A user of this API needs to handle these +## appropriately. +## +## API Notes +## ========= +## +## Valid inputs for Rod files +## -------------------------- +## ASTs, hopes, dreams, and anything as long as it and any children it may have +## support `copyMem`. This means anything that is not a pointer and that does not contain a pointer. At a glance these are: +## * string +## * objects & tuples (fields are recursed) +## * sequences AKA `seq[T]` +## +## Note on error handling style +## ---------------------------- +## A flag based approach is used where operations no-op in case of a +## preexisting error and set the flag if they encounter one. +## +## Misc +## ---- +## * 'Prim' is short for 'primitive', as in a non-sequence type type RodSection* = enum @@ -16,16 +77,15 @@ type stringsSection checkSumsSection depsSection - integersSection - floatsSection + numbersSection exportsSection + hiddenSection reexportsSection compilerProcsSection trmacrosSection convertersSection methodsSection pureEnumsSection - macroUsagesSection toReplaySection topLevelSection bodiesSection @@ -34,10 +94,16 @@ type typeInstCacheSection procInstCacheSection attachedOpsSection - methodsPerTypeSection + methodsPerGenericTypeSection enumToStringProcsSection + methodsPerTypeSection + dispatchersSection typeInfoSection # required by the backend + backendFlagsSection aliveSymsSection # beware, this is stored in a `.alivesyms` file. + sideChannelSection + namespaceSection + symnamesSection RodFileError* = enum ok, tooBig, cannotOpen, ioFailure, wrongHeader, wrongSection, configMismatch, @@ -50,8 +116,8 @@ type # better than exceptions. const - RodVersion = 1 - cookie = [byte(0), byte('R'), byte('O'), byte('D'), + RodVersion = 2 + defaultCookie = [byte(0), byte('R'), byte('O'), byte('D'), byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)] proc setError(f: var RodFile; err: RodFileError) {.inline.} = @@ -59,6 +125,8 @@ proc setError(f: var RodFile; err: RodFileError) {.inline.} = #raise newException(IOError, "IO error") proc storePrim*(f: var RodFile; s: string) = + ## Stores a string. + ## The len is prefixed to allow for later retreival. if f.err != ok: return if s.len >= high(int32): setError f, tooBig @@ -72,6 +140,9 @@ proc storePrim*(f: var RodFile; s: string) = setError f, ioFailure proc storePrim*[T](f: var RodFile; x: T) = + ## Stores a non-sequence/string `T`. + ## If `T` doesn't support `copyMem` and is an object or tuple then the fields + ## are written -- the user from context will need to know which `T` to load. if f.err != ok: return when supportsCopyMem(T): if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): @@ -89,6 +160,7 @@ proc storePrim*[T](f: var RodFile; x: T) = {.error: "unsupported type for 'storePrim'".} proc storeSeq*[T](f: var RodFile; s: seq[T]) = + ## Stores a sequence of `T`s, with the len as a prefix for later retrieval. if f.err != ok: return if s.len >= high(int32): setError f, tooBig @@ -100,7 +172,20 @@ proc storeSeq*[T](f: var RodFile; s: seq[T]) = for i in 0..<s.len: storePrim(f, s[i]) +proc storeOrderedTable*[K, T](f: var RodFile; s: OrderedTable[K, T]) = + if f.err != ok: return + if s.len >= high(int32): + setError f, tooBig + return + var lenPrefix = int32(s.len) + if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): + setError f, ioFailure + else: + for _, v in s: + storePrim(f, v) + proc loadPrim*(f: var RodFile; s: var string) = + ## Read a string, the length was stored as a prefix if f.err != ok: return var lenPrefix = int32(0) if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): @@ -112,6 +197,7 @@ proc loadPrim*(f: var RodFile; s: var string) = setError f, ioFailure proc loadPrim*[T](f: var RodFile; x: var T) = + ## Load a non-sequence/string `T`. if f.err != ok: return when supportsCopyMem(T): if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): @@ -129,6 +215,7 @@ proc loadPrim*[T](f: var RodFile; x: var T) = {.error: "unsupported type for 'loadPrim'".} proc loadSeq*[T](f: var RodFile; s: var seq[T]) = + ## `T` must be compatible with `copyMem`, see `loadPrim` if f.err != ok: return var lenPrefix = int32(0) if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): @@ -138,38 +225,59 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) = for i in 0..<lenPrefix: loadPrim(f, s[i]) -proc storeHeader*(f: var RodFile) = +proc loadOrderedTable*[K, T](f: var RodFile; s: var OrderedTable[K, T]) = + ## `T` must be compatible with `copyMem`, see `loadPrim` + if f.err != ok: return + var lenPrefix = int32(0) + if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): + setError f, ioFailure + else: + s = initOrderedTable[K, T](lenPrefix) + for i in 0..<lenPrefix: + var x = default T + loadPrim(f, x) + s[x.id] = x + +proc storeHeader*(f: var RodFile; cookie = defaultCookie) = + ## stores the header which is described by `cookie`. if f.err != ok: return if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len: setError f, ioFailure -proc loadHeader*(f: var RodFile) = +proc loadHeader*(f: var RodFile; cookie = defaultCookie) = + ## Loads the header which is described by `cookie`. if f.err != ok: return - var thisCookie: array[cookie.len, byte] + var thisCookie: array[cookie.len, byte] = default(array[cookie.len, byte]) if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len: setError f, ioFailure elif thisCookie != cookie: setError f, wrongHeader proc storeSection*(f: var RodFile; s: RodSection) = + ## update `currentSection` and writes the bytes value of s. if f.err != ok: return assert f.currentSection < s f.currentSection = s storePrim(f, s) proc loadSection*(f: var RodFile; expected: RodSection) = + ## read the bytes value of s, sets and error if the section is incorrect. if f.err != ok: return - var s: RodSection + var s: RodSection = default(RodSection) loadPrim(f, s) if expected != s and f.err == ok: setError f, wrongSection proc create*(filename: string): RodFile = + ## create the file and open it for writing + result = default(RodFile) if not open(result.f, filename, fmWrite): setError result, cannotOpen proc close*(f: var RodFile) = close(f.f) proc open*(filename: string): RodFile = + ## open the file for reading + result = default(RodFile) if not open(result.f, filename, fmRead): setError result, cannotOpen |