diff options
Diffstat (limited to 'compiler/ic/ic.nim')
| -rw-r--r-- | compiler/ic/ic.nim | 1343 |
1 files changed, 1343 insertions, 0 deletions
diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim new file mode 100644 index 000000000..8e81633ef --- /dev/null +++ b/compiler/ic/ic.nim @@ -0,0 +1,1343 @@ +# +# +# The Nim Compiler +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import std/[hashes, tables, intsets, monotimes] +import packed_ast, bitabs, rodfiles +import ".." / [ast, idents, lineinfos, msgs, ropes, options, + pathutils, condsyms, packages, modulepaths] +#import ".." / [renderer, astalgo] +from std/os import removeFile, isAbsolute + +import ../../dist/checksums/src/checksums/sha1 + +import iclineinfos + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, formatfloat] + +type + PackedConfig* = object + backend: TBackend + selectedGC: TGCMode + cCompiler: TSystemCC + options: TOptions + globalOptions: TGlobalOptions + + ModuleBackendFlag* = enum + HasDatInitProc + HasModuleInitProc + + PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file + definedSymbols: string + moduleFlags: TSymFlags + includes*: seq[(LitId, string)] # first entry is the module filename itself + imports: seq[LitId] # the modules this module depends on + toReplay*: PackedTree # pragmas and VM specific state to replay. + topLevel*: PackedTree # top level statements + bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position. + #producedGenerics*: Table[GenericKey, SymId] + exports*: seq[(LitId, int32)] + hidden: seq[(LitId, int32)] + reexports: seq[(LitId, PackedItemId)] + compilerProcs*: seq[(LitId, int32)] + converters*, methods*, trmacros*, pureEnums*: seq[int32] + + typeInstCache*: seq[(PackedItemId, PackedItemId)] + procInstCache*: seq[PackedInstantiation] + attachedOps*: seq[(PackedItemId, TTypeAttachedOp, PackedItemId)] + methodsPerGenericType*: seq[(PackedItemId, int, PackedItemId)] + enumToStringProcs*: seq[(PackedItemId, PackedItemId)] + methodsPerType*: seq[(PackedItemId, PackedItemId)] + dispatchers*: seq[PackedItemId] + + emittedTypeInfo*: seq[string] + backendFlags*: set[ModuleBackendFlag] + + syms*: OrderedTable[int32, PackedSym] + types*: OrderedTable[int32, PackedType] + strings*: BiTable[string] # we could share these between modules. + numbers*: BiTable[BiggestInt] # we also store floats in here so + # that we can assure that every bit is kept + man*: LineInfoManager + + cfg: PackedConfig + + PackedEncoder* = object + #m*: PackedModule + thisModule*: int32 + lastFile*: FileIndex # remember the last lookup entry. + lastLit*: LitId + filenames*: Table[FileIndex, LitId] + pendingTypes*: seq[PType] + pendingSyms*: seq[PSym] + typeMarker*: IntSet #Table[ItemId, TypeId] # ItemId.item -> TypeId + symMarker*: IntSet #Table[ItemId, SymId] # ItemId.item -> SymId + config*: ConfigRef + +proc toString*(tree: PackedTree; pos: NodePos; m: PackedModule; nesting: int; + result: var string) = + if result.len > 0 and result[^1] notin {' ', '\n'}: + result.add ' ' + + result.add $tree[pos].kind + case tree[pos].kind + of nkEmpty, nkNilLit, nkType: discard + of nkIdent, nkStrLit..nkTripleStrLit: + result.add " " + result.add m.strings[LitId tree[pos].uoperand] + of nkSym: + result.add " " + result.add m.strings[m.syms[tree[pos].soperand].name] + of directIntLit: + result.add " " + result.addInt tree[pos].soperand + of externSIntLit: + result.add " " + result.addInt m.numbers[LitId tree[pos].uoperand] + of externUIntLit: + result.add " " + result.addInt cast[uint64](m.numbers[LitId tree[pos].uoperand]) + of nkFloatLit..nkFloat128Lit: + result.add " " + result.addFloat cast[BiggestFloat](m.numbers[LitId tree[pos].uoperand]) + else: + result.add "(\n" + for i in 1..(nesting+1)*2: result.add ' ' + for child in sonsReadonly(tree, pos): + toString(tree, child, m, nesting + 1, result) + result.add "\n" + for i in 1..nesting*2: result.add ' ' + result.add ")" + #for i in 1..nesting*2: result.add ' ' + +proc toString*(tree: PackedTree; n: NodePos; m: PackedModule): string = + result = "" + toString(tree, n, m, 0, result) + +proc debug*(tree: PackedTree; m: PackedModule) = + stdout.write toString(tree, NodePos 0, m) + +proc isActive*(e: PackedEncoder): bool = e.config != nil +proc disable(e: var PackedEncoder) = e.config = nil + +template primConfigFields(fn: untyped) {.dirty.} = + fn backend + fn selectedGC + fn cCompiler + fn options + fn globalOptions + +proc definedSymbolsAsString(config: ConfigRef): string = + result = newStringOfCap(200) + result.add "config" + for d in definedSymbolNames(config.symbols): + result.add ' ' + result.add d + +proc rememberConfig(c: var PackedEncoder; m: var PackedModule; config: ConfigRef; pc: PackedConfig) = + m.definedSymbols = definedSymbolsAsString(config) + #template rem(x) = + # c.m.cfg.x = config.x + #primConfigFields rem + m.cfg = pc + +const + debugConfigDiff = defined(debugConfigDiff) + +when debugConfigDiff: + import hashes, tables, intsets, sha1, strutils, sets + +proc configIdentical(m: PackedModule; config: ConfigRef): bool = + result = m.definedSymbols == definedSymbolsAsString(config) + when debugConfigDiff: + if not result: + var wordsA = m.definedSymbols.split(Whitespace).toHashSet() + var wordsB = definedSymbolsAsString(config).split(Whitespace).toHashSet() + for c in wordsA - wordsB: + echo "in A but not in B ", c + for c in wordsB - wordsA: + echo "in B but not in A ", c + template eq(x) = + result = result and m.cfg.x == config.x + when debugConfigDiff: + if m.cfg.x != config.x: + echo "B ", m.cfg.x, " ", config.x + primConfigFields eq + +proc rememberStartupConfig*(dest: var PackedConfig, config: ConfigRef) = + template rem(x) = + dest.x = config.x + primConfigFields rem + dest.globalOptions.excl optForceFullMake + +proc hashFileCached(conf: ConfigRef; fileIdx: FileIndex): string = + result = msgs.getHash(conf, fileIdx) + if result.len == 0: + let fullpath = msgs.toFullPath(conf, fileIdx) + result = $secureHashFile(fullpath) + msgs.setHash(conf, fileIdx, result) + +proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId = + ## store a file index as a literal + if x == c.lastFile: + result = c.lastLit + else: + result = c.filenames.getOrDefault(x) + if result == LitId(0): + let p = msgs.toFullPath(c.config, x) + result = getOrIncl(m.strings, p) + c.filenames[x] = result + c.lastFile = x + c.lastLit = result + assert result != LitId(0) + +proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex = + result = msgs.fileInfoIdx(config, AbsoluteFile m.strings[x]) + +proc includesIdentical(m: var PackedModule; config: ConfigRef): bool = + for it in mitems(m.includes): + if hashFileCached(config, toFileIndex(it[0], m, config)) != it[1]: + return false + result = true + +proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; config: ConfigRef; pc: PackedConfig) = + ## setup a context for serializing to packed ast + c.thisModule = moduleSym.itemId.module + c.config = config + m.moduleFlags = moduleSym.flags + m.bodies = newTreeFrom(m.topLevel) + m.toReplay = newTreeFrom(m.topLevel) + + c.lastFile = FileIndex(-10) + + let thisNimFile = FileIndex c.thisModule + var h = msgs.getHash(config, thisNimFile) + if h.len == 0: + let fullpath = msgs.toFullPath(config, thisNimFile) + if isAbsolute(fullpath): + # For NimScript compiler API support the main Nim file might be from a stream. + h = $secureHashFile(fullpath) + msgs.setHash(config, thisNimFile, h) + m.includes.add((toLitId(thisNimFile, c, m), h)) # the module itself + + rememberConfig(c, m, config, pc) + +proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) = + m.includes.add((toLitId(f, c, m), hashFileCached(c.config, f))) + +proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) = + m.imports.add toLitId(f, c, m) + +proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + assert s.kind != skUnknown + let nameId = getOrIncl(m.strings, s.name.s) + m.hidden.add((nameId, s.itemId.item)) + assert s.itemId.module == c.thisModule + +proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + assert s.kind != skUnknown + assert s.itemId.module == c.thisModule + let nameId = getOrIncl(m.strings, s.name.s) + m.exports.add((nameId, s.itemId.item)) + +proc addConverter*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + assert c.thisModule == s.itemId.module + m.converters.add(s.itemId.item) + +proc addTrmacro*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + m.trmacros.add(s.itemId.item) + +proc addPureEnum*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + assert s.kind == skType + m.pureEnums.add(s.itemId.item) + +proc addMethod*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + m.methods.add s.itemId.item + +proc addReexport*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + assert s.kind != skUnknown + if s.kind == skModule: return + let nameId = getOrIncl(m.strings, s.name.s) + m.reexports.add((nameId, PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), + item: s.itemId.item))) + +proc addCompilerProc*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + let nameId = getOrIncl(m.strings, s.name.s) + m.compilerProcs.add((nameId, s.itemId.item)) + +proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) +proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId +proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId + +proc flush(c: var PackedEncoder; m: var PackedModule) = + ## serialize any pending types or symbols from the context + while true: + if c.pendingTypes.len > 0: + discard storeType(c.pendingTypes.pop, c, m) + elif c.pendingSyms.len > 0: + discard storeSym(c.pendingSyms.pop, c, m) + else: + break + +proc toLitId(x: string; m: var PackedModule): LitId = + ## store a string as a literal + result = getOrIncl(m.strings, x) + +proc toLitId(x: BiggestInt; m: var PackedModule): LitId = + ## store an integer as a literal + result = getOrIncl(m.numbers, x) + +proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo = + pack(m.man, toLitId(x.fileIndex, c, m), x.line.int32, x.col.int32) + #PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m)) + +proc safeItemId(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId {.inline.} = + ## given a symbol, produce an ItemId with the correct properties + ## for local or remote symbols, packing the symbol as necessary + if s == nil or s.kind == skPackage: + result = nilItemId + #elif s.itemId.module == c.thisModule: + # result = PackedItemId(module: LitId(0), item: s.itemId.item) + else: + assert int(s.itemId.module) >= 0 + result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), + item: s.itemId.item) + +proc addMissing(c: var PackedEncoder; p: PSym) = + ## consider queuing a symbol for later addition to the packed tree + if p != nil and p.itemId.module == c.thisModule: + if p.itemId.item notin c.symMarker: + if not (sfForward in p.flags and p.kind in routineKinds): + c.pendingSyms.add p + +proc addMissing(c: var PackedEncoder; p: PType) = + ## consider queuing a type for later addition to the packed tree + if p != nil and p.uniqueId.module == c.thisModule: + if p.uniqueId.item notin c.typeMarker: + c.pendingTypes.add p + +template storeNode(dest, src, field) = + var nodeId: NodeId + if src.field != nil: + nodeId = getNodeId(m.bodies) + toPackedNode(src.field, m.bodies, c, m) + else: + nodeId = emptyNodeId + dest.field = nodeId + +proc storeTypeLater(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId = + # We store multiple different trees in m.bodies. For this to work out, we + # cannot immediately store types/syms. We enqueue them instead to ensure + # we only write one tree into m.bodies after the other. + if t.isNil: return nilItemId + + assert t.uniqueId.module >= 0 + assert t.uniqueId.item > 0 + result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) + if t.uniqueId.module == c.thisModule: + # the type belongs to this module, so serialize it here, eventually. + addMissing(c, t) + +proc storeSymLater(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId = + if s.isNil: return nilItemId + assert s.itemId.module >= 0 + assert s.itemId.item >= 0 + result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) + if s.itemId.module == c.thisModule: + # the sym belongs to this module, so serialize it here, eventually. + addMissing(c, s) + +proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId = + ## serialize a ptype + if t.isNil: return nilItemId + + assert t.uniqueId.module >= 0 + assert t.uniqueId.item > 0 + result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) + + if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item): + #if t.uniqueId.item >= m.types.len: + # setLen m.types, t.uniqueId.item+1 + + var p = PackedType(id: t.uniqueId.item, kind: t.kind, flags: t.flags, callConv: t.callConv, + size: t.size, align: t.align, nonUniqueId: t.itemId.item, + paddingAtEnd: t.paddingAtEnd) + storeNode(p, t, n) + p.typeInst = t.typeInst.storeType(c, m) + for kid in kids t: + p.types.add kid.storeType(c, m) + c.addMissing t.sym + p.sym = t.sym.safeItemId(c, m) + c.addMissing t.owner + p.owner = t.owner.safeItemId(c, m) + + # fill the reserved slot, nothing else: + m.types[t.uniqueId.item] = p + +proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib = + ## the plib hangs off the psym via the .annex field + if l.isNil: return + result = PackedLib(kind: l.kind, generated: l.generated, + isOverridden: l.isOverridden, name: toLitId($l.name, m) + ) + storeNode(result, l, path) + +proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId = + ## serialize a psym + if s.isNil: return nilItemId + + assert s.itemId.module >= 0 + result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) + + if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item): + #if s.itemId.item >= m.syms.len: + # setLen m.syms, s.itemId.item+1 + + assert sfForward notin s.flags + + var p = PackedSym(id: s.itemId.item, kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic, + position: s.position, offset: s.offset, disamb: s.disamb, options: s.options, + name: s.name.s.toLitId(m)) + + storeNode(p, s, ast) + storeNode(p, s, constraint) + + if s.kind in {skLet, skVar, skField, skForVar}: + c.addMissing s.guard + p.guard = s.guard.safeItemId(c, m) + p.bitsize = s.bitsize + p.alignment = s.alignment + + p.externalName = toLitId(s.loc.snippet, m) + p.locFlags = s.loc.flags + c.addMissing s.typ + p.typ = s.typ.storeType(c, m) + c.addMissing s.owner + p.owner = s.owner.safeItemId(c, m) + p.annex = toPackedLib(s.annex, c, m) + when hasFFI: + p.cname = toLitId(s.cname, m) + p.instantiatedFrom = s.instantiatedFrom.safeItemId(c, m) + + # fill the reserved slot, nothing else: + m.syms[s.itemId.item] = p + +proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = + ## add a remote symbol reference to the tree + let info = n.info.toPackedInfo(c, m) + if n.typ != n.sym.typ: + ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total + info = info, flags = n.flags, + typeId = storeTypeLater(n.typ, c, m)) + else: + ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total + info = info, flags = n.flags) + ir.addNode(kind = nkNone, info = info, + operand = toLitId(n.sym.itemId.module.FileIndex, c, m).int32) + ir.addNode(kind = nkNone, info = info, + operand = n.sym.itemId.item) + +proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = + ## serialize a node into the tree + if n == nil: + ir.addNode(kind = nkNilRodNode, operand = 1, info = NoLineInfo) + return + let info = toPackedInfo(n.info, c, m) + case n.kind + of nkNone, nkEmpty, nkNilLit, nkType: + ir.addNode(kind = n.kind, flags = n.flags, operand = 0, + typeId = storeTypeLater(n.typ, c, m), info = info) + of nkIdent: + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.strings, n.ident.s), + typeId = storeTypeLater(n.typ, c, m), info = info) + of nkSym: + if n.sym.itemId.module == c.thisModule: + # it is a symbol that belongs to the module we're currently + # packing: + let id = n.sym.storeSymLater(c, m).item + if n.typ != n.sym.typ: + ir.addNode(kind = nkSym, flags = n.flags, operand = id, + info = info, + typeId = storeTypeLater(n.typ, c, m)) + else: + ir.addNode(kind = nkSym, flags = n.flags, operand = id, + info = info) + else: + # store it as an external module reference: + addModuleRef(n, ir, c, m) + of externIntLit: + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.numbers, n.intVal), + typeId = storeTypeLater(n.typ, c, m), info = info) + of nkStrLit..nkTripleStrLit: + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.strings, n.strVal), + typeId = storeTypeLater(n.typ, c, m), info = info) + of nkFloatLit..nkFloat128Lit: + ir.addNode(kind = n.kind, flags = n.flags, + operand = int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)), + typeId = storeTypeLater(n.typ, c, m), info = info) + else: + let patchPos = ir.prepare(n.kind, n.flags, + storeTypeLater(n.typ, c, m), info) + for i in 0..<n.len: + toPackedNode(n[i], ir, c, m) + ir.patch patchPos + +proc storeTypeInst*(c: var PackedEncoder; m: var PackedModule; s: PSym; inst: PType) = + m.typeInstCache.add (storeSymLater(s, c, m), storeTypeLater(inst, c, m)) + +proc addPragmaComputation*(c: var PackedEncoder; m: var PackedModule; n: PNode) = + toPackedNode(n, m.toReplay, c, m) + +proc toPackedProcDef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = + let info = toPackedInfo(n.info, c, m) + let patchPos = ir.prepare(n.kind, n.flags, + storeTypeLater(n.typ, c, m), info) + for i in 0..<n.len: + if i != bodyPos: + toPackedNode(n[i], ir, c, m) + else: + # do not serialize the body of the proc, it's unnecessary since + # n[0].sym.ast has the sem'checked variant of it which is what + # everybody should use instead. + ir.addNode(kind = nkEmpty, flags = {}, operand = 0, + typeId = nilItemId, info = info) + ir.patch patchPos + +proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var PackedModule) = + case n.kind + of routineDefs: + toPackedProcDef(n, m.topLevel, encoder, m) + when false: + # we serialize n[namePos].sym instead + if n[namePos].kind == nkSym: + let s = n[namePos].sym + discard storeSym(s, encoder, m) + if s.flags * {sfExportc, sfCompilerProc, sfCompileTime} == {sfExportc}: + m.exportCProcs.add(s.itemId.item) + else: + toPackedNode(n, m.topLevel, encoder, m) + of nkStmtList, nkStmtListExpr: + for it in n: + toPackedNodeIgnoreProcDefs(it, encoder, m) + of nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt, + nkFromStmt, nkIncludeStmt: + discard "nothing to do" + else: + toPackedNode(n, m.topLevel, encoder, m) + +proc toPackedNodeTopLevel*(n: PNode, encoder: var PackedEncoder; m: var PackedModule) = + toPackedNodeIgnoreProcDefs(n, encoder, m) + flush encoder, m + +proc toPackedGeneratedProcDef*(s: PSym, encoder: var PackedEncoder; m: var PackedModule) = + ## Generic procs and generated `=hook`'s need explicit top-level entries so + ## that the code generator can work without having to special case these. These + ## entries will also be useful for other tools and are the cleanest design + ## I can come up with. + assert s.kind in routineKinds + toPackedProcDef(s.ast, m.topLevel, encoder, m) + #flush encoder, m + +proc storeAttachedProcDef*(t: PType; op: TTypeAttachedOp; s: PSym, + encoder: var PackedEncoder; m: var PackedModule) = + assert s.kind in routineKinds + assert isActive(encoder) + let tid = storeTypeLater(t, encoder, m) + let sid = storeSymLater(s, encoder, m) + m.attachedOps.add (tid, op, sid) + toPackedGeneratedProcDef(s, encoder, m) + +proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) = + var t = newSeq[PackedItemId](i.concreteTypes.len) + for j in 0..high(i.concreteTypes): + t[j] = storeTypeLater(i.concreteTypes[j], c, m) + m.procInstCache.add PackedInstantiation(key: storeSymLater(s, c, m), + sym: storeSymLater(i.sym, c, m), + concreteTypes: t) + toPackedGeneratedProcDef(i.sym, c, m) + +proc storeExpansion*(c: var PackedEncoder; m: var PackedModule; info: TLineInfo; s: PSym) = + toPackedNode(newSymNode(s, info), m.bodies, c, m) + +proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) = + case err + of cannotOpen: + rawMessage(config, warnCannotOpenFile, filename.string) + of includeFileChanged: + rawMessage(config, warnFileChanged, filename.string) + else: + rawMessage(config, warnCannotOpenFile, filename.string & " reason: " & $err) + #echo "Error: ", $err, " loading file: ", filename.string + +proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile = + result = changeFileExt(completeGeneratedFilePath(conf, + mangleModuleName(conf, f).AbsoluteFile), ext) + +const + BenchIC* = false + +when BenchIC: + var gloadBodies: MonoTime + + template bench(x, body) = + let start = getMonoTime() + body + x = x + (getMonoTime() - start) + +else: + template bench(x, body) = body + +proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef; + ignoreConfig = false): RodFileError = + var f = rodfiles.open(filename.string) + f.loadHeader() + f.loadSection configSection + + f.loadPrim m.definedSymbols + f.loadPrim m.moduleFlags + f.loadPrim m.cfg + + if f.err == ok and not configIdentical(m, config) and not ignoreConfig: + f.err = configMismatch + + template loadSeqSection(section, data) {.dirty.} = + f.loadSection section + f.loadSeq data + + template loadTableSection(section, data) {.dirty.} = + f.loadSection section + f.loadOrderedTable data + + template loadTabSection(section, data) {.dirty.} = + f.loadSection section + f.load data + + loadTabSection stringsSection, m.strings + + loadSeqSection checkSumsSection, m.includes + if config.cmd != cmdM and not includesIdentical(m, config): + f.err = includeFileChanged + + loadSeqSection depsSection, m.imports + + bench gloadBodies: + + loadTabSection numbersSection, m.numbers + + loadSeqSection exportsSection, m.exports + loadSeqSection hiddenSection, m.hidden + loadSeqSection reexportsSection, m.reexports + + loadSeqSection compilerProcsSection, m.compilerProcs + + loadSeqSection trmacrosSection, m.trmacros + + loadSeqSection convertersSection, m.converters + loadSeqSection methodsSection, m.methods + loadSeqSection pureEnumsSection, m.pureEnums + + loadTabSection toReplaySection, m.toReplay + loadTabSection topLevelSection, m.topLevel + + loadTabSection bodiesSection, m.bodies + loadTableSection symsSection, m.syms + loadTableSection typesSection, m.types + + loadSeqSection typeInstCacheSection, m.typeInstCache + loadSeqSection procInstCacheSection, m.procInstCache + loadSeqSection attachedOpsSection, m.attachedOps + loadSeqSection methodsPerGenericTypeSection, m.methodsPerGenericType + loadSeqSection enumToStringProcsSection, m.enumToStringProcs + loadSeqSection methodsPerTypeSection, m.methodsPerType + loadSeqSection dispatchersSection, m.dispatchers + loadSeqSection typeInfoSection, m.emittedTypeInfo + + f.loadSection backendFlagsSection + f.loadPrim m.backendFlags + + f.loadSection sideChannelSection + f.load m.man + + close(f) + result = f.err + +# ------------------------------------------------------------------------- + +proc storeError(err: RodFileError; filename: AbsoluteFile) = + echo "Error: ", $err, "; couldn't write to ", filename.string + removeFile(filename.string) + +proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var PackedModule) = + flush encoder, m + #rememberConfig(encoder, encoder.config) + + var f = rodfiles.create(filename.string) + f.storeHeader() + f.storeSection configSection + f.storePrim m.definedSymbols + f.storePrim m.moduleFlags + f.storePrim m.cfg + + template storeSeqSection(section, data) {.dirty.} = + f.storeSection section + f.storeSeq data + + template storeTabSection(section, data) {.dirty.} = + f.storeSection section + f.store data + + template storeTableSection(section, data) {.dirty.} = + f.storeSection section + f.storeOrderedTable data + + storeTabSection stringsSection, m.strings + + storeSeqSection checkSumsSection, m.includes + + storeSeqSection depsSection, m.imports + + storeTabSection numbersSection, m.numbers + + storeSeqSection exportsSection, m.exports + storeSeqSection hiddenSection, m.hidden + storeSeqSection reexportsSection, m.reexports + + storeSeqSection compilerProcsSection, m.compilerProcs + + storeSeqSection trmacrosSection, m.trmacros + storeSeqSection convertersSection, m.converters + storeSeqSection methodsSection, m.methods + storeSeqSection pureEnumsSection, m.pureEnums + + storeTabSection toReplaySection, m.toReplay + storeTabSection topLevelSection, m.topLevel + + storeTabSection bodiesSection, m.bodies + storeTableSection symsSection, m.syms + + storeTableSection typesSection, m.types + + storeSeqSection typeInstCacheSection, m.typeInstCache + storeSeqSection procInstCacheSection, m.procInstCache + storeSeqSection attachedOpsSection, m.attachedOps + storeSeqSection methodsPerGenericTypeSection, m.methodsPerGenericType + storeSeqSection enumToStringProcsSection, m.enumToStringProcs + storeSeqSection methodsPerTypeSection, m.methodsPerType + storeSeqSection dispatchersSection, m.dispatchers + storeSeqSection typeInfoSection, m.emittedTypeInfo + + f.storeSection backendFlagsSection + f.storePrim m.backendFlags + + f.storeSection sideChannelSection + f.store m.man + + close(f) + encoder.disable() + if f.err != ok: + storeError(f.err, filename) + + when false: + # basic loader testing: + var m2: PackedModule + discard loadRodFile(filename, m2, encoder.config) + echo "loaded ", filename.string + +# ---------------------------------------------------------------------------- + +type + PackedDecoder* = object + lastModule: int + lastLit: LitId + lastFile: FileIndex # remember the last lookup entry. + config*: ConfigRef + cache*: IdentCache + +type + ModuleStatus* = enum + undefined, + storing, # state is strictly for stress-testing purposes + loading, + loaded, + outdated, + stored # store is complete, no further additions possible + + LoadedModule* = object + status*: ModuleStatus + symsInit, typesInit, loadedButAliveSetChanged*: bool + fromDisk*: PackedModule + syms: OrderedTable[int32, PSym] # indexed by itemId + types: OrderedTable[int32, PType] + module*: PSym # the one true module symbol. + iface, ifaceHidden: Table[PIdent, seq[PackedItemId]] + # PackedItemId so that it works with reexported symbols too + # ifaceHidden includes private symbols + +type + PackedModuleGraph* = object + pm*: seq[LoadedModule] # indexed by FileIndex + when BenchIC: + depAnalysis: MonoTime + loadBody: MonoTime + loadSym, loadType, loadBodies: MonoTime + +when BenchIC: + proc echoTimes*(m: PackedModuleGraph) = + echo "analysis: ", m.depAnalysis, " loadBody: ", m.loadBody, " loadSym: ", + m.loadSym, " loadType: ", m.loadType, " all bodies: ", gloadBodies + +template `[]`*(m: PackedModuleGraph; i: int): LoadedModule = m.pm[i] +template len*(m: PackedModuleGraph): int = m.pm.len + +proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t: PackedItemId): PType +proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s: PackedItemId): PSym + +proc toFileIndexCached*(c: var PackedDecoder; g: PackedModuleGraph; thisModule: int; f: LitId): FileIndex = + if f == LitId(0): + result = InvalidFileIdx + elif c.lastLit == f and c.lastModule == thisModule: + result = c.lastFile + else: + result = toFileIndex(f, g[thisModule].fromDisk, c.config) + c.lastModule = thisModule + c.lastLit = f + c.lastFile = result + +proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; + x: PackedLineInfo): TLineInfo = + assert g[thisModule].status in {loaded, storing, stored} + let (fileId, line, col) = unpack(g[thisModule].fromDisk.man, x) + result = TLineInfo(line: line.uint16, col: col.int16, + fileIndex: toFileIndexCached(c, g, thisModule, fileId)) + +proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; + tree: PackedTree; n: NodePos): PNode = + let k = n.kind + if k == nkNilRodNode: + return nil + when false: + echo "loading node ", c.config $ translateLineInfo(c, g, thisModule, n.info) + result = newNodeIT(k, translateLineInfo(c, g, thisModule, n.info), + loadType(c, g, thisModule, n.typ)) + result.flags = n.flags + + case k + of nkNone, nkEmpty, nkNilLit, nkType: + discard + of nkIdent: + result.ident = getIdent(c.cache, g[thisModule].fromDisk.strings[n.litId]) + of nkSym: + result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree[n].soperand)) + if result.typ == nil: + result.typ = result.sym.typ + of externIntLit: + result.intVal = g[thisModule].fromDisk.numbers[n.litId] + of nkStrLit..nkTripleStrLit: + result.strVal = g[thisModule].fromDisk.strings[n.litId] + of nkFloatLit..nkFloat128Lit: + result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.numbers[n.litId]) + of nkModuleRef: + let (n1, n2) = sons2(tree, n) + assert n1.kind == nkNone + assert n2.kind == nkNone + transitionNoneToSym(result) + result.sym = loadSym(c, g, thisModule, PackedItemId(module: n1.litId, item: tree[n2].soperand)) + if result.typ == nil: + result.typ = result.sym.typ + else: + for n0 in sonsReadonly(tree, n): + result.addAllowNil loadNodes(c, g, thisModule, tree, n0) + +proc initPackedDecoder*(config: ConfigRef; cache: IdentCache): PackedDecoder = + result = PackedDecoder( + lastModule: int32(-1), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + +proc loadProcHeader(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; + tree: PackedTree; n: NodePos): PNode = + # do not load the body of the proc. This will be done later in + # getProcBody, if required. + let k = n.kind + result = newNodeIT(k, translateLineInfo(c, g, thisModule, n.info), + loadType(c, g, thisModule, n.typ)) + result.flags = n.flags + assert k in {nkProcDef, nkMethodDef, nkIteratorDef, nkFuncDef, nkConverterDef, nkLambda} + var i = 0 + for n0 in sonsReadonly(tree, n): + if i != bodyPos: + result.add loadNodes(c, g, thisModule, tree, n0) + else: + result.addAllowNil nil + inc i + +proc loadProcBody(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; + tree: PackedTree; n: NodePos): PNode = + result = nil + var i = 0 + for n0 in sonsReadonly(tree, n): + if i == bodyPos: + result = loadNodes(c, g, thisModule, tree, n0) + inc i + +proc moduleIndex*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; + s: PackedItemId): int32 {.inline.} = + result = if s.module == LitId(0): thisModule.int32 + else: toFileIndexCached(c, g, thisModule, s.module).int32 + +proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; + s: PackedSym; si, item: int32): PSym = + result = PSym(itemId: ItemId(module: si, item: item), + kind: s.kind, magic: s.magic, flags: s.flags, + info: translateLineInfo(c, g, si, s.info), + options: s.options, + position: if s.kind in {skForVar, skVar, skLet, skTemp}: 0 else: s.position, + offset: if s.kind in routineKinds: defaultOffset else: s.offset, + disamb: s.disamb, + name: getIdent(c.cache, g[si].fromDisk.strings[s.name]) + ) + +template loadAstBody(p, field) = + if p.field != emptyNodeId: + result.field = loadNodes(c, g, si, g[si].fromDisk.bodies, NodePos p.field) + +template loadAstBodyLazy(p, field) = + if p.field != emptyNodeId: + result.field = loadProcHeader(c, g, si, g[si].fromDisk.bodies, NodePos p.field) + +proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph; + si, item: int32; l: PackedLib): PLib = + # XXX: hack; assume a zero LitId means the PackedLib is all zero (empty) + if l.name.int == 0: + result = nil + else: + result = PLib(generated: l.generated, isOverridden: l.isOverridden, + kind: l.kind, name: rope g[si].fromDisk.strings[l.name]) + loadAstBody(l, path) + +proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; + s: PackedSym; si, item: int32; result: PSym) = + result.typ = loadType(c, g, si, s.typ) + loadAstBody(s, constraint) + if result.kind in {skProc, skFunc, skIterator, skConverter, skMethod}: + loadAstBodyLazy(s, ast) + else: + loadAstBody(s, ast) + result.annex = loadLib(c, g, si, item, s.annex) + when hasFFI: + result.cname = g[si].fromDisk.strings[s.cname] + + if s.kind in {skLet, skVar, skField, skForVar}: + result.guard = loadSym(c, g, si, s.guard) + result.bitsize = s.bitsize + result.alignment = s.alignment + result.owner = loadSym(c, g, si, s.owner) + let externalName = g[si].fromDisk.strings[s.externalName] + if externalName != "": + result.loc.snippet = externalName + result.loc.flags = s.locFlags + result.instantiatedFrom = loadSym(c, g, si, s.instantiatedFrom) + +proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; + fileIdx: FileIndex; cachedModules: var seq[FileIndex]): bool +proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; + fileIdx: FileIndex; m: var LoadedModule) + +proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s: PackedItemId): PSym = + if s == nilItemId: + result = nil + else: + let si = moduleIndex(c, g, thisModule, s) + if si >= g.len: + g.pm.setLen(si+1) + + if g[si].status == undefined and c.config.cmd == cmdM: + var cachedModules: seq[FileIndex] = @[] + discard needsRecompile(g, c.config, c.cache, FileIndex(si), cachedModules) + for m in cachedModules: + loadToReplayNodes(g, c.config, c.cache, m, g[int m]) + + assert g[si].status in {loaded, storing, stored} + #if not g[si].symsInit: + # g[si].symsInit = true + # setLen g[si].syms, g[si].fromDisk.syms.len + + if g[si].syms.getOrDefault(s.item) == nil: + if g[si].fromDisk.syms[s.item].kind != skModule: + result = symHeaderFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item) + # store it here early on, so that recursions work properly: + g[si].syms[s.item] = result + symBodyFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item, result) + else: + result = g[si].module + assert result != nil + g[si].syms[s.item] = result + + else: + result = g[si].syms[s.item] + +proc typeHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; + t: PackedType; si, item: int32): PType = + result = PType(itemId: ItemId(module: si, item: t.nonUniqueId), kind: t.kind, + flags: t.flags, size: t.size, align: t.align, + paddingAtEnd: t.paddingAtEnd, + uniqueId: ItemId(module: si, item: item), + callConv: t.callConv) + +proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; + t: PackedType; si, item: int32; result: PType) = + result.sym = loadSym(c, g, si, t.sym) + result.owner = loadSym(c, g, si, t.owner) + when false: + for op, item in pairs t.attachedOps: + result.attachedOps[op] = loadSym(c, g, si, item) + result.typeInst = loadType(c, g, si, t.typeInst) + var sons = newSeq[PType]() + for son in items t.types: + sons.add loadType(c, g, si, son) + result.setSons(sons) + loadAstBody(t, n) + when false: + for gen, id in items t.methods: + result.methods.add((gen, loadSym(c, g, si, id))) + +proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t: PackedItemId): PType = + if t == nilItemId: + result = nil + else: + let si = moduleIndex(c, g, thisModule, t) + assert g[si].status in {loaded, storing, stored} + assert t.item > 0 + + #if not g[si].typesInit: + # g[si].typesInit = true + # setLen g[si].types, g[si].fromDisk.types.len + + if g[si].types.getOrDefault(t.item) == nil: + result = typeHeaderFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item) + # store it here early on, so that recursions work properly: + g[si].types[t.item] = result + typeBodyFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item, result) + #assert result.itemId.item == t.item, $(result.itemId.item, t.item) + assert result.itemId.item > 0, $(result.itemId.item, t.item) + else: + result = g[si].types[t.item] + assert result.itemId.item > 0, "2" + +proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; + fileIdx: FileIndex; m: var LoadedModule) = + m.iface = initTable[PIdent, seq[PackedItemId]]() + m.ifaceHidden = initTable[PIdent, seq[PackedItemId]]() + template impl(iface, e) = + let nameLit = e[0] + let e2 = + when e[1] is PackedItemId: e[1] + else: PackedItemId(module: LitId(0), item: e[1]) + iface.mgetOrPut(cache.getIdent(m.fromDisk.strings[nameLit]), @[]).add(e2) + + for e in m.fromDisk.exports: + m.iface.impl(e) + m.ifaceHidden.impl(e) + for e in m.fromDisk.reexports: + m.iface.impl(e) + m.ifaceHidden.impl(e) + for e in m.fromDisk.hidden: + m.ifaceHidden.impl(e) + + let filename = AbsoluteFile toFullPath(conf, fileIdx) + # We cannot call ``newSym`` here, because we have to circumvent the ID + # mechanism, which we do in order to assign each module a persistent ID. + m.module = PSym(kind: skModule, itemId: ItemId(module: int32(fileIdx), item: 0'i32), + name: getIdent(cache, splitFile(filename).name), + info: newLineInfo(fileIdx, 1, 1), + position: int(fileIdx)) + m.module.owner = getPackage(conf, cache, fileIdx) + m.module.flags = m.fromDisk.moduleFlags + +proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; + fileIdx: FileIndex; m: var LoadedModule) = + m.module.ast = newNode(nkStmtList) + if m.fromDisk.toReplay.len > 0: + var decoder = PackedDecoder( + lastModule: int32(-1), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: conf, + cache: cache) + for p in allNodes(m.fromDisk.toReplay): + m.module.ast.add loadNodes(decoder, g, int(fileIdx), m.fromDisk.toReplay, p) + +proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; + fileIdx: FileIndex; cachedModules: var seq[FileIndex]): bool = + # Does the file belong to the fileIdx need to be recompiled? + let m = int(fileIdx) + if m >= g.len: + g.pm.setLen(m+1) + + case g[m].status + of undefined: + g[m].status = loading + let fullpath = msgs.toFullPath(conf, fileIdx) + let rod = toRodFile(conf, AbsoluteFile fullpath) + let err = loadRodFile(rod, g[m].fromDisk, conf, ignoreConfig = conf.cmd == cmdM) + if err == ok: + if conf.cmd == cmdM: + setupLookupTables(g, conf, cache, fileIdx, g[m]) + cachedModules.add fileIdx + g[m].status = loaded + result = false + else: + result = optForceFullMake in conf.globalOptions + # check its dependencies: + let imp = g[m].fromDisk.imports + for dep in imp: + let fid = toFileIndex(dep, g[m].fromDisk, conf) + # Warning: we need to traverse the full graph, so + # do **not use break here**! + if needsRecompile(g, conf, cache, fid, cachedModules): + result = true + + if not result: + setupLookupTables(g, conf, cache, fileIdx, g[m]) + cachedModules.add fileIdx + g[m].status = loaded + else: + g.pm[m] = LoadedModule(status: outdated, module: g[m].module) + else: + loadError(err, rod, conf) + g[m].status = outdated + result = true + when false: loadError(err, rod, conf) + of loading, loaded: + # For loading: Assume no recompile is required. + result = false + of outdated, storing, stored: + result = true + +proc moduleFromRodFile*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; + fileIdx: FileIndex; cachedModules: var seq[FileIndex]): PSym = + ## Returns 'nil' if the module needs to be recompiled. + bench g.depAnalysis: + if needsRecompile(g, conf, cache, fileIdx, cachedModules): + result = nil + else: + result = g[int fileIdx].module + assert result != nil + assert result.position == int(fileIdx) + for m in cachedModules: + loadToReplayNodes(g, conf, cache, m, g[int m]) + +template setupDecoder() {.dirty.} = + var decoder = PackedDecoder( + lastModule: int32(-1), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + +proc loadProcBody*(config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; s: PSym): PNode = + bench g.loadBody: + let mId = s.itemId.module + var decoder = PackedDecoder( + lastModule: int32(-1), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + let pos = g[mId].fromDisk.syms[s.itemId.item].ast + assert pos != emptyNodeId + result = loadProcBody(decoder, g, mId, g[mId].fromDisk.bodies, NodePos pos) + +proc loadTypeFromId*(config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; module: int; id: PackedItemId): PType = + bench g.loadType: + result = g[module].types.getOrDefault(id.item) + if result == nil: + var decoder = PackedDecoder( + lastModule: int32(-1), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + result = loadType(decoder, g, module, id) + +proc loadSymFromId*(config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; module: int; id: PackedItemId): PSym = + bench g.loadSym: + result = g[module].syms.getOrDefault(id.item) + if result == nil: + var decoder = PackedDecoder( + lastModule: int32(-1), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + result = loadSym(decoder, g, module, id) + +proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; config: ConfigRef): ItemId = + if id.module == LitId(0): + ItemId(module: thisModule.int32, item: id.item) + else: + ItemId(module: toFileIndex(id.module, g[thisModule].fromDisk, config).int32, item: id.item) + +proc simulateLoadedModule*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; + moduleSym: PSym; m: PackedModule) = + # For now only used for heavy debugging. In the future we could use this to reduce the + # compiler's memory consumption. + let idx = moduleSym.position + assert g[idx].status in {storing} + g[idx].status = loaded + assert g[idx].module == moduleSym + setupLookupTables(g, conf, cache, FileIndex(idx), g[idx]) + loadToReplayNodes(g, conf, cache, FileIndex(idx), g[idx]) + +# ---------------- symbol table handling ---------------- + +type + RodIter* = object + decoder: PackedDecoder + values: seq[PackedItemId] + i, module: int + +template interfSelect(a: LoadedModule, importHidden: bool): auto = + var ret = a.iface.addr + if importHidden: ret = a.ifaceHidden.addr + ret[] + +proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; module: FileIndex; + name: PIdent, importHidden: bool): PSym = + it.decoder = PackedDecoder( + lastModule: int32(-1), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + it.values = g[int module].interfSelect(importHidden).getOrDefault(name) + it.i = 0 + it.module = int(module) + if it.i < it.values.len: + result = loadSym(it.decoder, g, int(module), it.values[it.i]) + inc it.i + else: + result = nil + +proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; module: FileIndex; importHidden: bool): PSym = + it.decoder = PackedDecoder( + lastModule: int32(-1), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + it.values = @[] + it.module = int(module) + for v in g[int module].interfSelect(importHidden).values: + it.values.add v + it.i = 0 + if it.i < it.values.len: + result = loadSym(it.decoder, g, int(module), it.values[it.i]) + inc it.i + else: + result = nil + +proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym = + if it.i < it.values.len: + result = loadSym(it.decoder, g, it.module, it.values[it.i]) + inc it.i + else: + result = nil + +iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; module: FileIndex; + name: PIdent, importHidden: bool): PSym = + setupDecoder() + let values = g[int module].interfSelect(importHidden).getOrDefault(name) + for pid in values: + let s = loadSym(decoder, g, int(module), pid) + assert s != nil + yield s + +proc interfaceSymbol*(config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; module: FileIndex; + name: PIdent, importHidden: bool): PSym = + setupDecoder() + let values = g[int module].interfSelect(importHidden).getOrDefault(name) + result = loadSym(decoder, g, int(module), values[0]) + +proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator = + IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.syms.len, + typeId: int32 m.fromDisk.types.len) + +proc searchForCompilerproc*(m: LoadedModule; name: string): int32 = + # slow, linear search, but the results are cached: + for it in items(m.fromDisk.compilerProcs): + if m.fromDisk.strings[it[0]] == name: + return it[1] + return -1 + +# ------------------------- .rod file viewer --------------------------------- + +proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = + var m: PackedModule = PackedModule() + let err = loadRodFile(rodfile, m, config, ignoreConfig=true) + if err != ok: + config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err + + when false: + echo "exports:" + for ex in m.exports: + echo " ", m.strings[ex[0]], " local ID: ", ex[1] + assert ex[0] == m.syms[ex[1]].name + # ex[1] int32 + + echo "reexports:" + for ex in m.reexports: + echo " ", m.strings[ex[0]] + # reexports*: seq[(LitId, PackedItemId)] + + echo "hidden: " & $m.hidden.len + for ex in m.hidden: + echo " ", m.strings[ex[0]], " local ID: ", ex[1] + + when false: + echo "all symbols" + for i in 0..high(m.syms): + if m.syms[i].name != LitId(0): + echo " ", m.strings[m.syms[i].name], " local ID: ", i, " kind ", m.syms[i].kind + else: + echo " <anon symbol?> local ID: ", i, " kind ", m.syms[i].kind + + echo "symbols: ", m.syms.len, " types: ", m.types.len, + " top level nodes: ", m.topLevel.len, " other nodes: ", m.bodies.len, + " strings: ", m.strings.len, " numbers: ", m.numbers.len + + echo "SIZES:" + echo "symbols: ", m.syms.len * sizeof(PackedSym), " types: ", m.types.len * sizeof(PackedType), + " top level nodes: ", m.topLevel.len * sizeof(PackedNode), + " other nodes: ", m.bodies.len * sizeof(PackedNode), + " strings: ", sizeOnDisc(m.strings) + when false: + var tt = 0 + var fc = 0 + for x in m.topLevel: + if x.kind == nkSym or x.typeId == nilItemId: inc tt + if x.flags == {}: inc fc + for x in m.bodies: + if x.kind == nkSym or x.typeId == nilItemId: inc tt + if x.flags == {}: inc fc + let total = float(m.topLevel.len + m.bodies.len) + echo "nodes with nil type: ", tt, " in % ", tt.float / total + echo "nodes with empty flags: ", fc.float / total |