diff options
Diffstat (limited to 'compiler/modulegraphs.nim')
| -rw-r--r-- | compiler/modulegraphs.nim | 673 |
1 files changed, 593 insertions, 80 deletions
diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index da2a222a4..77762d23a 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -9,35 +9,85 @@ ## This module implements the module graph data structure. The module graph ## represents a complete Nim project. Single modules can either be kept in RAM -## or stored in a Sqlite database. -## -## The caching of modules is critical for 'nimsuggest' and is tricky to get -## right. If module E is being edited, we need autocompletion (and type -## checking) for E but we don't want to recompile depending -## modules right away for faster turnaround times. Instead we mark the module's -## dependencies as 'dirty'. Let D be a dependency of E. If D is dirty, we -## need to recompile it and all of its dependencies that are marked as 'dirty'. -## 'nimsuggest sug' actually is invoked for the file being edited so we know -## its content changed and there is no need to compute any checksums. -## Instead of a recursive algorithm, we use an iterative algorithm: -## -## - If a module gets recompiled, its dependencies need to be updated. -## - Its dependent module stays the same. -## - -import ast, intsets, tables, options, lineinfos, hashes, idents, - incremental, btrees, md5 +## or stored in a rod-file. + +import std/[intsets, tables, hashes, strtabs, algorithm, os, strutils, parseutils] +import ../dist/checksums/src/checksums/md5 +import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages, suggestsymdb +import ic / [packed_ast, ic] + + +when defined(nimPreviewSlimSystem): + import std/assertions type SigHash* = distinct MD5Digest - ModuleGraph* = ref object - modules*: seq[PSym] ## indexed by int32 fileIdx + LazySym* = object + id*: FullId + sym*: PSym + + Iface* = object ## data we don't want to store directly in the + ## ast.PSym type for s.kind == skModule + module*: PSym ## module this "Iface" belongs to + converters*: seq[LazySym] + patterns*: seq[LazySym] + pureEnums*: seq[LazySym] + interf: TStrTable + interfHidden: TStrTable + uniqueName*: Rope + + Operators* = object + opNot*, opContains*, opLe*, opLt*, opAnd*, opOr*, opIsNil*, opEq*: PSym + opAdd*, opSub*, opMul*, opDiv*, opLen*: PSym + + FullId* = object + module*: int + packed*: PackedItemId + + LazyType* = object + id*: FullId + typ*: PType + + LazyInstantiation* = object + module*: int + sym*: FullId + concreteTypes*: seq[FullId] + inst*: PInstantiation + + PipelinePass* = enum + NonePass + SemPass + JSgenPass + CgenPass + EvalPass + InterpreterPass + GenDependPass + Docgen2TexPass + Docgen2JsonPass + Docgen2Pass + + ModuleGraph* {.acyclic.} = ref object + ifaces*: seq[Iface] ## indexed by int32 fileIdx + packed*: PackedModuleGraph + encoders*: seq[PackedEncoder] + + typeInstCache*: Table[ItemId, seq[LazyType]] # A symbol's ItemId. + procInstCache*: Table[ItemId, seq[LazyInstantiation]] # A symbol's ItemId. + attachedOps*: array[TTypeAttachedOp, Table[ItemId, LazySym]] # Type ID, destructors, etc. + methodsPerGenericType*: Table[ItemId, seq[(int, LazySym)]] # Type ID, attached methods + memberProcsPerType*: Table[ItemId, seq[PSym]] # Type ID, attached member procs (only c++, virtual,member and ctor so far). + initializersPerType*: Table[ItemId, PNode] # Type ID, AST call to the default ctor (c++ only) + enumToStringProcs*: Table[ItemId, LazySym] + emittedTypeInfo*: Table[string, FileIndex] + + startupPackedConfig*: PackedConfig packageSyms*: TStrTable deps*: IntSet # the dependency graph or potentially its transitive closure. importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies suggestMode*: bool # whether we are in nimsuggest mode or not. invalidTransitiveClosure: bool + interactive*: bool inclToMod*: Table[FileIndex, FileIndex] # mapping of include file to the # first module that included it importStack*: seq[FileIndex] # The current import stack. Used for detecting recursive @@ -47,38 +97,49 @@ type cache*: IdentCache vm*: RootRef # unfortunately the 'vm' state is shared project-wise, this will # be clarified in later compiler implementations. + repl*: RootRef # REPL state is shared project-wise. doStopCompile*: proc(): bool {.closure.} usageSym*: PSym # for nimsuggest owners*: seq[PSym] + suggestSymbols*: SuggestSymbolDatabase + suggestErrors*: Table[FileIndex, seq[Suggest]] methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization! + bucketTable*: CountTable[ItemId] + objectTree*: Table[ItemId, seq[tuple[depth: int, value: PType]]] + methodsPerType*: Table[ItemId, seq[LazySym]] + dispatchers*: seq[LazySym] + systemModule*: PSym sysTypes*: array[TTypeKind, PType] compilerprocs*: TStrTable exposed*: TStrTable - intTypeCache*: array[-5..64, PType] - opContains*, opNot*: PSym + packageTypes*: TStrTable emptyNode*: PNode - incr*: IncrementalCtx canonTypes*: Table[SigHash, PType] symBodyHashes*: Table[int, SigHash] # symId to digest mapping importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.} includeFileCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PNode {.nimcall.} - recordStmt*: proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.} - cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API - cacheCounters*: Table[string, BiggestInt] - cacheTables*: Table[string, BTree[string, PNode]] + cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API; IC: implemented + cacheCounters*: Table[string, BiggestInt] # IC: implemented + cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented passes*: seq[TPass] + pipelinePass*: PipelinePass onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.} onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.} onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.} globalDestructors*: seq[PNode] strongSemCheck*: proc (graph: ModuleGraph; owner: PSym; body: PNode) {.nimcall.} compatibleProps*: proc (graph: ModuleGraph; formal, actual: PType): bool {.nimcall.} + idgen*: IdGenerator + operators*: Operators + + cachedFiles*: StringTableRef TPassContext* = object of RootObj # the pass's context + idgen*: IdGenerator PPassContext* = ref TPassContext - TPassOpen* = proc (graph: ModuleGraph; module: PSym): PPassContext {.nimcall.} + TPassOpen* = proc (graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nimcall.} TPassClose* = proc (graph: ModuleGraph; p: PPassContext, n: PNode): PNode {.nimcall.} TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.} @@ -87,6 +148,16 @@ type close: TPassClose, isFrontend: bool] +proc resetForBackend*(g: ModuleGraph) = + g.compilerprocs = initStrTable() + g.typeInstCache.clear() + g.procInstCache.clear() + for a in mitems(g.attachedOps): + a.clear() + g.methodsPerGenericType.clear() + g.enumToStringProcs.clear() + g.dispatchers.setLen(0) + g.methodsPerType.clear() const cb64 = [ @@ -122,6 +193,248 @@ proc toBase64a(s: cstring, len: int): string = result.add cb64[a shr 2] result.add cb64[(a and 3) shl 4] +template interfSelect(iface: Iface, importHidden: bool): TStrTable = + var ret = iface.interf.addr # without intermediate ptr, it creates a copy and compiler becomes 15x slower! + if importHidden: ret = iface.interfHidden.addr + ret[] + +template semtab(g: ModuleGraph, m: PSym): TStrTable = + g.ifaces[m.position].interf + +template semtabAll*(g: ModuleGraph, m: PSym): TStrTable = + g.ifaces[m.position].interfHidden + +proc initStrTables*(g: ModuleGraph, m: PSym) = + semtab(g, m) = initStrTable() + semtabAll(g, m) = initStrTable() + +proc strTableAdds*(g: ModuleGraph, m: PSym, s: PSym) = + strTableAdd(semtab(g, m), s) + strTableAdd(semtabAll(g, m), s) + +proc isCachedModule(g: ModuleGraph; module: int): bool {.inline.} = + result = module < g.packed.len and g.packed[module].status == loaded + +proc isCachedModule*(g: ModuleGraph; m: PSym): bool {.inline.} = + isCachedModule(g, m.position) + +proc simulateCachedModule(g: ModuleGraph; moduleSym: PSym; m: PackedModule) = + when false: + echo "simulating ", moduleSym.name.s, " ", moduleSym.position + simulateLoadedModule(g.packed, g.config, g.cache, moduleSym, m) + +proc initEncoder*(g: ModuleGraph; module: PSym) = + let id = module.position + if id >= g.encoders.len: + setLen g.encoders, id+1 + ic.initEncoder(g.encoders[id], + g.packed[id].fromDisk, module, g.config, g.startupPackedConfig) + +type + ModuleIter* = object + fromRod: bool + modIndex: int + ti: TIdentIter + rodIt: RodIter + importHidden: bool + +proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym = + assert m.kind == skModule + mi.modIndex = m.position + mi.fromRod = isCachedModule(g, mi.modIndex) + mi.importHidden = optImportHidden in m.options + if mi.fromRod: + result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name, mi.importHidden) + else: + result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name) + +proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym = + if mi.fromRod: + result = nextRodIter(mi.rodIt, g.packed) + else: + result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden)) + +iterator allSyms*(g: ModuleGraph; m: PSym): PSym = + let importHidden = optImportHidden in m.options + if isCachedModule(g, m): + var rodIt: RodIter = default(RodIter) + var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position, importHidden) + while r != nil: + yield r + r = nextRodIter(rodIt, g.packed) + else: + for s in g.ifaces[m.position].interfSelect(importHidden).data: + if s != nil: + yield s + +proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym = + let importHidden = optImportHidden in m.options + if isCachedModule(g, m): + result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden) + else: + result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name) + +proc someSymAmb*(g: ModuleGraph; m: PSym; name: PIdent; amb: var bool): PSym = + let importHidden = optImportHidden in m.options + if isCachedModule(g, m): + result = nil + for s in interfaceSymbols(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden): + if result == nil: + # set result to the first symbol + result = s + else: + # another symbol found + amb = true + break + else: + var ti: TIdentIter = default(TIdentIter) + result = initIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden), name) + if result != nil and nextIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden)) != nil: + # another symbol exists with same name + amb = true + +proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym = + result = someSym(g, g.systemModule, name) + +iterator systemModuleSyms*(g: ModuleGraph; name: PIdent): PSym = + var mi: ModuleIter = default(ModuleIter) + var r = initModuleIter(mi, g, g.systemModule, name) + while r != nil: + yield r + r = nextModuleIter(mi, g) + +proc resolveType(g: ModuleGraph; t: var LazyType): PType = + result = t.typ + if result == nil and isCachedModule(g, t.id.module): + result = loadTypeFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed) + t.typ = result + assert result != nil + +proc resolveSym(g: ModuleGraph; t: var LazySym): PSym = + result = t.sym + if result == nil and isCachedModule(g, t.id.module): + result = loadSymFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed) + t.sym = result + assert result != nil + +proc resolveInst(g: ModuleGraph; t: var LazyInstantiation): PInstantiation = + result = t.inst + if result == nil and isCachedModule(g, t.module): + result = PInstantiation(sym: loadSymFromId(g.config, g.cache, g.packed, t.sym.module, t.sym.packed)) + result.concreteTypes = newSeq[PType](t.concreteTypes.len) + for i in 0..high(result.concreteTypes): + result.concreteTypes[i] = loadTypeFromId(g.config, g.cache, g.packed, + t.concreteTypes[i].module, t.concreteTypes[i].packed) + t.inst = result + assert result != nil + +proc resolveAttachedOp*(g: ModuleGraph; t: var LazySym): PSym = + result = t.sym + if result == nil: + result = loadSymFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed) + t.sym = result + assert result != nil + +iterator typeInstCacheItems*(g: ModuleGraph; s: PSym): PType = + if g.typeInstCache.contains(s.itemId): + let x = addr(g.typeInstCache[s.itemId]) + for t in mitems(x[]): + yield resolveType(g, t) + +iterator procInstCacheItems*(g: ModuleGraph; s: PSym): PInstantiation = + if g.procInstCache.contains(s.itemId): + let x = addr(g.procInstCache[s.itemId]) + for t in mitems(x[]): + yield resolveInst(g, t) + + +proc getAttachedOp*(g: ModuleGraph; t: PType; op: TTypeAttachedOp): PSym = + ## returns the requested attached operation for type `t`. Can return nil + ## if no such operation exists. + if g.attachedOps[op].contains(t.itemId): + result = resolveAttachedOp(g, g.attachedOps[op][t.itemId]) + else: + result = nil + +proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) = + ## we also need to record this to the packed module. + g.attachedOps[op][t.itemId] = LazySym(sym: value) + +proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) = + ## we also need to record this to the packed module. + g.attachedOps[op][t.itemId] = LazySym(sym: value) + +proc completePartialOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) = + if g.config.symbolFiles != disabledSf: + assert module < g.encoders.len + assert isActive(g.encoders[module]) + toPackedGeneratedProcDef(value, g.encoders[module], g.packed[module].fromDisk) + #storeAttachedProcDef(t, op, value, g.encoders[module], g.packed[module].fromDisk) + +iterator getDispatchers*(g: ModuleGraph): PSym = + for i in g.dispatchers.mitems: + yield resolveSym(g, i) + +proc addDispatchers*(g: ModuleGraph, value: PSym) = + # TODO: add it for packed modules + g.dispatchers.add LazySym(sym: value) + +iterator resolveLazySymSeq(g: ModuleGraph, list: var seq[LazySym]): PSym = + for it in list.mitems: + yield resolveSym(g, it) + +proc setMethodsPerType*(g: ModuleGraph; id: ItemId, methods: seq[LazySym]) = + # TODO: add it for packed modules + g.methodsPerType[id] = methods + +iterator getMethodsPerType*(g: ModuleGraph; t: PType): PSym = + if g.methodsPerType.contains(t.itemId): + for it in mitems g.methodsPerType[t.itemId]: + yield resolveSym(g, it) + +proc getToStringProc*(g: ModuleGraph; t: PType): PSym = + result = resolveSym(g, g.enumToStringProcs[t.itemId]) + assert result != nil + +proc setToStringProc*(g: ModuleGraph; t: PType; value: PSym) = + g.enumToStringProcs[t.itemId] = LazySym(sym: value) + +iterator methodsForGeneric*(g: ModuleGraph; t: PType): (int, PSym) = + if g.methodsPerGenericType.contains(t.itemId): + for it in mitems g.methodsPerGenericType[t.itemId]: + yield (it[0], resolveSym(g, it[1])) + +proc addMethodToGeneric*(g: ModuleGraph; module: int; t: PType; col: int; m: PSym) = + g.methodsPerGenericType.mgetOrPut(t.itemId, @[]).add (col, LazySym(sym: m)) + +proc hasDisabledAsgn*(g: ModuleGraph; t: PType): bool = + let op = getAttachedOp(g, t, attachedAsgn) + result = op != nil and sfError in op.flags + +proc copyTypeProps*(g: ModuleGraph; module: int; dest, src: PType) = + for k in low(TTypeAttachedOp)..high(TTypeAttachedOp): + let op = getAttachedOp(g, src, k) + if op != nil: + setAttachedOp(g, module, dest, k, op) + +proc loadCompilerProc*(g: ModuleGraph; name: string): PSym = + result = nil + if g.config.symbolFiles == disabledSf: return nil + + # slow, linear search, but the results are cached: + for module in 0..<len(g.packed): + #if isCachedModule(g, module): + let x = searchForCompilerproc(g.packed[module], name) + if x >= 0: + result = loadSymFromId(g.config, g.cache, g.packed, module, toPackedItemId(x)) + if result != nil: + strTableAdd(g.compilerprocs, result) + return result + +proc loadPackedSym*(g: ModuleGraph; s: var LazySym) = + if s.sym == nil: + s.sym = loadSymFromId(g.config, g.cache, g.packed, s.id.module, s.id.packed) + proc `$`*(u: SigHash): string = toBase64a(cast[cstring](unsafeAddr u), sizeof(u)) @@ -135,27 +448,13 @@ proc hash*(u: SigHash): Hash = proc hash*(x: FileIndex): Hash {.borrow.} -when defined(nimfind): - template onUse*(info: TLineInfo; s: PSym) = - when compiles(c.c.graph): - if c.c.graph.onUsage != nil: c.c.graph.onUsage(c.c.graph, s, info) - else: - if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info) - - template onDef*(info: TLineInfo; s: PSym) = - when compiles(c.c.graph): - if c.c.graph.onDefinition != nil: c.c.graph.onDefinition(c.c.graph, s, info) - else: - if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info) - - template onDefResolveForward*(info: TLineInfo; s: PSym) = - when compiles(c.c.graph): - if c.c.graph.onDefinitionResolveForward != nil: - c.c.graph.onDefinitionResolveForward(c.c.graph, s, info) - else: - if c.graph.onDefinitionResolveForward != nil: - c.graph.onDefinitionResolveForward(c.graph, s, info) +template getPContext(): untyped = + when c is PContext: c + else: c.c +when defined(nimsuggest): + template onUse*(info: TLineInfo; s: PSym) = discard + template onDefResolveForward*(info: TLineInfo; s: PSym) = discard else: template onUse*(info: TLineInfo; s: PSym) = discard template onDef*(info: TLineInfo; s: PSym) = discard @@ -164,58 +463,188 @@ else: proc stopCompile*(g: ModuleGraph): bool {.inline.} = result = g.doStopCompile != nil and g.doStopCompile() -proc createMagic*(g: ModuleGraph; name: string, m: TMagic): PSym = - result = newSym(skProc, getIdent(g.cache, name), nil, unknownLineInfo, {}) +proc createMagic*(g: ModuleGraph; idgen: IdGenerator; name: string, m: TMagic): PSym = + result = newSym(skProc, getIdent(g.cache, name), idgen, nil, unknownLineInfo, {}) result.magic = m result.flags = {sfNeverRaises} -proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = - result = ModuleGraph() - initStrTable(result.packageSyms) +proc createMagic(g: ModuleGraph; name: string, m: TMagic): PSym = + result = createMagic(g, g.idgen, name, m) + +proc uniqueModuleName*(conf: ConfigRef; m: PSym): string = + ## The unique module name is guaranteed to only contain {'A'..'Z', 'a'..'z', '0'..'9', '_'} + ## so that it is useful as a C identifier snippet. + let fid = FileIndex(m.position) + let path = AbsoluteFile toFullPath(conf, fid) + var isLib = false + var rel = "" + if path.string.startsWith(conf.libpath.string): + isLib = true + rel = relativeTo(path, conf.libpath).string + else: + rel = relativeTo(path, conf.projectPath).string + + if not isLib and not belongsToProjectPackage(conf, m): + # special handlings for nimble packages + when DirSep == '\\': + let rel2 = replace(rel, '\\', '/') + else: + let rel2 = rel + const pkgs2 = "pkgs2/" + var start = rel2.find(pkgs2) + if start >= 0: + start += pkgs2.len + start += skipUntil(rel2, {'/'}, start) + if start+1 < rel2.len: + rel = "pkg/" & rel2[start+1..<rel.len] # strips paths + + let trunc = if rel.endsWith(".nim"): rel.len - len(".nim") else: rel.len + result = newStringOfCap(trunc) + for i in 0..<trunc: + let c = rel[i] + case c + of 'a'..'z', '0'..'9': + result.add c + of {os.DirSep, os.AltSep}: + result.add 'Z' # because it looks a bit like '/' + of '.': + result.add 'O' # a circle + else: + # We mangle upper letters too so that there cannot + # be clashes with our special meanings of 'Z' and 'O' + result.addInt ord(c) + +proc registerModule*(g: ModuleGraph; m: PSym) = + assert m != nil + assert m.kind == skModule + + if m.position >= g.ifaces.len: + setLen(g.ifaces, m.position + 1) + + if m.position >= g.packed.len: + setLen(g.packed.pm, m.position + 1) + + g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[], + uniqueName: rope(uniqueModuleName(g.config, m))) + initStrTables(g, m) + +proc registerModuleById*(g: ModuleGraph; m: FileIndex) = + registerModule(g, g.packed[int m].module) + +proc initOperators*(g: ModuleGraph): Operators = + # These are safe for IC. + # Public because it's used by DrNim. + result = Operators( + opLe: createMagic(g, "<=", mLeI), + opLt: createMagic(g, "<", mLtI), + opAnd: createMagic(g, "and", mAnd), + opOr: createMagic(g, "or", mOr), + opIsNil: createMagic(g, "isnil", mIsNil), + opEq: createMagic(g, "==", mEqI), + opAdd: createMagic(g, "+", mAddI), + opSub: createMagic(g, "-", mSubI), + opMul: createMagic(g, "*", mMulI), + opDiv: createMagic(g, "div", mDivI), + opLen: createMagic(g, "len", mLengthSeq), + opNot: createMagic(g, "not", mNot), + opContains: createMagic(g, "contains", mInSet) + ) + +proc initModuleGraphFields(result: ModuleGraph) = + # A module ID of -1 means that the symbol is not attached to a module at all, + # but to the module graph: + result.idgen = IdGenerator(module: -1'i32, symId: 0'i32, typeId: 0'i32) + result.packageSyms = initStrTable() result.deps = initIntSet() result.importDeps = initTable[FileIndex, seq[FileIndex]]() - result.modules = @[] + result.ifaces = @[] result.importStack = @[] result.inclToMod = initTable[FileIndex, FileIndex]() - result.config = config - result.cache = cache result.owners = @[] + result.suggestSymbols = initTable[FileIndex, SuggestFileSymbolDatabase]() + result.suggestErrors = initTable[FileIndex, seq[Suggest]]() result.methods = @[] - initStrTable(result.compilerprocs) - initStrTable(result.exposed) - result.opNot = createMagic(result, "not", mNot) - result.opContains = createMagic(result, "contains", mInSet) + result.compilerprocs = initStrTable() + result.exposed = initStrTable() + result.packageTypes = initStrTable() result.emptyNode = newNode(nkEmpty) - init(result.incr) - result.recordStmt = proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.} = - discard result.cacheSeqs = initTable[string, PNode]() result.cacheCounters = initTable[string, BiggestInt]() result.cacheTables = initTable[string, BTree[string, PNode]]() result.canonTypes = initTable[SigHash, PType]() result.symBodyHashes = initTable[int, SigHash]() + result.operators = initOperators(result) + result.emittedTypeInfo = initTable[string, FileIndex]() + result.cachedFiles = newStringTable() + +proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = + result = ModuleGraph() + result.config = config + result.cache = cache + initModuleGraphFields(result) proc resetAllModules*(g: ModuleGraph) = - initStrTable(g.packageSyms) + g.packageSyms = initStrTable() g.deps = initIntSet() - g.modules = @[] + g.ifaces = @[] g.importStack = @[] g.inclToMod = initTable[FileIndex, FileIndex]() g.usageSym = nil g.owners = @[] g.methods = @[] - initStrTable(g.compilerprocs) - initStrTable(g.exposed) + g.compilerprocs = initStrTable() + g.exposed = initStrTable() + initModuleGraphFields(g) proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym = - if fileIdx.int32 >= 0 and fileIdx.int32 < g.modules.len: - result = g.modules[fileIdx.int32] + result = nil + if fileIdx.int32 >= 0: + if isCachedModule(g, fileIdx.int32): + result = g.packed[fileIdx.int32].module + elif fileIdx.int32 < g.ifaces.len: + result = g.ifaces[fileIdx.int32].module + +proc moduleOpenForCodegen*(g: ModuleGraph; m: FileIndex): bool {.inline.} = + if g.config.symbolFiles == disabledSf: + result = true + else: + result = g.packed[m.int32].status notin {undefined, stored, loaded} + +proc rememberEmittedTypeInfo*(g: ModuleGraph; m: FileIndex; ti: string) = + #assert(not isCachedModule(g, m.int32)) + if g.config.symbolFiles != disabledSf: + #assert g.encoders[m.int32].isActive + assert g.packed[m.int32].status != stored + g.packed[m.int32].fromDisk.emittedTypeInfo.add ti + #echo "added typeinfo ", m.int32, " ", ti, " suspicious ", not g.encoders[m.int32].isActive + +proc rememberFlag*(g: ModuleGraph; m: PSym; flag: ModuleBackendFlag) = + if g.config.symbolFiles != disabledSf: + #assert g.encoders[m.int32].isActive + assert g.packed[m.position].status != stored + g.packed[m.position].fromDisk.backendFlags.incl flag + +proc closeRodFile*(g: ModuleGraph; m: PSym) = + if g.config.symbolFiles in {readOnlySf, v2Sf}: + # For stress testing we seek to reload the symbols from memory. This + # way much of the logic is tested but the test is reproducible as it does + # not depend on the hard disk contents! + let mint = m.position + saveRodFile(toRodFile(g.config, AbsoluteFile toFullPath(g.config, FileIndex(mint))), + g.encoders[mint], g.packed[mint].fromDisk) + g.packed[mint].status = stored + + elif g.config.symbolFiles == stressTest: + # debug code, but maybe a good idea for production? Could reduce the compiler's + # memory consumption considerably at the cost of more loads from disk. + let mint = m.position + simulateCachedModule(g, m, g.packed[mint].fromDisk) + g.packed[mint].status = loaded proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) = assert m.position == m.info.fileIndex.int32 - addModuleDep(g.incr, g.config, m.info.fileIndex, dep, isIncludeFile = false) if g.suggestMode: g.deps.incl m.position.dependsOn(dep.int) # we compute the transitive closure later when querying the graph lazily. @@ -223,14 +652,13 @@ proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) = #invalidTransitiveClosure = true proc addIncludeDep*(g: ModuleGraph; module, includeFile: FileIndex) = - addModuleDep(g.incr, g.config, module, includeFile, isIncludeFile = true) discard hasKeyOrPut(g.inclToMod, includeFile, module) proc parentModule*(g: ModuleGraph; fileIdx: FileIndex): FileIndex = ## returns 'fileIdx' if the file belonging to this index is ## directly used as a module or else the module that first ## references this include file. - if fileIdx.int32 >= 0 and fileIdx.int32 < g.modules.len and g.modules[fileIdx.int32] != nil: + if fileIdx.int32 >= 0 and fileIdx.int32 < g.ifaces.len and g.ifaces[fileIdx.int32].module != nil: result = fileIdx else: result = g.inclToMod.getOrDefault(fileIdx) @@ -246,7 +674,19 @@ proc transitiveClosure(g: var IntSet; n: int) = proc markDirty*(g: ModuleGraph; fileIdx: FileIndex) = let m = g.getModule fileIdx - if m != nil: incl m.flags, sfDirty + if m != nil: + g.suggestSymbols.del(fileIdx) + g.suggestErrors.del(fileIdx) + incl m.flags, sfDirty + +proc unmarkAllDirty*(g: ModuleGraph) = + for i in 0i32..<g.ifaces.len.int32: + let m = g.ifaces[i].module + if m != nil: + m.flags.excl sfDirty + +proc isDirty*(g: ModuleGraph; m: PSym): bool = + result = g.suggestMode and sfDirty in m.flags proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) = # we need to mark its dependent modules D as dirty right away because after @@ -254,13 +694,86 @@ proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) = # cleared but D still needs to be remembered as 'dirty'. if g.invalidTransitiveClosure: g.invalidTransitiveClosure = false - transitiveClosure(g.deps, g.modules.len) + transitiveClosure(g.deps, g.ifaces.len) # every module that *depends* on this file is also dirty: - for i in 0i32..<g.modules.len.int32: - let m = g.modules[i] - if m != nil and g.deps.contains(i.dependsOn(fileIdx.int)): - incl m.flags, sfDirty + for i in 0i32..<g.ifaces.len.int32: + if g.deps.contains(i.dependsOn(fileIdx.int)): + g.markDirty(FileIndex(i)) -proc isDirty*(g: ModuleGraph; m: PSym): bool = - result = g.suggestMode and sfDirty in m.flags +proc needsCompilation*(g: ModuleGraph): bool = + # every module that *depends* on this file is also dirty: + result = false + for i in 0i32..<g.ifaces.len.int32: + let m = g.ifaces[i].module + if m != nil: + if sfDirty in m.flags: + return true + +proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool = + result = false + let module = g.getModule(fileIdx) + if module != nil and g.isDirty(module): + return true + + for i in 0i32..<g.ifaces.len.int32: + let m = g.ifaces[i].module + if m != nil and g.isDirty(m) and g.deps.contains(fileIdx.int32.dependsOn(i)): + return true + +proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} = + result = s.ast[bodyPos] + if result == nil and g.config.symbolFiles in {readOnlySf, v2Sf, stressTest}: + result = loadProcBody(g.config, g.cache, g.packed, s) + s.ast[bodyPos] = result + assert result != nil + +proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex; + cachedModules: var seq[FileIndex]): PSym = + ## Returns 'nil' if the module needs to be recompiled. + if g.config.symbolFiles in {readOnlySf, v2Sf, stressTest}: + result = moduleFromRodFile(g.packed, g.config, g.cache, fileIdx, cachedModules) + else: + result = nil + +proc configComplete*(g: ModuleGraph) = + rememberStartupConfig(g.startupPackedConfig, g.config) + +proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, fromModule: PSym, ) = + let conf = graph.config + let isNimscript = conf.isDefined("nimscript") + if (not isNimscript) or hintProcessing in conf.cmdlineNotes: + let path = toFilenameOption(conf, fileIdx, conf.filenameOption) + let indent = ">".repeat(graph.importStack.len) + let fromModule2 = if fromModule != nil: $fromModule.name.s else: "(toplevel)" + let mode = if isNimscript: "(nims) " else: "" + rawMessage(conf, hintProcessing, "$#$# $#: $#: $#" % [mode, indent, fromModule2, moduleStatus, path]) + +proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym = + ## Returns a package symbol for yet to be defined module for fileIdx. + ## The package symbol is added to the graph if it doesn't exist. + let pkgSym = getPackage(graph.config, graph.cache, fileIdx) + # check if the package is already in the graph + result = graph.packageSyms.strTableGet(pkgSym.name) + if result == nil: + # the package isn't in the graph, so create and add it + result = pkgSym + graph.packageSyms.strTableAdd(pkgSym) + +func belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool = + ## Check if symbol belongs to the 'stdlib' package. + sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId + +proc fileSymbols*(graph: ModuleGraph, fileIdx: FileIndex): SuggestFileSymbolDatabase = + result = graph.suggestSymbols.getOrDefault(fileIdx, newSuggestFileSymbolDatabase(fileIdx, optIdeExceptionInlayHints in graph.config.globalOptions)) + doAssert(result.fileIndex == fileIdx) + +iterator suggestSymbolsIter*(g: ModuleGraph): SymInfoPair = + for xs in g.suggestSymbols.values: + for i in xs.lineInfo.low..xs.lineInfo.high: + yield xs.getSymInfoPair(i) + +iterator suggestErrorsIter*(g: ModuleGraph): Suggest = + for xs in g.suggestErrors.values: + for x in xs: + yield x |