diff options
author | Timothee Cour <timothee.cour2@gmail.com> | 2021-04-16 00:16:39 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-04-16 09:16:39 +0200 |
commit | 8161b02897a75c4b30593dbcc189cbd49d3832ea (patch) | |
tree | 399ca0d086a7e6917fae6c47c8dd899bd80f9102 | |
parent | 12783dbcf0075492a3d090e4dc3ead3628c18a3a (diff) | |
download | Nim-8161b02897a75c4b30593dbcc189cbd49d3832ea.tar.gz |
`import foo {.all.}` reboot (#17706)
38 files changed, 584 insertions, 76 deletions
diff --git a/changelog.md b/changelog.md index aeaeb598d..6dcfab9c6 100644 --- a/changelog.md +++ b/changelog.md @@ -307,6 +307,15 @@ - Added `iterable[T]` type class to match called iterators, which enables writing: `template fn(a: iterable)` instead of `template fn(a: untyped)` +- A new import syntax `import foo {.all.}` now allows to import all symbols (public or private) + from `foo`. It works in combination with all pre-existing import features. + This reduces or eliminates the need for workarounds such as using `include` (which has known issues) + when you need a private symbol for testing or making some internal APIs public just because + another internal module needs those. + It also helps mitigate the lack of cyclic imports in some cases. + +- Added a new module `std/importutils`, and an API `privateAccess`, which allows access to private fields + for an object type in the current scope. ## Compiler changes diff --git a/compiler/ast.nim b/compiler/ast.nim index 155af6375..4c835871f 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -605,6 +605,8 @@ type const routineKinds* = {skProc, skFunc, skMethod, skIterator, skConverter, skMacro, skTemplate} + ExportableSymKinds* = {skVar, skLet, skConst, skType, skEnumField, skStub, skAlias} + routineKinds + tfUnion* = tfNoSideEffect tfGcSafe* = tfThread tfObjHasKids* = tfEnumHasHoles @@ -691,7 +693,7 @@ type mInstantiationInfo, mGetTypeInfo, mGetTypeInfoV2, mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples, mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf, - mSymIsInstantiationOf, mNodeId + mSymIsInstantiationOf, mNodeId, mPrivateAccess # things that we can evaluate safely at compile time, even if not asked for it: @@ -841,6 +843,7 @@ type depthLevel*: int symbols*: TStrTable parent*: PScope + allowPrivateAccess*: seq[PSym] # # enable access to private fields PScope* = ref TScope @@ -1011,9 +1014,6 @@ const NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, tyProc, tyError} # TODO PtrLikeKinds*: TTypeKinds = {tyPointer, tyPtr} # for VM - ExportableSymKinds* = {skVar, skConst, skProc, skFunc, skMethod, skType, - skIterator, - skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias} PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfDotSetter, nfDotField, nfIsRef, nfIsPtr, nfPreventCg, nfLL, diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 4f63ce03a..1cd03ecfa 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -36,6 +36,7 @@ type bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position. #producedGenerics*: Table[GenericKey, SymId] exports*: seq[(LitId, int32)] + hidden*: seq[(LitId, int32)] reexports*: seq[(LitId, PackedItemId)] compilerProcs*: seq[(LitId, int32)] converters*, methods*, trmacros*, pureEnums*: seq[int32] @@ -177,6 +178,10 @@ proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) = m.imports.add toLitId(f, c, m) +proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + let nameId = getOrIncl(m.sh.strings, s.name.s) + m.hidden.add((nameId, s.itemId.item)) + proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) = let nameId = getOrIncl(m.sh.strings, s.name.s) m.exports.add((nameId, s.itemId.item)) @@ -524,7 +529,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef loadTabSection numbersSection, m.sh.numbers loadSeqSection exportsSection, m.exports - + loadSeqSection hiddenSection, m.hidden loadSeqSection reexportsSection, m.reexports loadSeqSection compilerProcsSection, m.compilerProcs @@ -589,7 +594,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac storeTabSection numbersSection, m.sh.numbers storeSeqSection exportsSection, m.exports - + storeSeqSection hiddenSection, m.hidden storeSeqSection reexportsSection, m.reexports storeSeqSection compilerProcsSection, m.compilerProcs @@ -655,7 +660,9 @@ type syms: seq[PSym] # indexed by itemId types: seq[PType] module*: PSym # the one true module symbol. - iface: Table[PIdent, seq[PackedItemId]] # PackedItemId so that it works with reexported symbols too + iface, ifaceHidden: Table[PIdent, seq[PackedItemId]] + # PackedItemId so that it works with reexported symbols too + # ifaceHidden includes private symbols PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex @@ -882,12 +889,22 @@ proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex; m: var LoadedModule) = m.iface = initTable[PIdent, seq[PackedItemId]]() - for e in m.fromDisk.exports: + m.ifaceHidden = initTable[PIdent, seq[PackedItemId]]() + template impl(iface, e) = let nameLit = e[0] - m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(PackedItemId(module: LitId(0), item: e[1])) - for re in m.fromDisk.reexports: - let nameLit = re[0] - m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(re[1]) + let e2 = + when e[1] is PackedItemId: e[1] + else: PackedItemId(module: LitId(0), item: e[1]) + iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(e2) + + for e in m.fromDisk.exports: + m.iface.impl(e) + m.ifaceHidden.impl(e) + for e in m.fromDisk.reexports: + m.iface.impl(e) + m.ifaceHidden.impl(e) + for e in m.fromDisk.hidden: + m.ifaceHidden.impl(e) let filename = AbsoluteFile toFullPath(conf, fileIdx) # We cannot call ``newSym`` here, because we have to circumvent the ID @@ -1053,16 +1070,21 @@ type values: seq[PackedItemId] i, module: int +template interfSelect(a: LoadedModule, importHidden: bool): auto = + var ret = a.iface.addr + if importHidden: ret = a.ifaceHidden.addr + ret[] + proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache; g: var PackedModuleGraph; module: FileIndex; - name: PIdent): PSym = + name: PIdent, importHidden: bool): PSym = it.decoder = PackedDecoder( lastModule: int32(-1), lastLit: LitId(0), lastFile: FileIndex(-1), config: config, cache: cache) - it.values = g[int module].iface.getOrDefault(name) + it.values = g[int module].interfSelect(importHidden).getOrDefault(name) it.i = 0 it.module = int(module) if it.i < it.values.len: @@ -1070,7 +1092,7 @@ proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache; inc it.i proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: FileIndex): PSym = + g: var PackedModuleGraph; module: FileIndex, importHidden: bool): PSym = it.decoder = PackedDecoder( lastModule: int32(-1), lastLit: LitId(0), @@ -1079,7 +1101,7 @@ proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache; cache: cache) it.values = @[] it.module = int(module) - for v in g[int module].iface.values: + for v in g[int module].interfSelect(importHidden).values: it.values.add v it.i = 0 if it.i < it.values.len: @@ -1093,9 +1115,9 @@ proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym = iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache; g: var PackedModuleGraph; module: FileIndex; - name: PIdent): PSym = + name: PIdent, importHidden: bool): PSym = setupDecoder() - let values = g[int module].iface.getOrDefault(name) + let values = g[int module].interfSelect(importHidden).getOrDefault(name) for pid in values: let s = loadSym(decoder, g, int(module), pid) assert s != nil @@ -1103,9 +1125,9 @@ iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache; proc interfaceSymbol*(config: ConfigRef, cache: IdentCache; g: var PackedModuleGraph; module: FileIndex; - name: PIdent): PSym = + name: PIdent, importHidden: bool): PSym = setupDecoder() - let values = g[int module].iface.getOrDefault(name) + let values = g[int module].interfSelect(importHidden).getOrDefault(name) result = loadSym(decoder, g, int(module), values[0]) proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator = @@ -1140,6 +1162,10 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = echo " ", m.sh.strings[ex[0]] # reexports*: seq[(LitId, PackedItemId)] + echo "hidden: " & $m.hidden.len + for ex in m.hidden: + echo " ", m.sh.strings[ex[0]], " local ID: ", ex[1] + echo "all symbols" for i in 0..high(m.sh.syms): if m.sh.syms[i].name != LitId(0): diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index fbe1cad33..47362da0b 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -18,6 +18,7 @@ type depsSection numbersSection exportsSection + hiddenSection reexportsSection compilerProcsSection trmacrosSection diff --git a/compiler/importer.nim b/compiler/importer.nim index f8ee0d483..692cdb604 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -12,7 +12,7 @@ import intsets, ast, astalgo, msgs, options, idents, lookups, semdata, modulepaths, sigmatch, lineinfos, sets, - modulegraphs + modulegraphs, wordrecg proc readExceptSet*(c: PContext, n: PNode): IntSet = assert n.kind in {nkImportExceptStmt, nkExportExceptStmt} @@ -107,7 +107,25 @@ proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) = if s.owner != origin: c.exportIndirections.incl((origin.id, s.id)) +proc splitPragmas(c: PContext, n: PNode): (PNode, seq[TSpecialWord]) = + template bail = globalError(c.config, n.info, "invalid pragma") + if n.kind == nkPragmaExpr: + if n.len == 2 and n[1].kind == nkPragma: + result[0] = n[0] + for ni in n[1]: + if ni.kind == nkIdent: result[1].add whichKeyword(ni.ident) + else: bail() + else: bail() + else: + result[0] = n + if result[0].safeLen > 0: + (result[0][^1], result[1]) = splitPragmas(c, result[0][^1]) + proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) = + let (n, kws) = splitPragmas(c, n) + if kws.len > 0: + globalError(c.config, n.info, "unexpected pragma") + let ident = lookups.considerQuotedIdent(c, n) let s = someSym(c.graph, fromMod, ident) if s == nil: @@ -204,18 +222,43 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; im for i in 0..n.safeLen-1: importForwarded(c, n[i], exceptSet, fromMod, importSet) -proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym = +proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool): PSym = result = realModule c.unusedImports.add((realModule, n.info)) + template createModuleAliasImpl(ident): untyped = + createModuleAlias(realModule, nextSymId c.idgen, ident, realModule.info, c.config.options) if n.kind != nkImportAs: discard elif n.len != 2 or n[1].kind != nkIdent: localError(c.config, n.info, "module alias must be an identifier") elif n[1].ident.id != realModule.name.id: # some misguided guy will write 'import abc.foo as foo' ... - result = createModuleAlias(realModule, nextSymId c.idgen, n[1].ident, realModule.info, - c.config.options) + result = createModuleAliasImpl(n[1].ident) + if importHidden: + if result == realModule: # avoids modifying `realModule`, see D20201209T194412. + result = createModuleAliasImpl(realModule.name) + result.options.incl optImportHidden + +proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] = + var ret: typeof(result) + proc processPragma(n2: PNode): PNode = + let (result2, kws) = splitPragmas(c, n2) + result = result2 + for ai in kws: + case ai + of wImportHidden: ret.importHidden = true + else: globalError(c.config, n.info, "invalid pragma, expected: " & ${wImportHidden}) + + if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as": + ret.node = newNodeI(nkImportAs, n.info) + ret.node.add n[1].processPragma + ret.node.add n[2] + else: + ret.node = n.processPragma + return ret -proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym = +proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym = + let transf = transformImportAs(c, n) + n = transf.node let f = checkModuleName(c.config, n) if f != InvalidFileIdx: addImportFileDep(c, f) @@ -232,7 +275,7 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym = c.recursiveDep = err discard pushOptionEntry(c) - result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f)) + result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f), transf.importHidden) popOptionEntry(c) #echo "set back to ", L @@ -252,16 +295,8 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym = importStmtResult.add newSymNode(result, n.info) #newStrNode(toFullPath(c.config, f), n.info) -proc transformImportAs(c: PContext; n: PNode): PNode = - if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as": - result = newNodeI(nkImportAs, n.info) - result.add n[1] - result.add n[2] - else: - result = n - proc impMod(c: PContext; it: PNode; importStmtResult: PNode) = - let it = transformImportAs(c, it) + var it = it let m = myImportModule(c, it, importStmtResult) if m != nil: # ``addDecl`` needs to be done before ``importAllSymbols``! @@ -296,7 +331,6 @@ proc evalImport*(c: PContext, n: PNode): PNode = proc evalFrom*(c: PContext, n: PNode): PNode = result = newNodeI(nkImportStmt, n.info) checkMinSonsLen(n, 2, c.config) - n[0] = transformImportAs(c, n[0]) var m = myImportModule(c, n[0], result) if m != nil: n[0] = newSymNode(m) @@ -311,7 +345,6 @@ proc evalFrom*(c: PContext, n: PNode): PNode = proc evalImportExcept*(c: PContext, n: PNode): PNode = result = newNodeI(nkImportStmt, n.info) checkMinSonsLen(n, 2, c.config) - n[0] = transformImportAs(c, n[0]) var m = myImportModule(c, n[0], result) if m != nil: n[0] = newSymNode(m) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 15a22c778..7ceadfb96 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -75,7 +75,7 @@ proc closeScope*(c: PContext) = ensureNoMissingOrUnusedSymbols(c, c.currentScope) rawCloseScope(c) -iterator allScopes(scope: PScope): PScope = +iterator allScopes*(scope: PScope): PScope = var current = scope while current != nil: yield current @@ -311,11 +311,17 @@ proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) = if conflict != nil: wrongRedefinition(c, sym.info, sym.name.s, conflict.info) -proc addInterfaceDeclAux(c: PContext, sym: PSym) = - if sfExported in sym.flags: +from ic / ic import addHidden + +proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) = + if sfExported in sym.flags or forceExport: # add to interface: if c.module != nil: exportSym(c, sym) else: internalError(c.config, sym.info, "addInterfaceDeclAux") + elif sym.kind in ExportableSymKinds and c.module != nil and isTopLevelInsideDeclaration(c, sym): + strTableAdd(semtabAll(c.graph, c.module), sym) + if c.config.symbolFiles != disabledSf: + addHidden(c.encoder, c.packedRepr, sym) proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) = addDeclAt(c, scope, sym) diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index e91bdf272..38d2345c8 100644 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -50,6 +50,7 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType = result = g.sysTypes[kind] if result == nil: case kind + of tyVoid: result = sysTypeFromName("void") of tyInt: result = sysTypeFromName("int") of tyInt8: result = sysTypeFromName("int8") of tyInt16: result = sysTypeFromName("int16") diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 02c745018..9df66269b 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -29,6 +29,7 @@ type patterns*: seq[LazySym] pureEnums*: seq[LazySym] interf: TStrTable + interfHidden: TStrTable uniqueName*: Rope Operators* = object @@ -160,9 +161,25 @@ proc toBase64a(s: cstring, len: int): string = result.add cb64[a shr 2] result.add cb64[(a and 3) shl 4] -template semtab*(m: PSym; g: ModuleGraph): TStrTable = +template interfSelect(iface: Iface, importHidden: bool): TStrTable = + var ret = iface.interf.addr # without intermediate ptr, it creates a copy and compiler becomes 15x slower! + if importHidden: ret = iface.interfHidden.addr + ret[] + +template semtab(g: ModuleGraph, m: PSym): TStrTable = g.ifaces[m.position].interf +template semtabAll*(g: ModuleGraph, m: PSym): TStrTable = + g.ifaces[m.position].interfHidden + +proc initStrTables*(g: ModuleGraph, m: PSym) = + initStrTable(semtab(g, m)) + initStrTable(semtabAll(g, m)) + +proc strTableAdds*(g: ModuleGraph, m: PSym, s: PSym) = + strTableAdd(semtab(g, m), s) + strTableAdd(semtabAll(g, m), s) + proc isCachedModule(g: ModuleGraph; module: int): bool {.inline.} = result = module < g.packed.len and g.packed[module].status == loaded @@ -187,39 +204,43 @@ type modIndex: int ti: TIdentIter rodIt: RodIter + importHidden: bool proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym = assert m.kind == skModule mi.modIndex = m.position mi.fromRod = isCachedModule(g, mi.modIndex) + mi.importHidden = optImportHidden in m.options if mi.fromRod: - result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name) + result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name, mi.importHidden) else: - result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interf, name) + result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name) proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym = if mi.fromRod: result = nextRodIter(mi.rodIt, g.packed) else: - result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interf) + result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden)) iterator allSyms*(g: ModuleGraph; m: PSym): PSym = + let importHidden = optImportHidden in m.options if isCachedModule(g, m): var rodIt: RodIter - var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position) + var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position, importHidden) while r != nil: yield r r = nextRodIter(rodIt, g.packed) else: - for s in g.ifaces[m.position].interf.data: + for s in g.ifaces[m.position].interfSelect(importHidden).data: if s != nil: yield s proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym = + let importHidden = optImportHidden in m.options if isCachedModule(g, m): - result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name) + result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden) else: - result = strTableGet(g.ifaces[m.position].interf, name) + result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name) proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym = result = someSym(g, g.systemModule, name) @@ -343,26 +364,23 @@ proc hash*(u: SigHash): Hash = proc hash*(x: FileIndex): Hash {.borrow.} +template getPContext(): untyped = + when c is PContext: c + else: c.c + when defined(nimfind): template onUse*(info: TLineInfo; s: PSym) = - when compiles(c.c.graph): - if c.c.graph.onUsage != nil: c.c.graph.onUsage(c.c.graph, s, info) - else: - if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info) + let c = getPContext() + if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info) template onDef*(info: TLineInfo; s: PSym) = - when compiles(c.c.graph): - if c.c.graph.onDefinition != nil: c.c.graph.onDefinition(c.c.graph, s, info) - else: - if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info) + let c = getPContext() + if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info) template onDefResolveForward*(info: TLineInfo; s: PSym) = - when compiles(c.c.graph): - if c.c.graph.onDefinitionResolveForward != nil: - c.c.graph.onDefinitionResolveForward(c.c.graph, s, info) - else: - if c.graph.onDefinitionResolveForward != nil: - c.graph.onDefinitionResolveForward(c.graph, s, info) + let c = getPContext() + if c.graph.onDefinitionResolveForward != nil: + c.graph.onDefinitionResolveForward(c.graph, s, info) else: template onUse*(info: TLineInfo; s: PSym) = discard @@ -392,7 +410,7 @@ proc registerModule*(g: ModuleGraph; m: PSym) = g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[], uniqueName: rope(uniqueModuleName(g.config, FileIndex(m.position)))) - initStrTable(g.ifaces[m.position].interf) + initStrTables(g, m) proc registerModuleById*(g: ModuleGraph; m: FileIndex) = registerModule(g, g.packed[int m].module) diff --git a/compiler/modules.nim b/compiler/modules.nim index 7d7a2b6f7..27b93e8df 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -115,7 +115,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P elif graph.isDirty(result): result.flags.excl sfDirty # reset module fields: - initStrTable(result.semtab(graph)) + initStrTables(graph, result) result.ast = nil processModuleAux() graph.markClientsDirty(fileIdx) diff --git a/compiler/nimeval.nim b/compiler/nimeval.nim index 9be595cce..577c7b586 100644 --- a/compiler/nimeval.nim +++ b/compiler/nimeval.nim @@ -67,7 +67,7 @@ proc evalScript*(i: Interpreter; scriptStream: PLLStream = nil) = ## This can also be used to *reload* the script. assert i != nil assert i.mainModule != nil, "no main module selected" - initStrTable(i.mainModule.semtab(i.graph)) + initStrTables(i.graph, i.mainModule) i.mainModule.ast = nil let s = if scriptStream != nil: scriptStream diff --git a/compiler/options.nim b/compiler/options.nim index 9eeffb4fb..9ba4d62e4 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -41,7 +41,7 @@ type # please make sure we have under 32 options optMemTracker, optSinkInference # 'sink T' inference optCursorInference - + optImportHidden TOptions* = set[TOption] TGlobalOption* = enum # **keep binary compatible** diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 6b0dcb0e2..9e7258a69 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -361,12 +361,12 @@ proc addPattern*(c: PContext, p: LazySym) = addTrmacro(c.encoder, c.packedRepr, p.sym) proc exportSym*(c: PContext; s: PSym) = - strTableAdd(c.module.semtab(c.graph), s) + strTableAdds(c.graph, c.module, s) if c.config.symbolFiles != disabledSf: addExported(c.encoder, c.packedRepr, s) proc reexportSym*(c: PContext; s: PSym) = - strTableAdd(c.module.semtab(c.graph), s) + strTableAdds(c.graph, c.module, s) if c.config.symbolFiles != disabledSf: addReexport(c.encoder, c.packedRepr, s) @@ -536,6 +536,10 @@ proc checkMinSonsLen*(n: PNode, length: int; conf: ConfigRef) = proc isTopLevel*(c: PContext): bool {.inline.} = result = c.currentScope.depthLevel <= 2 +proc isTopLevelInsideDeclaration*(c: PContext, sym: PSym): bool {.inline.} = + # for routeKinds the scope isn't closed yet: + c.currentScope.depthLevel <= 2 + ord(sym.kind in routineKinds) + proc pushCaseContext*(c: PContext, caseNode: PNode) = c.p.caseContext.add((caseNode, 0)) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index c80e689a5..d8be85273 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -577,5 +577,10 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, if n[1].typ.skipTypes(abstractInst).kind in {tyUInt..tyUInt64}: n[0].sym.magic = mSubU result = n + of mPrivateAccess: + let sym = n[1].typ[0].sym + assert sym != nil + c.currentScope.allowPrivateAccess.add sym + result = newNodeIT(nkEmpty, n.info, getSysType(c.graph, n.info, tyVoid)) else: result = n diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 2c257bd3d..bedc5a7e5 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1067,6 +1067,7 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) = elif typsym.kind == skType and sfForward in typsym.flags: s = typsym addInterfaceDecl(c, s) + # PRTEMP no onDef here? else: localError(c.config, name.info, typsym.name.s & " is not a type that can be forwarded") s = typsym diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 34a90a3f2..4763f0fd3 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -138,9 +138,10 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = identToReplace[] = symNode if e.position == 0: hasNull = true if result.sym != nil and sfExported in result.sym.flags: - incl(e.flags, sfUsed) - incl(e.flags, sfExported) - if not isPure: exportSym(c, e) + incl(e.flags, {sfUsed, sfExported}) + if result.sym != nil and not isPure: + addInterfaceDeclAux(c, e, forceExport = sfExported in result.sym.flags) + result.n.add symNode styleCheckDef(c.config, e) onDef(e.info, e) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index cad776015..7a2c0d964 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -253,10 +253,15 @@ proc filterSymNoOpr(s: PSym; prefix: PNode; res: var PrefixMatch): bool {.inline proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} = let fmoduleId = getModule(f).id result = sfExported in f.flags or fmoduleId == c.module.id - for module in c.friendModules: - if fmoduleId == module.id: - result = true - break + + if not result: + for module in c.friendModules: + if fmoduleId == module.id: return true + if f.kind == skField: + let symObj = f.owner + for scope in allScopes(c.currentScope): + for sym in scope.allowPrivateAccess: + if symObj.id == sym.id: return true proc getQuality(s: PSym): range[0..100] = result = 100 diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 6a68e2c70..827efcfa0 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -109,7 +109,7 @@ type wStdIn = "stdin", wStdOut = "stdout", wStdErr = "stderr", wInOut = "inout", wByCopy = "bycopy", wByRef = "byref", wOneWay = "oneway", - wBitsize = "bitsize" + wBitsize = "bitsize", wImportHidden = "all", TSpecialWords* = set[TSpecialWord] diff --git a/lib/std/importutils.nim b/lib/std/importutils.nim new file mode 100644 index 000000000..4efe0e140 --- /dev/null +++ b/lib/std/importutils.nim @@ -0,0 +1,34 @@ +##[ +Utilities related to import and symbol resolution. + +Experimental API, subject to change. +]## + +#[ +Possible future APIs: +* module symbols (https://github.com/nim-lang/Nim/pull/9560) +* whichModule (subsumes canImport / moduleExists) (https://github.com/timotheecour/Nim/issues/376) +* getCurrentPkgDir (https://github.com/nim-lang/Nim/pull/10530) +* import from a computed string + related APIs (https://github.com/nim-lang/Nim/pull/10527) +]# + +when defined(nimImportutilsExample): + type Foo = object + x1: int # private + proc initFoo*(): auto = Foo() + +proc privateAccess*(t: typedesc) {.magic: "PrivateAccess".} = + ## Enables access to private fields of `t` in current scope. + runnableExamples("-d:nimImportutilsExample"): + # here we're importing a module containing: + # type Foo = object + # x1: int # private + # proc initFoo*(): auto = Foo() + var a = initFoo() + block: + assert not compiles(a.x1) + privateAccess(a.type) + a.x1 = 1 # accessible in this scope + block: + assert a.x1 == 1 # still in scope + assert not compiles(a.x1) diff --git a/testament/categories.nim b/testament/categories.nim index ee5c46519..18508d70c 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -500,6 +500,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string) const tempExt = "_temp.nim" for it in walkDirRec(testsDir / "ic"): + # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test if isTestFile(it) and not it.endsWith(tempExt): let nimcache = nimcacheDir(it, options, getTestSpecTarget()) removeDir(nimcache) diff --git a/tests/ic/mimports.nim b/tests/ic/mimports.nim new file mode 100644 index 000000000..50773e001 --- /dev/null +++ b/tests/ic/mimports.nim @@ -0,0 +1,9 @@ +from mimportsb {.all.} import fnb1, hfnb3 + +proc fn1*(): int = 1 +proc fn2*(): int = 2 +proc hfn3(): int = 3 +proc hfn4(): int = 4 +proc hfn5(): int = 5 + +export mimportsb.fnb2, hfnb3 diff --git a/tests/ic/mimportsb.nim b/tests/ic/mimportsb.nim new file mode 100644 index 000000000..3cae925c5 --- /dev/null +++ b/tests/ic/mimportsb.nim @@ -0,0 +1,4 @@ +proc fnb1*(): int = 1 +proc fnb2*(): int = 2 +proc hfnb3(): int = 3 +proc hfnb4(): int = 4 diff --git a/tests/ic/timports.nim b/tests/ic/timports.nim new file mode 100644 index 000000000..518a689f5 --- /dev/null +++ b/tests/ic/timports.nim @@ -0,0 +1,29 @@ +import mimports +doAssert fn1() == 1 +doAssert not declared(hfn3) + +#!EDIT!# + +import mimports {.all.} +doAssert fn1() == 1 +doAssert declared(hfn3) +doAssert hfn3() == 3 +doAssert mimports.hfn4() == 4 + +# reexports +doAssert not declared(fnb1) +doAssert not declared(hfnb4) +doAssert fnb2() == 2 +doAssert hfnb3() == 3 + +#!EDIT!# + +from mimports {.all.} import hfn3 +doAssert not declared(fn1) +from mimports {.all.} as bar import fn1 +doAssert fn1() == 1 +doAssert hfn3() == 3 +doAssert not declared(hfn4) +doAssert declared(mimports.hfn4) +doAssert mimports.hfn4() == 4 +doAssert bar.hfn4() == 4 diff --git a/tests/importalls/m1.nim b/tests/importalls/m1.nim new file mode 100644 index 000000000..b6ccbf5da --- /dev/null +++ b/tests/importalls/m1.nim @@ -0,0 +1,70 @@ +import ./m2 +import ./m3 {.all.} as m3 +from ./m3 as m3Bis import nil + +doAssert m3h2 == 2 +export m3h2 + +export m3Bis.m3p1 + +const foo0* = 2 +const foo1 = bar1 + +const foo1Aux = 2 +export foo1Aux + +doAssert not declared(bar2) +doAssert not compiles(bar2) + +var foo2 = 2 +let foo3 = 2 + +type Foo4 = enum + kg1, kg2 + +type Foo4b {.pure.} = enum + foo4b1, foo4b2 + +type Foo5 = object + z1: string + z2: Foo4 + z3: int + z4*: int + +proc `z3`*(a: Foo5): auto = + a.z3 * 10 + +proc foo6(): auto = 2 +proc foo6b*(): auto = 2 +template foo7: untyped = 2 +macro foo8(): untyped = discard +template foo9(a: int) = discard + +block: + template foo10: untyped = 2 + type Foo11 = enum + kg1b, kg2b + proc foo12(): auto = 2 + +proc initFoo5*(z3: int): Foo5 = Foo5(z3: z3) + +func foo13(): auto = 2 +iterator foo14a(): int = discard +iterator foo14b*(): int = discard +iterator foo14c(): int {.closure.} = discard +iterator foo14d(): int {.inline.} = discard + +# fwd declare +proc foo15(): int +proc foo15(): int = 2 + +proc foo16*(): int +proc foo16(): int = 2 + +proc foo17*(): int +proc foo17*(): int = 2 + +# other +type A1 = distinct int +type A2 = distinct int +converter foo18(x: A1): A2 = discard diff --git a/tests/importalls/m2.nim b/tests/importalls/m2.nim new file mode 100644 index 000000000..186650f68 --- /dev/null +++ b/tests/importalls/m2.nim @@ -0,0 +1,3 @@ +const bar1* = 2 +const bar2 = 2 +const bar3 = 3 diff --git a/tests/importalls/m3.nim b/tests/importalls/m3.nim new file mode 100644 index 000000000..caeae2a01 --- /dev/null +++ b/tests/importalls/m3.nim @@ -0,0 +1,5 @@ +# xxx use below naming convention in other test files: p: public, h: hidden +const m3p1* = 2 +const m3h2 = 2 +const m3h3 = 3 +const m3h4 = 4 diff --git a/tests/importalls/m4.nim b/tests/importalls/m4.nim new file mode 100644 index 000000000..b682b766a --- /dev/null +++ b/tests/importalls/m4.nim @@ -0,0 +1,10 @@ +{.warning[UnusedImport]: off.} # xxx bug: this shouldn't be needed since we have `export m3` +import ./m3 {.all.} +import ./m3 as m3b +export m3b +export m3h3 +export m3.m3h4 + +import ./m2 {.all.} as m2b +export m2b except bar3 + diff --git a/tests/importalls/mt0.nim b/tests/importalls/mt0.nim new file mode 100644 index 000000000..154d30ef9 --- /dev/null +++ b/tests/importalls/mt0.nim @@ -0,0 +1,9 @@ +import ./m1 as m +doAssert compiles(foo0) +doAssert not compiles(foo1) +doAssert foo6b() == 2 +doAssert m3h2 == 2 + +var f = initFoo5(z3=3) +doAssert f.z3 == 30 +doAssert z3(f) == 30 diff --git a/tests/importalls/mt1.nim b/tests/importalls/mt1.nim new file mode 100644 index 000000000..87c4eff16 --- /dev/null +++ b/tests/importalls/mt1.nim @@ -0,0 +1,23 @@ +import ./m1 {.all.} as m +doAssert foo1 == 2 +doAssert m.foo1 == 2 + +doAssert m.m3h2 == 2 +doAssert m3h2 == 2 +doAssert m.foo1Aux == 2 +doAssert m.m3p1 == 2 + +## field access +import std/importutils +privateAccess(Foo5) +# var x = Foo5(z1: "foo", z2: m.kg1) +# doAssert x.z1 == "foo" + +var f0: Foo5 +f0.z3 = 3 +doAssert f0.z3 == 3 +var f = initFoo5(z3=3) +doAssert f.z3 == 3 +doAssert z3(f) == 30 +doAssert m.z3(f) == 30 +doAssert not compiles(mt1.`z3`(f)) # z3 is an imported symbol diff --git a/tests/importalls/mt2.nim b/tests/importalls/mt2.nim new file mode 100644 index 000000000..52edbb69e --- /dev/null +++ b/tests/importalls/mt2.nim @@ -0,0 +1,104 @@ +from ./m1 {.all.} as r1 import foo1 +from ./m1 {.all.} as r2 import foo7 + +block: # different symbol kinds + doAssert foo1 == 2 + doAssert r1.foo1 == 2 + doAssert r1.foo2 == 2 + doAssert compiles(foo1) + doAssert compiles(r1.foo2) + doAssert not compiles(foo2) + doAssert not compiles(m3h2) + doAssert r1.foo3 == 2 + + block: # enum + var a: r1.Foo4 + let a1 = r1.kg1 + doAssert a1 == r1.Foo4.kg1 + type A = r1.Foo4 + doAssert a1 == A.kg1 + doAssert not compiles(kg1) + doAssert compiles(A.kg1) + var witness = false + for ai in r1.Foo4: + doAssert ai == a + doAssert ai == a1 + witness = true + break + doAssert witness + + block: # {.pure.} enum + var a: r1.Foo4b + doAssert not compiles(r1.foo4b1) # because pure + doAssert not compiles(foo4b1) + let a1 = r1.Foo4b.foo4b1 + doAssert a1 == a + type A = r1.Foo4b + doAssert a1 == A.foo4b1 + var witness = false + for ai in A: + doAssert ai == a + doAssert ai == a1 + witness = true + break + doAssert witness + + block: # object + doAssert compiles(r1.Foo5) + var a: r1.Foo5 + doAssert compiles(a.z4) + doAssert not compiles(a.z3) + + block: # remaining symbol kinds + doAssert r1.foo6() == 2 + doAssert r1.foo6b() == 2 + doAssert foo7() == 2 + doAssert r2.foo6b() == 2 + + r1.foo8() + r1.foo9(1) + doAssert r1.foo13() == 2 + for a in r1.foo14a(): discard + for a in r1.foo14b(): discard + for a in r1.foo14c(): discard + for a in r1.foo14d(): discard + doAssert r1.foo15() == 2 + doAssert r1.foo16() == 2 + doAssert r1.foo17() == 2 + doAssert compiles(r1.foo18) + doAssert declared(r1.foo18) + + block: # declarations at block scope should not be visible + doAssert declared(foo7) + doAssert declared(r1.foo6) + doAssert not declared(foo10) + doAssert not declared(foo6) + doAssert not declared(r1.Foo11) + doAssert not declared(r1.kg1b) + doAssert not declared(r1.foo12) + doAssert not compiles(r1.foo12()) + +## field access +import std/importutils +privateAccess(r1.Foo5) +var x = r1.Foo5(z1: "foo", z2: r1.kg1) +doAssert x.z1 == "foo" + +var f0: r1.Foo5 +f0.z3 = 3 +doAssert f0.z3 == 3 +var f = r1.initFoo5(z3=3) +doAssert f.z3 == 3 +doAssert r1.z3(f) == 30 + +import ./m1 as r3 +doAssert not declared(foo2) +doAssert not declared(r3.foo2) + +from ./m1 {.all.} as r4 import nil +doAssert not declared(foo2) +doAssert declared(r4.foo2) + +from ./m1 {.all.} import nil +doAssert not declared(foo2) +doAssert declared(m1.foo2) diff --git a/tests/importalls/mt3.nim b/tests/importalls/mt3.nim new file mode 100644 index 000000000..2bd7fd79d --- /dev/null +++ b/tests/importalls/mt3.nim @@ -0,0 +1,12 @@ +import ./m1 {.all.} + # D20201209T194412:here keep this as is, without `as`, so that mt8.nim test keeps + # checking that the original module symbol for `m1` isn't modified and that + # only the alias in `createModuleAlias` is affected. +doAssert declared(m1.foo1) +doAssert foo1 == 2 + + +doAssert m1.foo1 == 2 + +doAssert not compiles(mt3.foo0) # foo0 is an imported symbol +doAssert not compiles(mt3.foo1) # ditto diff --git a/tests/importalls/mt4.nim b/tests/importalls/mt4.nim new file mode 100644 index 000000000..cd7d32aad --- /dev/null +++ b/tests/importalls/mt4.nim @@ -0,0 +1,4 @@ +import ./m1 {.all.} except foo1 +doAssert foo2 == 2 +doAssert declared(foo2) +doAssert not compiles(foo1) diff --git a/tests/importalls/mt4b.nim b/tests/importalls/mt4b.nim new file mode 100644 index 000000000..4578bc54c --- /dev/null +++ b/tests/importalls/mt4b.nim @@ -0,0 +1,3 @@ +from ./m1 {.all.} as m2 import nil +doAssert not compiles(foo1) +doAssert m2.foo1 == 2 diff --git a/tests/importalls/mt5.nim b/tests/importalls/mt5.nim new file mode 100644 index 000000000..5c5785af6 --- /dev/null +++ b/tests/importalls/mt5.nim @@ -0,0 +1,9 @@ +import ./m1 {.all.} as m2 except foo1 +doAssert foo2 == 2 +doAssert not compiles(foo1) +doAssert m2.foo1 == 2 +doAssert compiles(m2.foo1) + +from system {.all.} as s import ThisIsSystem +doAssert ThisIsSystem +doAssert s.ThisIsSystem diff --git a/tests/importalls/mt6.nim b/tests/importalls/mt6.nim new file mode 100644 index 000000000..c6d8b2193 --- /dev/null +++ b/tests/importalls/mt6.nim @@ -0,0 +1,13 @@ +import ./m1 {.all.} as m2 +doAssert compiles(foo1) +doAssert compiles(m2.foo1) +doAssert declared(foo1) +doAssert declared(m2.foo0) # public: works fine + +doAssert m2.foo1 == 2 +doAssert declared(m2.foo1) +doAssert not declared(m2.nonexistant) + +# also tests the quoted `""` import +import "."/"m1" {.all.} as m1b +doAssert compiles(m1b.foo1) diff --git a/tests/importalls/mt7.nim b/tests/importalls/mt7.nim new file mode 100644 index 000000000..45a664610 --- /dev/null +++ b/tests/importalls/mt7.nim @@ -0,0 +1,14 @@ +include ./m1 +doAssert compiles(foo1) +doAssert compiles(mt7.foo1) +doAssert declared(foo1) +doAssert declared(mt7.foo1) +doAssert declared(mt7.foo0) + +var f0: Foo5 +f0.z3 = 3 +doAssert f0.z3 == 3 +var f = initFoo5(z3=3) +doAssert f.z3 == 3 +doAssert mt7.z3(f) == 30 +doAssert z3(f) == 30 diff --git a/tests/importalls/mt8.nim b/tests/importalls/mt8.nim new file mode 100644 index 000000000..f484e7b01 --- /dev/null +++ b/tests/importalls/mt8.nim @@ -0,0 +1,23 @@ +#[ +test multiple imports +]# + +{.warning[UnusedImport]: off.} +import ./m1, m2 {.all.}, ./m3 {.all.} + # make sure this keeps using `import ./m1` without as. + +# m1 is regularly imported +doAssert declared(m1.foo0) +doAssert declared(foo0) + +doAssert not declared(m1.foo1) + # if we didn't call `createModuleAlias` even for `import f1 {.all.}`, + # this would fail, see D20201209T194412. + +# m2 +doAssert declared(m2.bar2) +doAssert declared(bar2) + +# m3 +doAssert declared(m3.m3h2) +doAssert declared(m3h2) diff --git a/tests/importalls/mt9.nim b/tests/importalls/mt9.nim new file mode 100644 index 000000000..42d48ecbd --- /dev/null +++ b/tests/importalls/mt9.nim @@ -0,0 +1,12 @@ +# tests re-export of a module with import {.all.} + +import ./m4 + +doAssert m3p1 == 2 +doAssert not declared(m3h2) +doAssert m3h3 == 3 +doAssert m3h4 == 4 + +doAssert bar1 == 2 +doAssert bar2 == 2 +doAssert not declared(bar3) diff --git a/tests/importalls/tmain2.nim b/tests/importalls/tmain2.nim new file mode 100644 index 000000000..596f0f135 --- /dev/null +++ b/tests/importalls/tmain2.nim @@ -0,0 +1,7 @@ +discard """ +joinable: false # for clarity, but not necessary +""" + +{.warning[UnusedImport]: off.} +# only import `mt*.nim` here; these depend on `m*.nim` +import "."/[mt0,mt1,mt2,mt3,mt4,mt4b,mt5,mt6,mt7,mt8,mt9] |