diff options
Diffstat (limited to 'compiler')
65 files changed, 2697 insertions, 860 deletions
diff --git a/compiler/ast.nim b/compiler/ast.nim index d8939fc60..8f4acfc3b 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -317,6 +317,10 @@ type TTypeKind* = enum # order is important! # Don't forget to change hti.nim if you make a change here # XXX put this into an include file to avoid this issue! + # several types are no longer used (guess which), but a + # spot in the sequence is kept for backwards compatibility + # (apparently something with bootstrapping) + # if you need to add a type, they can apparently be reused tyNone, tyBool, tyChar, tyEmpty, tyArrayConstr, tyNil, tyExpr, tyStmt, tyTypeDesc, tyGenericInvocation, # ``T[a, b]`` for types to invoke @@ -345,9 +349,9 @@ type tyInt, tyInt8, tyInt16, tyInt32, tyInt64, # signed integers tyFloat, tyFloat32, tyFloat64, tyFloat128, tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64, - tyBigNum, - tyConst, tyMutable, tyVarargs, - tyIter, # unused + tyUnused0, tyUnused1, tyUnused2, + tyVarargs, + tyUnused, tyProxy # used as errornous type (for idetools) tyBuiltInTypeClass #\ @@ -1046,8 +1050,6 @@ proc newSym*(symKind: TSymKind, name: PIdent, owner: PSym, var emptyNode* = newNode(nkEmpty) # There is a single empty node that is shared! Do not overwrite it! -var anyGlobal* = newSym(skVar, getIdent("*"), nil, unknownLineInfo()) - proc isMetaType*(t: PType): bool = return t.kind in tyMetaTypes or (t.kind == tyStatic and t.n == nil) or @@ -1579,14 +1581,6 @@ proc skipStmtList*(n: PNode): PNode = else: result = n -proc createMagic*(name: string, m: TMagic): PSym = - result = newSym(skProc, getIdent(name), nil, unknownLineInfo()) - result.magic = m - -let - opNot* = createMagic("not", mNot) - opContains* = createMagic("contains", mInSet) - when false: proc containsNil*(n: PNode): bool = # only for debugging diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index be49ddc87..2761f888b 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -355,6 +355,14 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src)) else: internalError("genAssignment: " & $ty.kind) + if optMemTracker in p.options and dest.s in {OnHeap, OnUnknown}: + #writeStackTrace() + #echo p.currLineInfo, " requesting" + linefmt(p, cpsStmts, "#memTrackerWrite((void*)$1, $2, $3, $4);$n", + addrLoc(dest), rope getSize(dest.t), + makeCString(p.currLineInfo.toFullPath), + rope p.currLineInfo.safeLineNm) + proc genDeepCopy(p: BProc; dest, src: TLoc) = var ty = skipTypes(dest.t, abstractVarRange) case ty.kind @@ -592,9 +600,9 @@ proc genEqProc(p: BProc, e: PNode, d: var TLoc) = proc genIsNil(p: BProc, e: PNode, d: var TLoc) = let t = skipTypes(e.sons[1].typ, abstractRange) if t.kind == tyProc and t.callConv == ccClosure: - unaryExpr(p, e, d, "$1.ClPrc == 0") + unaryExpr(p, e, d, "($1.ClPrc == 0)") else: - unaryExpr(p, e, d, "$1 == 0") + unaryExpr(p, e, d, "($1 == 0)") proc unaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) = const @@ -1946,6 +1954,7 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) = d.s = OnStatic proc expr(p: BProc, n: PNode, d: var TLoc) = + p.currLineInfo = n.info case n.kind of nkSym: var sym = n.sym diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index eac734b3d..60ee0eaee 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -122,7 +122,7 @@ proc mapType(typ: PType): TCTypeKind = of tyOpenArray, tyArrayConstr, tyArray, tyVarargs: result = ctArray of tyObject, tyTuple: result = ctStruct of tyGenericBody, tyGenericInst, tyGenericParam, tyDistinct, tyOrdinal, - tyConst, tyMutable, tyIter, tyTypeDesc: + tyTypeDesc: result = mapType(lastSon(typ)) of tyEnum: if firstOrd(typ) < 0: @@ -206,6 +206,10 @@ proc cacheGetType(tab: TIdTable, key: PType): Rope = # linear search is not necessary anymore: result = Rope(idTableGet(tab, key)) +proc addAbiCheck(m: BModule, t: PType, name: Rope) = + if isDefined("checkabi"): + addf(m.s[cfsTypeInfo], "NIM_CHECK_SIZE($1, $2);$n", [name, rope(getSize(t))]) + proc getTempName(m: BModule): Rope = result = m.tmpBase & rope(m.labels) inc m.labels @@ -267,6 +271,11 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = result = getSimpleTypeDesc(m, lastSon typ) else: result = nil + if result != nil and typ.isImportedType(): + if cacheGetType(m.typeCache, typ) == nil: + idTablePut(m.typeCache, typ, result) + addAbiCheck(m, typ, result) + proc pushType(m: BModule, typ: PType) = add(m.typeStack, typ) @@ -656,6 +665,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = let foo = getTypeDescAux(m, t.sons[1], check) addf(m.s[cfsTypes], "typedef $1 $2[$3];$n", [foo, result, rope(n)]) + else: addAbiCheck(m, t, result) of tyObject, tyTuple: if isImportedCppType(t) and typ.kind == tyGenericInst: # for instantiated templates we do not go through the type cache as the @@ -701,7 +711,9 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = idTablePut(m.typeCache, t, result) # always call for sideeffects: let recdesc = if t.kind != tyTuple: getRecordDesc(m, t, result, check) else: getTupleDesc(m, t, result, check) - if not isImportedType(t): add(m.s[cfsTypes], recdesc) + if not isImportedType(t): + add(m.s[cfsTypes], recdesc) + elif tfIncompleteStruct notin t.flags: addAbiCheck(m, t, result) of tySet: result = getTypeName(t.lastSon) & "Set" idTablePut(m.typeCache, t, result) @@ -711,8 +723,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = of 1, 2, 4, 8: addf(m.s[cfsTypes], "typedef NU$2 $1;$n", [result, rope(s*8)]) else: addf(m.s[cfsTypes], "typedef NU8 $1[$2];$n", [result, rope(getSize(t))]) - of tyGenericInst, tyDistinct, tyOrdinal, tyConst, tyMutable, - tyIter, tyTypeDesc: + of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc: result = getTypeDescAux(m, lastSon(t), check) else: internalError("getTypeDescAux(" & $t.kind & ')') diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim index ecd98a2bf..2216cb4fd 100644 --- a/compiler/ccgutils.nim +++ b/compiler/ccgutils.nim @@ -93,7 +93,7 @@ proc getUniqueType*(key: PType): PType = # produced instead of ``NI``. result = key of tyEmpty, tyNil, tyExpr, tyStmt, tyPointer, tyString, - tyCString, tyNone, tyBigNum, tyVoid: + tyCString, tyNone, tyVoid: result = gCanonicalTypes[k] if result == nil: gCanonicalTypes[k] = key @@ -106,7 +106,7 @@ proc getUniqueType*(key: PType): PType = of tyDistinct: if key.deepCopy != nil: result = key else: result = getUniqueType(lastSon(key)) - of tyGenericInst, tyOrdinal, tyMutable, tyConst, tyIter, tyStatic: + of tyGenericInst, tyOrdinal, tyStatic: result = getUniqueType(lastSon(key)) #let obj = lastSon(key) #if obj.sym != nil and obj.sym.name.s == "TOption": @@ -153,6 +153,7 @@ proc getUniqueType*(key: PType): PType = else: # ugh, we need the canon here: result = slowSearch(key, k) + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("getUniqueType") proc tableGetType*(tab: TIdTable, key: PType): RootRef = # returns nil if we need to declare this type diff --git a/compiler/cgen.nim b/compiler/cgen.nim index d80a68609..6e18c8389 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -16,6 +16,8 @@ import condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, semparallel +from modulegraphs import ModuleGraph + import strutils except `%` # collides with ropes.`%` when options.hasTinyCBackend: @@ -870,7 +872,7 @@ proc genMainProc(m: BModule) = NimMainInner = "N_CDECL(void, NimMainInner)(void) {$N" & "$1" & "}$N$N" - + NimMainProc = "N_CDECL(void, NimMain)(void) {$N" & "\tvoid (*volatile inner)();$N" & @@ -972,7 +974,13 @@ proc getSomeInitName(m: PSym, suffix: string): Rope = result.add m.name.s result.add suffix -proc getInitName(m: PSym): Rope = getSomeInitName(m, "Init000") +proc getInitName(m: PSym): Rope = + if sfMainModule in m.flags: + # generate constant name for main module, for "easy" debugging. + result = rope"NimMainModule" + else: + result = getSomeInitName(m, "Init000") + proc getDatInitName(m: PSym): Rope = getSomeInitName(m, "DatInit000") proc registerModuleToMain(m: PSym) = @@ -1168,7 +1176,7 @@ proc newModule(module: PSym): BModule = if (sfDeadCodeElim in module.flags): internalError("added pending module twice: " & module.filename) -proc myOpen(module: PSym): PPassContext = +proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = result = newModule(module) if optGenIndex in gGlobalOptions and generatedHeader == nil: let f = if headerFile.len > 0: headerFile else: gProjectFull @@ -1203,7 +1211,7 @@ proc getCFile(m: BModule): string = else: ".c" result = changeFileExt(completeCFilePath(m.cfilename.withPackageName), ext) -proc myOpenCached(module: PSym, rd: PRodReader): PPassContext = +proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext = assert optSymbolFiles in gGlobalOptions var m = newModule(module) readMergeInfo(getCFile(m), m) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index a94950029..faeea7afb 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -68,6 +68,7 @@ type beforeRetNeeded*: bool # true iff 'BeforeRet' label for proc is needed threadVarAccessed*: bool # true if the proc already accessed some threadvar lastLineInfo*: TLineInfo # to avoid generating excessive 'nimln' statements + currLineInfo*: TLineInfo # AST codegen will make this superfluous nestedTryStmts*: seq[PNode] # in how many nested try statements we are # (the vars must be volatile then) inExceptBlock*: int # are we currently inside an except block? diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index bcf0b535b..5f0d71cc6 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -165,8 +165,9 @@ proc methodDef*(s: PSym, fromCache: bool) = if witness.isNil: witness = gMethods[i].methods[0] # create a new dispatcher: add(gMethods, (methods: @[s], dispatcher: createDispatcher(s))) - if fromCache: - internalError(s.info, "no method dispatcher found") + #echo "adding ", s.info + #if fromCache: + # internalError(s.info, "no method dispatcher found") if witness != nil: localError(s.info, "invalid declaration order; cannot attach '" & s.name.s & "' to method defined here: " & $witness.info) diff --git a/compiler/commands.nim b/compiler/commands.nim index de1197292..590c4871d 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -124,17 +124,17 @@ proc splitSwitch(switch: string, cmd, arg: var string, pass: TCmdLinePass, proc processOnOffSwitch(op: TOptions, arg: string, pass: TCmdLinePass, info: TLineInfo) = - case whichKeyword(arg) - of wOn: gOptions = gOptions + op - of wOff: gOptions = gOptions - op + case arg.normalize + of "on": gOptions = gOptions + op + of "off": gOptions = gOptions - op else: localError(info, errOnOrOffExpectedButXFound, arg) proc processOnOffSwitchOrList(op: TOptions, arg: string, pass: TCmdLinePass, info: TLineInfo): bool = result = false - case whichKeyword(arg) - of wOn: gOptions = gOptions + op - of wOff: gOptions = gOptions - op + case arg.normalize + of "on": gOptions = gOptions + op + of "off": gOptions = gOptions - op else: if arg == "list": result = true @@ -143,9 +143,9 @@ proc processOnOffSwitchOrList(op: TOptions, arg: string, pass: TCmdLinePass, proc processOnOffSwitchG(op: TGlobalOptions, arg: string, pass: TCmdLinePass, info: TLineInfo) = - case whichKeyword(arg) - of wOn: gGlobalOptions = gGlobalOptions + op - of wOff: gGlobalOptions = gGlobalOptions - op + case arg.normalize + of "on": gGlobalOptions = gGlobalOptions + op + of "off": gGlobalOptions = gGlobalOptions - op else: localError(info, errOnOrOffExpectedButXFound, arg) proc expectArg(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = @@ -178,12 +178,12 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, var x = findStr(msgs.WarningsToStr, id) if x >= 0: n = TNoteKind(x + ord(warnMin)) else: localError(info, "unknown warning: " & id) - case whichKeyword(substr(arg, i)) - of wOn: + case substr(arg, i).normalize + of "on": incl(gNotes, n) incl(gMainPackageNotes, n) incl(enableNotes, n) - of wOff: + of "off": excl(gNotes, n) excl(gMainPackageNotes, n) incl(disableNotes, n) @@ -242,6 +242,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool = of "linetrace": result = contains(gOptions, optLineTrace) of "debugger": result = contains(gOptions, optEndb) of "profiler": result = contains(gOptions, optProfiler) + of "memtracker": result = contains(gOptions, optMemTracker) of "checks", "x": result = gOptions * ChecksOptions == ChecksOptions of "floatchecks": result = gOptions * {optNaNCheck, optInfCheck} == {optNaNCheck, optInfCheck} @@ -264,6 +265,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool = of "implicitstatic": result = contains(gOptions, optImplicitStatic) of "patterns": result = contains(gOptions, optPatterns) of "experimental": result = gExperimentalMode + of "excessivestacktrace": result = contains(gGlobalOptions, optExcessiveStackTrace) else: invalidCmdLineOption(passCmd1, switch, info) proc processPath(path: string, info: TLineInfo, @@ -445,6 +447,10 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = processOnOffSwitch({optProfiler}, arg, pass, info) if optProfiler in gOptions: defineSymbol("profiler") else: undefSymbol("profiler") + of "memtracker": + processOnOffSwitch({optMemTracker}, arg, pass, info) + if optMemTracker in gOptions: defineSymbol("memtracker") + else: undefSymbol("memtracker") of "checks", "x": processOnOffSwitch(ChecksOptions, arg, pass, info) of "floatchecks": processOnOffSwitch({optNaNCheck, optInfCheck}, arg, pass, info) @@ -630,12 +636,8 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = of "dynliboverride": dynlibOverride(switch, arg, pass, info) of "cs": + # only supported for compatibility. Does nothing. expectArg(switch, arg, pass, info) - case arg - of "partial": idents.firstCharIsCS = true - of "none": idents.firstCharIsCS = false - else: localError(info, errGenerated, - "'partial' or 'none' expected, but found " & arg) of "experimental": expectNoArg(switch, arg, pass, info) gExperimentalMode = true diff --git a/compiler/depends.nim b/compiler/depends.nim index 1ccb134f2..9087f89f2 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -12,6 +12,8 @@ import os, options, ast, astalgo, msgs, ropes, idents, passes, importer +from modulegraphs import ModuleGraph + proc generateDot*(project: string) type @@ -46,7 +48,7 @@ proc generateDot(project: string) = rope(changeFileExt(extractFilename(project), "")), gDotGraph], changeFileExt(project, "dot")) -proc myOpen(module: PSym): PPassContext = +proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = var g: PGen new(g) g.module = module diff --git a/compiler/docgen.nim b/compiler/docgen.nim index c220902ff..76b36d796 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -184,7 +184,7 @@ proc genRecComment(d: PDoc, n: PNode): Rope = if n == nil: return nil result = genComment(d, n).rope if result == nil: - if n.kind notin {nkEmpty..nkNilLit, nkEnumTy}: + if n.kind notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}: for i in countup(0, len(n)-1): result = genRecComment(d, n.sons[i]) if result != nil: return @@ -514,7 +514,7 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = result["code"] = %r.buf proc checkForFalse(n: PNode): bool = - result = n.kind == nkIdent and identEq(n.ident, "false") + result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0 proc traceDeps(d: PDoc, n: PNode) = const k = skModule @@ -691,7 +691,7 @@ proc writeOutputJson*(d: PDoc, filename, outExt: string, discard "fixme: error report" proc commandDoc*() = - var ast = parseFile(gProjectMainIdx) + var ast = parseFile(gProjectMainIdx, newIdentCache()) if ast == nil: return var d = newDocumentor(gProjectFull, options.gConfigVars) d.hasToc = true @@ -721,7 +721,7 @@ proc commandRst2TeX*() = commandRstAux(gProjectFull, TexExt) proc commandJson*() = - var ast = parseFile(gProjectMainIdx) + var ast = parseFile(gProjectMainIdx, newIdentCache()) if ast == nil: return var d = newDocumentor(gProjectFull, options.gConfigVars) d.hasToc = true diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index d70d5406c..9504ab52f 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -13,6 +13,8 @@ import os, options, ast, astalgo, msgs, ropes, idents, passes, docgen +from modulegraphs import ModuleGraph + type TGen = object of TPassContext doc: PDoc @@ -49,7 +51,7 @@ proc processNodeJson(c: PPassContext, n: PNode): PNode = var g = PGen(c) generateJson(g.doc, n) -proc myOpen(module: PSym): PPassContext = +proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = var g: PGen new(g) g.module = module diff --git a/compiler/filter_tmpl.nim b/compiler/filter_tmpl.nim index 9e123e3a1..361b3d276 100644 --- a/compiler/filter_tmpl.nim +++ b/compiler/filter_tmpl.nim @@ -62,10 +62,7 @@ proc withInExpr(p: TTmplParser): bool {.inline.} = result = p.par > 0 or p.bracket > 0 or p.curly > 0 proc parseLine(p: var TTmplParser) = - var - d, j, curly: int - keyw: string - j = 0 + var j = 0 while p.x[j] == ' ': inc(j) if p.x[0] == p.nimDirective and p.x[1] == '?': newLine(p) @@ -73,16 +70,16 @@ proc parseLine(p: var TTmplParser) = newLine(p) inc(j) while p.x[j] == ' ': inc(j) - d = j - keyw = "" + let d = j + var keyw = "" while p.x[j] in PatternChars: add(keyw, p.x[j]) inc(j) scanPar(p, j) p.pendingExprLine = withInExpr(p) or llstream.endsWithOpr(p.x) - case whichKeyword(keyw) - of wEnd: + case keyw + of "end": if p.indent >= 2: dec(p.indent, 2) else: @@ -90,15 +87,15 @@ proc parseLine(p: var TTmplParser) = localError(p.info, errXNotAllowedHere, "end") llStreamWrite(p.outp, spaces(p.indent)) llStreamWrite(p.outp, "#end") - of wIf, wWhen, wTry, wWhile, wFor, wBlock, wCase, wProc, wIterator, - wConverter, wMacro, wTemplate, wMethod: + of "if", "when", "try", "while", "for", "block", "case", "proc", "iterator", + "converter", "macro", "template", "method": llStreamWrite(p.outp, spaces(p.indent)) llStreamWrite(p.outp, substr(p.x, d)) inc(p.indent, 2) - of wElif, wOf, wElse, wExcept, wFinally: + of "elif", "of", "else", "except", "finally": llStreamWrite(p.outp, spaces(p.indent - 2)) llStreamWrite(p.outp, substr(p.x, d)) - of wLet, wVar, wConst, wType: + of "wLet", "wVar", "wConst", "wType": llStreamWrite(p.outp, spaces(p.indent)) llStreamWrite(p.outp, substr(p.x, d)) if not p.x.contains({':', '='}): @@ -158,7 +155,7 @@ proc parseLine(p: var TTmplParser) = llStreamWrite(p.outp, p.toStr) llStreamWrite(p.outp, '(') inc(j) - curly = 0 + var curly = 0 while true: case p.x[j] of '\0': diff --git a/compiler/filters.nim b/compiler/filters.nim index adafe464e..d1a6409ff 100644 --- a/compiler/filters.nim +++ b/compiler/filters.nim @@ -30,7 +30,7 @@ proc getArg(n: PNode, name: string, pos: int): PNode = for i in countup(1, sonsLen(n) - 1): if n.sons[i].kind == nkExprEqExpr: if n.sons[i].sons[0].kind != nkIdent: invalidPragma(n) - if identEq(n.sons[i].sons[0].ident, name): + if cmpIgnoreStyle(n.sons[i].sons[0].ident.s, name) == 0: return n.sons[i].sons[1] elif i == pos: return n.sons[i] @@ -50,8 +50,8 @@ proc strArg(n: PNode, name: string, pos: int, default: string): string = proc boolArg(n: PNode, name: string, pos: int, default: bool): bool = var x = getArg(n, name, pos) if x == nil: result = default - elif (x.kind == nkIdent) and identEq(x.ident, "true"): result = true - elif (x.kind == nkIdent) and identEq(x.ident, "false"): result = false + elif x.kind == nkIdent and cmpIgnoreStyle(x.ident.s, "true") == 0: result = true + elif x.kind == nkIdent and cmpIgnoreStyle(x.ident.s, "false") == 0: result = false else: invalidPragma(n) proc filterStrip(stdin: PLLStream, filename: string, call: PNode): PLLStream = @@ -62,7 +62,7 @@ proc filterStrip(stdin: PLLStream, filename: string, call: PNode): PLLStream = var line = newStringOfCap(80) while llStreamReadLine(stdin, line): var stripped = strip(line, leading, trailing) - if (len(pattern) == 0) or startsWith(stripped, pattern): + if len(pattern) == 0 or startsWith(stripped, pattern): llStreamWriteln(result, stripped) else: llStreamWriteln(result, line) diff --git a/compiler/idents.nim b/compiler/idents.nim index d9b72baf0..eecfa60a1 100644 --- a/compiler/idents.nim +++ b/compiler/idents.nim @@ -12,7 +12,7 @@ # id. This module is essential for the compiler's performance. import - hashes, strutils, etcpriv + hashes, strutils, etcpriv, wordrecg type TIdObj* = object of RootObj @@ -25,12 +25,20 @@ type next*: PIdent # for hash-table chaining h*: Hash # hash value of s -var firstCharIsCS*: bool = true -var buckets*: array[0..4096 * 2 - 1, PIdent] + IdentCache* = ref object + buckets: array[0..4096 * 2 - 1, PIdent] + wordCounter: int + idAnon*, idDelegator*, emptyIdent*: PIdent + +var + legacy: IdentCache + +proc resetIdentCache*() = + for i in low(legacy.buckets)..high(legacy.buckets): + legacy.buckets[i] = nil proc cmpIgnoreStyle(a, b: cstring, blen: int): int = - if firstCharIsCS: - if a[0] != b[0]: return 1 + if a[0] != b[0]: return 1 var i = 0 var j = 0 result = 1 @@ -65,9 +73,9 @@ proc cmpExact(a, b: cstring, blen: int): int = if result == 0: if a[i] != '\0': result = 1 -var wordCounter = 1 +{.this: self.} -proc getIdent*(identifier: cstring, length: int, h: Hash): PIdent = +proc getIdent*(self: IdentCache; identifier: cstring, length: int, h: Hash): PIdent = var idx = h and high(buckets) result = buckets[idx] var last: PIdent = nil @@ -97,16 +105,33 @@ proc getIdent*(identifier: cstring, length: int, h: Hash): PIdent = else: result.id = id -proc getIdent*(identifier: string): PIdent = +proc getIdent*(self: IdentCache; identifier: string): PIdent = result = getIdent(cstring(identifier), len(identifier), hashIgnoreStyle(identifier)) -proc getIdent*(identifier: string, h: Hash): PIdent = +proc getIdent*(self: IdentCache; identifier: string, h: Hash): PIdent = result = getIdent(cstring(identifier), len(identifier), h) -proc identEq*(id: PIdent, name: string): bool = - result = id.id == getIdent(name).id +proc newIdentCache*(): IdentCache = + if legacy.isNil: + result = IdentCache() + result.idAnon = result.getIdent":anonymous" + result.wordCounter = 1 + result.idDelegator = result.getIdent":delegator" + result.emptyIdent = result.getIdent("") + # initialize the keywords: + for s in countup(succ(low(specialWords)), high(specialWords)): + result.getIdent(specialWords[s], hashIgnoreStyle(specialWords[s])).id = ord(s) + legacy = result + else: + result = legacy -var idAnon* = getIdent":anonymous" -let idDelegator* = getIdent":delegator" +proc whichKeyword*(id: PIdent): TSpecialWord = + if id.id < 0: result = wInvalid + else: result = TSpecialWord(id.id) +proc getIdent*(identifier: string): PIdent = + ## for backwards compatibility. + if legacy.isNil: + discard newIdentCache() + legacy.getIdent identifier diff --git a/compiler/importer.nim b/compiler/importer.nim index 87415733b..feebf97c4 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -100,7 +100,7 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) = let ident = lookups.considerQuotedIdent(n) let s = strTableGet(fromMod.tab, ident) if s == nil: - localError(n.info, errUndeclaredIdentifier, ident.s) + errorUndeclaredIdentifier(c, n.info, ident.s) else: if s.kind == skStub: loadStub(s) if s.kind notin ExportableSymKinds: @@ -162,12 +162,26 @@ proc importModuleAs(n: PNode, realModule: PSym): PSym = proc myImportModule(c: PContext, n: PNode): PSym = var f = checkModuleName(n) if f != InvalidFileIDX: - result = importModuleAs(n, gImportModule(c.module, f)) + let L = c.graph.importStack.len + let recursion = c.graph.importStack.find(f) + c.graph.importStack.add f + #echo "adding ", toFullPath(f), " at ", L+1 + if recursion >= 0: + var err = "" + for i in countup(recursion, L-1): + if i > recursion: err.add "\n" + err.add toFullPath(c.graph.importStack[i]) & " imports " & + toFullPath(c.graph.importStack[i+1]) + c.recursiveDep = err + result = importModuleAs(n, gImportModule(c.graph, c.module, f, c.cache)) + #echo "set back to ", L + c.graph.importStack.setLen(L) # we cannot perform this check reliably because of # test: modules/import_in_config) - if result.info.fileIndex == c.module.info.fileIndex and - result.info.fileIndex == n.info.fileIndex: - localError(n.info, errGenerated, "A module cannot import itself") + when true: + if result.info.fileIndex == c.module.info.fileIndex and + result.info.fileIndex == n.info.fileIndex: + localError(n.info, errGenerated, "A module cannot import itself") if sfDeprecated in result.flags: message(n.info, warnDeprecated, result.name.s) #suggestSym(n.info, result, false) diff --git a/compiler/installer.ini b/compiler/installer.ini index 2a9fa36a5..5e4b3ddbc 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -41,8 +41,8 @@ Files: "config/nimdoc.tex.cfg" ; Files: "doc/*.cfg" ; Files: "doc/*.pdf" ; Files: "doc/*.ini" -Files: "doc/overview.html" -Start: "doc/overview.html" +Files: "doc/html/overview.html" +Start: "doc/html/overview.html" [Other] @@ -63,6 +63,7 @@ Files: "icons/koch_icon.o" Files: "compiler" Files: "doc" +Files: "doc/html" Files: "tools" Files: "web/website.ini" Files: "web/ticker.html" @@ -89,12 +90,11 @@ Files: "bin/c2nim.exe" Files: "bin/nimgrep.exe" Files: "bin/nimsuggest.exe" Files: "bin/nimble.exe" -Files: "bin/makelink.exe" -Files: "bin/*.dll" Files: "koch.exe" +Files: "finish.exe" ; Files: "dist/mingw" -Files: "start.bat" +Files: r"tools\start.bat" BinPath: r"bin;dist\mingw\bin;dist" ; Section | dir | zipFile | size hint (in KB) | url | exe start menu entry @@ -104,6 +104,10 @@ Download: r"Support DLLs|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls. Download: r"Aporia Text Editor|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe" ; for now only NSIS supports optional downloads +[WinBin] +Files: "$NIMINSTDEPS/makelink.exe" +Files: "$NIMINSTDEPS/*.dll" + [UnixBin] Files: "bin/nim" diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index e7fe8cc27..028dd00f0 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -35,6 +35,8 @@ import times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils, intsets, cgmeth, lowerings +from modulegraphs import ModuleGraph + type TTarget = enum targetJS, targetPHP @@ -138,7 +140,7 @@ proc declareGlobal(p: PProc; id: int; r: Rope) = const MappedToObject = {tyObject, tyArray, tyArrayConstr, tyTuple, tyOpenArray, - tySet, tyBigNum, tyVarargs} + tySet, tyVarargs} proc mapType(typ: PType): TJSTypeKind = let t = skipTypes(typ, abstractInst) @@ -151,15 +153,13 @@ proc mapType(typ: PType): TJSTypeKind = of tyPointer: # treat a tyPointer like a typed pointer to an array of bytes result = etyBaseIndex - of tyRange, tyDistinct, tyOrdinal, tyConst, tyMutable, tyIter, tyProxy: - result = mapType(t.sons[0]) + of tyRange, tyDistinct, tyOrdinal, tyProxy: result = mapType(t.sons[0]) of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar: result = etyInt of tyBool: result = etyBool of tyFloat..tyFloat128: result = etyFloat of tySet: result = etyObject # map a set to a table of tyString, tySequence: result = etySeq - of tyObject, tyArray, tyArrayConstr, tyTuple, tyOpenArray, tyBigNum, - tyVarargs: + of tyObject, tyArray, tyArrayConstr, tyTuple, tyOpenArray, tyVarargs: result = etyObject of tyNil: result = etyNull of tyGenericInst, tyGenericParam, tyGenericBody, tyGenericInvocation, @@ -171,6 +171,7 @@ proc mapType(typ: PType): TJSTypeKind = else: result = etyNone of tyProc: result = etyProc of tyCString: result = etyString + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("mapType") proc mapType(p: PProc; typ: PType): TJSTypeKind = if p.target == targetPHP: result = etyObject @@ -2273,11 +2274,11 @@ proc myClose(b: PPassContext, n: PNode): PNode = for obj, content in items(globals.classes): genClass(obj, content, ext) -proc myOpenCached(s: PSym, rd: PRodReader): PPassContext = +proc myOpenCached(graph: ModuleGraph; s: PSym, rd: PRodReader): PPassContext = internalError("symbol files are not possible with the JS code generator") result = nil -proc myOpen(s: PSym): PPassContext = +proc myOpen(graph: ModuleGraph; s: PSym; cache: IdentCache): PPassContext = var r = newModule(s) r.target = if gCmd == cmdCompileToPHP: targetPHP else: targetJS result = r diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 9c513034b..2769d757c 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -127,8 +127,9 @@ type # this is needed because scanning comments # needs so much look-ahead currLineIndent*: int - strongSpaces*: bool + strongSpaces*, allowTabs*: bool errorHandler*: TErrorHandler + cache*: IdentCache var gLinesCompiled*: int # all lines that have been compiled @@ -164,7 +165,6 @@ proc tokToStr*(tok: TToken): string = if tok.ident != nil: result = tok.ident.s else: - internalError("tokToStr") result = "" proc prettyTok*(tok: TToken): string = @@ -175,8 +175,6 @@ proc printTok*(tok: TToken) = msgWriteln($tok.line & ":" & $tok.col & "\t" & TokTypeToStr[tok.tokType] & " " & tokToStr(tok)) -var dummyIdent: PIdent - proc initToken*(L: var TToken) = L.tokType = tkInvalid L.iNumber = 0 @@ -185,7 +183,7 @@ proc initToken*(L: var TToken) = L.literal = "" L.fNumber = 0.0 L.base = base10 - L.ident = dummyIdent + L.ident = nil proc fillToken(L: var TToken) = L.tokType = tkInvalid @@ -195,17 +193,20 @@ proc fillToken(L: var TToken) = setLen(L.literal, 0) L.fNumber = 0.0 L.base = base10 - L.ident = dummyIdent + L.ident = nil -proc openLexer*(lex: var TLexer, fileIdx: int32, inputstream: PLLStream) = +proc openLexer*(lex: var TLexer, fileIdx: int32, inputstream: PLLStream; + cache: IdentCache) = openBaseLexer(lex, inputstream) lex.fileIdx = fileidx lex.indentAhead = - 1 lex.currLineIndent = 0 inc(lex.lineNumber, inputstream.lineOffset) + lex.cache = cache -proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream) = - openLexer(lex, filename.fileInfoIdx, inputstream) +proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream; + cache: IdentCache) = + openLexer(lex, filename.fileInfoIdx, inputstream, cache) proc closeLexer*(lex: var TLexer) = inc(gLinesCompiled, lex.lineNumber) @@ -746,7 +747,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) = else: break h = !$h - tok.ident = getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) + tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) L.bufpos = pos if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)): @@ -757,7 +758,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) = proc endOperator(L: var TLexer, tok: var TToken, pos: int, hash: Hash) {.inline.} = var h = !$hash - tok.ident = getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) + tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr else: tok.tokType = TTokType(tok.ident.id - oprLow + ord(tkColon)) L.bufpos = pos @@ -847,34 +848,23 @@ proc scanComment(L: var TLexer, tok: var TToken) = tok.tokType = tkComment # iNumber contains the number of '\n' in the token tok.iNumber = 0 - when not defined(nimfix): - assert buf[pos+1] == '#' - if buf[pos+2] == '[': - skipMultiLineComment(L, tok, pos+3, true) - return - inc(pos, 2) + assert buf[pos+1] == '#' + if buf[pos+2] == '[': + skipMultiLineComment(L, tok, pos+3, true) + return + inc(pos, 2) var toStrip = 0 while buf[pos] == ' ': inc pos inc toStrip - when defined(nimfix): - var col = getColNumber(L, pos) while true: var lastBackslash = -1 while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: if buf[pos] == '\\': lastBackslash = pos+1 add(tok.literal, buf[pos]) inc(pos) - when defined(nimfix): - if lastBackslash > 0: - # a backslash is a continuation character if only followed by spaces - # plus a newline: - while buf[lastBackslash] == ' ': inc(lastBackslash) - if buf[lastBackslash] notin {CR, LF, nimlexbase.EndOfFile}: - # false positive: - lastBackslash = -1 pos = handleCRLF(L, pos) buf = L.buf @@ -883,21 +873,13 @@ proc scanComment(L: var TLexer, tok: var TToken) = inc(pos) inc(indent) - when defined(nimfix): - template doContinue(): untyped = - buf[pos] == '#' and (col == indent or lastBackslash > 0) - else: - template doContinue(): untyped = - buf[pos] == '#' and buf[pos+1] == '#' - if doContinue(): + if buf[pos] == '#' and buf[pos+1] == '#': tok.literal.add "\n" - when defined(nimfix): col = indent - else: - inc(pos, 2) - var c = toStrip - while buf[pos] == ' ' and c > 0: - inc pos - dec c + inc(pos, 2) + var c = toStrip + while buf[pos] == ' ' and c > 0: + inc pos + dec c inc tok.iNumber else: if buf[pos] > ' ': @@ -915,7 +897,7 @@ proc skip(L: var TLexer, tok: var TToken) = inc(pos) inc(tok.strongSpaceA) of '\t': - lexMessagePos(L, errTabulatorsAreNotAllowed, pos) + if not L.allowTabs: lexMessagePos(L, errTabulatorsAreNotAllowed, pos) inc(pos) of CR, LF: pos = handleCRLF(L, pos) @@ -932,27 +914,19 @@ proc skip(L: var TLexer, tok: var TToken) = else: break tok.strongSpaceA = 0 - when defined(nimfix): - template doBreak(): untyped = buf[pos] > ' ' - else: - template doBreak(): untyped = - buf[pos] > ' ' and (buf[pos] != '#' or buf[pos+1] == '#') - if doBreak(): + if buf[pos] > ' ' and (buf[pos] != '#' or buf[pos+1] == '#'): tok.indent = indent L.currLineIndent = indent break of '#': - when defined(nimfix): - break + # do not skip documentation comment: + if buf[pos+1] == '#': break + if buf[pos+1] == '[': + skipMultiLineComment(L, tok, pos+2, false) + pos = L.bufpos + buf = L.buf else: - # do not skip documentation comment: - if buf[pos+1] == '#': break - if buf[pos+1] == '[': - skipMultiLineComment(L, tok, pos+2, false) - pos = L.bufpos - buf = L.buf - else: - while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) + while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) else: break # EndOfFile also leaves the loop L.bufpos = pos @@ -1051,7 +1025,7 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = if L.buf[L.bufpos] notin SymChars+{'_'} and not isMagicIdentSeparatorRune(L.buf, L.bufpos): tok.tokType = tkSymbol - tok.ident = getIdent("_") + tok.ident = L.cache.getIdent("_") else: tok.literal = $c tok.tokType = tkInvalid @@ -1084,5 +1058,3 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = tok.tokType = tkInvalid lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')') inc(L.bufpos) - -dummyIdent = getIdent("") diff --git a/compiler/lookups.nim b/compiler/lookups.nim index df19a6afb..fe159011c 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -242,6 +242,15 @@ proc errorUseQualifier*(c: PContext; info: TLineInfo; s: PSym) = inc i localError(info, errGenerated, err) +proc errorUndeclaredIdentifier*(c: PContext; info: TLineInfo; name: string) = + var err = "undeclared identifier: '" & name & "'" + if c.recursiveDep.len > 0: + err.add "\nThis might be caused by a recursive module dependency: " + err.add c.recursiveDep + # prevent excessive errors for 'nim check' + c.recursiveDep = nil + localError(info, errGenerated, err) + proc lookUp*(c: PContext, n: PNode): PSym = # Looks up a symbol. Generates an error in case of nil. case n.kind @@ -249,7 +258,7 @@ proc lookUp*(c: PContext, n: PNode): PSym = result = searchInScopes(c, n.ident).skipAlias(n) if result == nil: fixSpelling(n, n.ident, searchInScopes) - localError(n.info, errUndeclaredIdentifier, n.ident.s) + errorUndeclaredIdentifier(c, n.info, n.ident.s) result = errorSym(c, n) of nkSym: result = n.sym @@ -258,7 +267,7 @@ proc lookUp*(c: PContext, n: PNode): PSym = result = searchInScopes(c, ident).skipAlias(n) if result == nil: fixSpelling(n, ident, searchInScopes) - localError(n.info, errUndeclaredIdentifier, ident.s) + errorUndeclaredIdentifier(c, n.info, ident.s) result = errorSym(c, n) else: internalError(n.info, "lookUp") @@ -282,7 +291,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym = result = searchInScopes(c, ident, allExceptModule).skipAlias(n) if result == nil and checkUndeclared in flags: fixSpelling(n, ident, searchInScopes) - localError(n.info, errUndeclaredIdentifier, ident.s) + errorUndeclaredIdentifier(c, n.info, ident.s) result = errorSym(c, n) elif checkAmbiguity in flags and result != nil and contains(c.ambiguousSymbols, result.id): @@ -307,7 +316,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym = result = strTableGet(m.tab, ident).skipAlias(n) if result == nil and checkUndeclared in flags: fixSpelling(n.sons[1], ident, searchInScopes) - localError(n.sons[1].info, errUndeclaredIdentifier, ident.s) + errorUndeclaredIdentifier(c, n.sons[1].info, ident.s) result = errorSym(c, n.sons[1]) elif n.sons[1].kind == nkSym: result = n.sons[1].sym diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 9db4383f6..6a8eccb83 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -284,7 +284,7 @@ proc addLocalVar(varSection, varInit: PNode; owner: PSym; typ: PType; varInit.add newFastAsgnStmt(newSymNode(result), v) else: let deepCopyCall = newNodeI(nkCall, varInit.info, 3) - deepCopyCall.sons[0] = newSymNode(createMagic("deepCopy", mDeepCopy)) + deepCopyCall.sons[0] = newSymNode(getSysMagic("deepCopy", mDeepCopy)) deepCopyCall.sons[1] = newSymNode(result) deepCopyCall.sons[2] = v varInit.add deepCopyCall @@ -356,7 +356,7 @@ proc createWrapperProc(f: PNode; threadParam, argsParam: PSym; if fk == fvGC: "data" else: "blob", fv.info), call) if fk == fvGC: let incRefCall = newNodeI(nkCall, fv.info, 2) - incRefCall.sons[0] = newSymNode(createMagic("GCref", mGCref)) + incRefCall.sons[0] = newSymNode(getSysMagic("GCref", mGCref)) incRefCall.sons[1] = indirectAccess(threadLocalProm.newSymNode, "data", fv.info) body.add incRefCall @@ -446,7 +446,7 @@ proc genHigh*(n: PNode): PNode = else: result = newNodeI(nkCall, n.info, 2) result.typ = getSysType(tyInt) - result.sons[0] = newSymNode(createMagic("high", mHigh)) + result.sons[0] = newSymNode(getSysMagic("high", mHigh)) result.sons[1] = n proc setupArgsForParallelism(n: PNode; objType: PType; scratchObj: PSym; diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index 13b365d04..6a9d69082 100644 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -38,6 +38,14 @@ proc getSysSym*(name: string): PSym = if result.kind == skStub: loadStub(result) if result.kind == skAlias: result = result.owner +proc createMagic*(name: string, m: TMagic): PSym = + result = newSym(skProc, getIdent(name), nil, unknownLineInfo()) + result.magic = m + +let + opNot* = createMagic("not", mNot) + opContains* = createMagic("contains", mInSet) + proc getSysMagic*(name: string, m: TMagic): PSym = var ti: TIdentIter let id = getIdent(name) @@ -46,7 +54,7 @@ proc getSysMagic*(name: string, m: TMagic): PSym = if r.kind == skStub: loadStub(r) if r.magic == m: # prefer the tyInt variant: - if r.typ.sons[0].kind == tyInt: return r + if r.typ.sons[0] != nil and r.typ.sons[0].kind == tyInt: return r result = r r = nextIdentIter(ti, systemModule.tab) if result != nil: return result diff --git a/compiler/main.nim b/compiler/main.nim index 0db66b53e..2118078be 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -15,7 +15,8 @@ import wordrecg, sem, semdata, idents, passes, docgen, extccomp, cgen, jsgen, json, nversion, platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen, - docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists + docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists, + modulegraphs from magicsys import systemModule, resetSysTypes @@ -30,80 +31,44 @@ proc semanticPasses = registerPass verbosePass registerPass semPass -proc commandGenDepend = +proc commandGenDepend(graph: ModuleGraph; cache: IdentCache) = semanticPasses() registerPass(gendependPass) registerPass(cleanupPass) - compileProject() + compileProject(graph, cache) generateDot(gProjectFull) execExternalProgram("dot -Tpng -o" & changeFileExt(gProjectFull, "png") & ' ' & changeFileExt(gProjectFull, "dot")) -proc commandCheck = +proc commandCheck(graph: ModuleGraph; cache: IdentCache) = msgs.gErrorMax = high(int) # do not stop after first error defineSymbol("nimcheck") semanticPasses() # use an empty backend for semantic checking only rodPass() - compileProject() + compileProject(graph, cache) -proc commandDoc2(json: bool) = +proc commandDoc2(graph: ModuleGraph; cache: IdentCache; json: bool) = msgs.gErrorMax = high(int) # do not stop after first error semanticPasses() if json: registerPass(docgen2JsonPass) else: registerPass(docgen2Pass) #registerPass(cleanupPass()) - compileProject() + compileProject(graph, cache) finishDoc2Pass(gProjectName) -proc commandCompileToC = +proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) = extccomp.initVars() semanticPasses() registerPass(cgenPass) rodPass() #registerPass(cleanupPass()) - compileProject() + compileProject(graph, cache) cgenWriteModules() if gCmd != cmdRun: extccomp.callCCompiler(changeFileExt(gProjectFull, "")) - if isServing: - # caas will keep track only of the compilation commands - lastCaasCmd = curCaasCmd - resetCgenModules() - for i in 0 .. <gMemCacheData.len: - gMemCacheData[i].hashStatus = hashCached - gMemCacheData[i].needsRecompile = Maybe - - # XXX: clean these global vars - # ccgstmts.gBreakpoints - # ccgthreadvars.nimtv - # ccgthreadvars.nimtVDeps - # ccgthreadvars.nimtvDeclared - # cgendata - # cgmeth? - # condsyms? - # depends? - # lexer.gLinesCompiled - # msgs - error counts - # magicsys, when system.nim changes - # rodread.rodcompilerProcs - # rodread.gTypeTable - # rodread.gMods - - # !! ropes.cache - # semthreads.computed? - # - # suggest.usageSym - # - # XXX: can we run out of IDs? - # XXX: detect config reloading (implement as error/require restart) - # XXX: options are appended (they will accumulate over time) - resetCompilationLists() - ccgutils.resetCaches() - GC_fullCollect() - -proc commandCompileToJS = +proc commandCompileToJS(graph: ModuleGraph; cache: IdentCache) = #incl(gGlobalOptions, optSafeCode) setTarget(osJS, cpuJS) #initDefines() @@ -113,9 +78,9 @@ proc commandCompileToJS = if gCmd == cmdCompileToPHP: defineSymbol("nimphp") semanticPasses() registerPass(JSgenPass) - compileProject() + compileProject(graph, cache) -proc interactivePasses = +proc interactivePasses(graph: ModuleGraph; cache: IdentCache) = #incl(gGlobalOptions, optSafeCode) #setTarget(osNimrodVM, cpuNimrodVM) initDefines() @@ -125,30 +90,30 @@ proc interactivePasses = registerPass(semPass) registerPass(evalPass) -proc commandInteractive = +proc commandInteractive(graph: ModuleGraph; cache: IdentCache) = msgs.gErrorMax = high(int) # do not stop after first error - interactivePasses() - compileSystemModule() + interactivePasses(graph, cache) + compileSystemModule(graph, cache) if commandArgs.len > 0: - discard compileModule(fileInfoIdx(gProjectFull), {}) + discard graph.compileModule(fileInfoIdx(gProjectFull), cache, {}) else: - var m = makeStdinModule() + var m = graph.makeStdinModule() incl(m.flags, sfMainModule) - processModule(m, llStreamOpenStdIn(), nil) + processModule(graph, m, llStreamOpenStdIn(), nil, cache) const evalPasses = [verbosePass, semPass, evalPass] -proc evalNim(nodes: PNode, module: PSym) = - carryPasses(nodes, module, evalPasses) +proc evalNim(graph: ModuleGraph; nodes: PNode, module: PSym; cache: IdentCache) = + carryPasses(graph, nodes, module, cache, evalPasses) -proc commandEval(exp: string) = +proc commandEval(graph: ModuleGraph; cache: IdentCache; exp: string) = if systemModule == nil: - interactivePasses() - compileSystemModule() - var echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")" - evalNim(echoExp.parseString, makeStdinModule()) + interactivePasses(graph, cache) + compileSystemModule(graph, cache) + let echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")" + evalNim(graph, echoExp.parseString(cache), makeStdinModule(graph), cache) -proc commandScan = +proc commandScan(cache: IdentCache) = var f = addFileExt(mainCommandArg(), NimExt) var stream = llStreamOpen(f, fmRead) if stream != nil: @@ -156,7 +121,7 @@ proc commandScan = L: TLexer tok: TToken initToken(tok) - openLexer(L, f, stream) + openLexer(L, f, stream, cache) while true: rawGetTok(L, tok) printTok(tok) @@ -165,77 +130,11 @@ proc commandScan = else: rawMessage(errCannotOpenFile, f) -proc commandSuggest = - if isServing: - # XXX: hacky work-around ahead - # Currently, it's possible to issue a idetools command, before - # issuing the first compile command. This will leave the compiler - # cache in a state where "no recompilation is necessary", but the - # cgen pass was never executed at all. - commandCompileToC() - let gDirtyBufferIdx = gTrackPos.fileIndex - discard compileModule(gDirtyBufferIdx, {sfDirty}) - resetModule(gDirtyBufferIdx) - else: - msgs.gErrorMax = high(int) # do not stop after first error - semanticPasses() - rodPass() - # XXX: this handles the case when the dirty buffer is the main file, - # but doesn't handle the case when it's imported module - #var projFile = if gProjectMainIdx == gDirtyOriginalIdx: gDirtyBufferIdx - # else: gProjectMainIdx - compileProject() #(projFile) - -proc resetMemory = - resetCompilationLists() - ccgutils.resetCaches() - resetAllModules() - resetRopeCache() - resetSysTypes() - gOwners = @[] - for i in low(buckets)..high(buckets): - buckets[i] = nil - idAnon = nil - - # XXX: clean these global vars - # ccgstmts.gBreakpoints - # ccgthreadvars.nimtv - # ccgthreadvars.nimtVDeps - # ccgthreadvars.nimtvDeclared - # cgendata - # cgmeth? - # condsyms? - # depends? - # lexer.gLinesCompiled - # msgs - error counts - # magicsys, when system.nim changes - # rodread.rodcompilerProcs - # rodread.gTypeTable - # rodread.gMods - - # !! ropes.cache - # - # suggest.usageSym - # - # XXX: can we run out of IDs? - # XXX: detect config reloading (implement as error/require restart) - # XXX: options are appended (they will accumulate over time) - # vis = visimpl - when compileOption("gc", "v2"): - gcDebugging = true - echo "COLLECT 1" - GC_fullCollect() - echo "COLLECT 2" - GC_fullCollect() - echo "COLLECT 3" - GC_fullCollect() - echo GC_getStatistics() - const SimulateCaasMemReset = false PrintRopeCacheStats = false -proc mainCommand* = +proc mainCommand*(graph: ModuleGraph; cache: IdentCache) = when SimulateCaasMemReset: gGlobalOptions.incl(optCaasEnabled) @@ -251,66 +150,66 @@ proc mainCommand* = of "c", "cc", "compile", "compiletoc": # compile means compileToC currently gCmd = cmdCompileToC - commandCompileToC() + commandCompileToC(graph, cache) of "cpp", "compiletocpp": gCmd = cmdCompileToCpp defineSymbol("cpp") - commandCompileToC() + commandCompileToC(graph, cache) of "objc", "compiletooc": gCmd = cmdCompileToOC defineSymbol("objc") - commandCompileToC() + commandCompileToC(graph, cache) of "run": gCmd = cmdRun when hasTinyCBackend: extccomp.setCC("tcc") - commandCompileToC() + commandCompileToC(graph, cache) else: rawMessage(errInvalidCommandX, command) of "js", "compiletojs": gCmd = cmdCompileToJS - commandCompileToJS() + commandCompileToJS(graph, cache) of "php": gCmd = cmdCompileToPHP - commandCompileToJS() + commandCompileToJS(graph, cache) of "doc": wantMainModule() gCmd = cmdDoc - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) commandDoc() of "doc2": gCmd = cmdDoc - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) defineSymbol("nimdoc") - commandDoc2(false) + commandDoc2(graph, cache, false) of "rst2html": gCmd = cmdRst2html - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) commandRst2Html() of "rst2tex": gCmd = cmdRst2tex - loadConfigs(DocTexConfig) + loadConfigs(DocTexConfig, cache) commandRst2TeX() of "jsondoc": wantMainModule() gCmd = cmdDoc - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) wantMainModule() defineSymbol("nimdoc") commandJson() of "jsondoc2": gCmd = cmdDoc - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) wantMainModule() defineSymbol("nimdoc") - commandDoc2(true) + commandDoc2(graph, cache, true) of "buildindex": gCmd = cmdDoc - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) commandBuildIndex() of "gendepend": gCmd = cmdGenDepend - commandGenDepend() + commandGenDepend(graph, cache) of "dump": gCmd = cmdDump if getConfigVar("dump.format") == "json": @@ -339,35 +238,21 @@ proc mainCommand* = for it in iterSearchPath(searchPaths): msgWriteln(it) of "check": gCmd = cmdCheck - commandCheck() + commandCheck(graph, cache) of "parse": gCmd = cmdParse wantMainModule() - discard parseFile(gProjectMainIdx) + discard parseFile(gProjectMainIdx, cache) of "scan": gCmd = cmdScan wantMainModule() - commandScan() - msgWriteln("Beware: Indentation tokens depend on the parser\'s state!") + commandScan(cache) + msgWriteln("Beware: Indentation tokens depend on the parser's state!") of "secret": gCmd = cmdInteractive - commandInteractive() + commandInteractive(graph, cache) of "e": - # XXX: temporary command for easier testing - commandEval(mainCommandArg()) - of "reset": - resetMemory() - of "idetools": - gCmd = cmdIdeTools - if gEvalExpr != "": - commandEval(gEvalExpr) - else: - commandSuggest() - of "serve": - isServing = true - gGlobalOptions.incl(optCaasEnabled) - msgs.gErrorMax = high(int) # do not stop after first error - serve(mainCommand) + commandEval(graph, cache, mainCommandArg()) of "nop", "help": # prevent the "success" message: gCmd = cmdDump @@ -394,3 +279,5 @@ proc mainCommand* = resetMemory() resetAttributes() + +proc mainCommand*() = mainCommand(newModuleGraph(), newIdentCache()) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim new file mode 100644 index 000000000..38fd4f89f --- /dev/null +++ b/compiler/modulegraphs.nim @@ -0,0 +1,112 @@ +# +# +# The Nim Compiler +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements the module graph data structure. The module graph +## represents a complete Nim project. Single modules can either be kept in RAM +## or stored in a ROD file. The ROD file mechanism is not yet integrated here. +## +## The caching of modules is critical for 'nimsuggest' and is tricky to get +## right. If module E is being edited, we need autocompletion (and type +## checking) for E but we don't want to recompile depending +## modules right away for faster turnaround times. Instead we mark the module's +## dependencies as 'dirty'. Let D be a dependency of E. If D is dirty, we +## need to recompile it and all of its dependencies that are marked as 'dirty'. +## 'nimsuggest sug' actually is invoked for the file being edited so we know +## its content changed and there is no need to compute any checksums. +## Instead of a recursive algorithm, we use an iterative algorithm: +## +## - If a module gets recompiled, its dependencies need to be updated. +## - Its dependent module stays the same. +## + +import ast, intsets, tables + +type + ModuleGraph* = ref object + modules*: seq[PSym] ## indexed by int32 fileIdx + packageSyms*: TStrTable + deps*: IntSet # the dependency graph or potentially its transitive closure. + suggestMode*: bool # whether we are in nimsuggest mode or not. + invalidTransitiveClosure: bool + inclToMod*: Table[int32, int32] # mapping of include file to the + # first module that included it + importStack*: seq[int32] # The current import stack. Used for detecting recursive + # module dependencies. + +{.this: g.} + +proc newModuleGraph*(): ModuleGraph = + result = ModuleGraph() + initStrTable(result.packageSyms) + result.deps = initIntSet() + result.modules = @[] + result.importStack = @[] + result.inclToMod = initTable[int32, int32]() + +proc resetAllModules*(g: ModuleGraph) = + initStrTable(packageSyms) + deps = initIntSet() + modules = @[] + importStack = @[] + inclToMod = initTable[int32, int32]() + +proc getModule*(g: ModuleGraph; fileIdx: int32): PSym = + if fileIdx >= 0 and fileIdx < modules.len: + result = modules[fileIdx] + +proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b + +proc addDep*(g: ModuleGraph; m: PSym, dep: int32) = + if suggestMode: + deps.incl m.position.dependsOn(dep) + # we compute the transitive closure later when quering the graph lazily. + # this improve efficiency quite a lot: + invalidTransitiveClosure = true + +proc addIncludeDep*(g: ModuleGraph; module, includeFile: int32) = + discard hasKeyOrPut(inclToMod, includeFile, module) + +proc parentModule*(g: ModuleGraph; fileIdx: int32): int32 = + ## returns 'fileIdx' if the file belonging to this index is + ## directly used as a module or else the module that first + ## references this include file. + if fileIdx >= 0 and fileIdx < modules.len and modules[fileIdx] != nil: + result = fileIdx + else: + result = inclToMod.getOrDefault(fileIdx) + +proc transitiveClosure(g: var IntSet; n: int) = + # warshall's algorithm + for k in 0..<n: + for i in 0..<n: + for j in 0..<n: + if i != j and not g.contains(i.dependsOn(j)): + if g.contains(i.dependsOn(k)) and g.contains(k.dependsOn(j)): + g.incl i.dependsOn(j) + +proc markDirty*(g: ModuleGraph; fileIdx: int32) = + let m = getModule fileIdx + if m != nil: incl m.flags, sfDirty + +proc markClientsDirty*(g: ModuleGraph; fileIdx: int32) = + # we need to mark its dependent modules D as dirty right away because after + # nimsuggest is done with this module, the module's dirty flag will be + # cleared but D still needs to be remembered as 'dirty'. + if invalidTransitiveClosure: + invalidTransitiveClosure = false + transitiveClosure(deps, modules.len) + + # every module that *depends* on this file is also dirty: + for i in 0i32..<modules.len.int32: + let m = modules[i] + if m != nil and deps.contains(i.dependsOn(fileIdx)): + incl m.flags, sfDirty + +proc isDirty*(g: ModuleGraph; m: PSym): bool = + result = suggestMode and sfDirty in m.flags diff --git a/compiler/modules.nim b/compiler/modules.nim index 711fb6aa4..3451d85ec 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -7,130 +7,121 @@ # distribution, for details about the copyright. # -## implements the module handling +## Implements the module handling, including the caching of modules. import ast, astalgo, magicsys, securehash, rodread, msgs, cgendata, sigmatch, options, - idents, os, lexer, idgen, passes, syntaxes, llstream - -type - TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled - THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged - - TModuleInMemory* = object - compiledAt*: float - hash*: SecureHash - deps*: seq[int32] ## XXX: slurped files are currently not tracked - needsRecompile*: TNeedRecompile - hashStatus*: THashStatus - -var - gCompiledModules: seq[PSym] = @[] - gMemCacheData*: seq[TModuleInMemory] = @[] - ## XXX: we should implement recycling of file IDs - ## if the user keeps renaming modules, the file IDs will keep growing - gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why. - packageSyms: TStrTable - -initStrTable(packageSyms) - -proc getModule*(fileIdx: int32): PSym = - if fileIdx >= 0 and fileIdx < gCompiledModules.len: - result = gCompiledModules[fileIdx] - -proc hashChanged(fileIdx: int32): bool = - internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len - - template updateStatus = - gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged - else: hashNotChanged - # echo "TESTING Hash: ", fileIdx.toFilename, " ", result - - case gMemCacheData[fileIdx].hashStatus - of hashHasChanged: - result = true - of hashNotChanged: - result = false - of hashCached: - let newHash = secureHashFile(fileIdx.toFullPath) - result = newHash != gMemCacheData[fileIdx].hash - gMemCacheData[fileIdx].hash = newHash - updateStatus() - of hashNotTaken: - gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath) - result = true - updateStatus() - -proc doHash(fileIdx: int32) = - if gMemCacheData[fileIdx].hashStatus == hashNotTaken: - # echo "FIRST Hash: ", fileIdx.ToFilename - gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath) - -proc addDep(x: PSym, dep: int32) = - growCache gMemCacheData, dep - gMemCacheData[x.position].deps.safeAdd(dep) - -proc resetModule*(fileIdx: int32) = - # echo "HARD RESETTING ", fileIdx.toFilename - if fileIdx <% gMemCacheData.len: - gMemCacheData[fileIdx].needsRecompile = Yes - if fileIdx <% gCompiledModules.len: - gCompiledModules[fileIdx] = nil - if fileIdx <% cgendata.gModules.len: - cgendata.gModules[fileIdx] = nil - -proc resetModule*(module: PSym) = - let conflict = getModule(module.position.int32) - if conflict == nil: return - doAssert conflict == module - resetModule(module.position.int32) - initStrTable(module.tab) - -proc resetAllModules* = - for i in 0..gCompiledModules.high: - if gCompiledModules[i] != nil: - resetModule(i.int32) - resetPackageCache() - initStrTable(packageSyms) - # for m in cgenModules(): echo "CGEN MODULE FOUND" - -proc resetAllModulesHard* = - resetPackageCache() - gCompiledModules.setLen 0 - gMemCacheData.setLen 0 - magicsys.resetSysTypes() - initStrTable(packageSyms) - # XXX - #gOwners = @[] - -proc checkDepMem(fileIdx: int32): TNeedRecompile = - template markDirty = - resetModule(fileIdx) - return Yes - - if gFuzzyGraphChecking: - if gMemCacheData[fileIdx].needsRecompile != Maybe: - return gMemCacheData[fileIdx].needsRecompile - else: - # cycle detection: We claim that a cycle does no harm. - if gMemCacheData[fileIdx].needsRecompile == Probing: - return No + idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs + +when false: + type + TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled + THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged + + TModuleInMemory* = object + hash*: SecureHash + deps*: seq[int32] ## XXX: slurped files are currently not tracked + + needsRecompile*: TNeedRecompile + hashStatus*: THashStatus + + var + gCompiledModules: seq[PSym] = @[] + gMemCacheData*: seq[TModuleInMemory] = @[] + ## XXX: we should implement recycling of file IDs + ## if the user keeps renaming modules, the file IDs will keep growing + gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why. + + proc hashChanged(fileIdx: int32): bool = + internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len + + template updateStatus = + gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged + else: hashNotChanged + # echo "TESTING Hash: ", fileIdx.toFilename, " ", result + + case gMemCacheData[fileIdx].hashStatus + of hashHasChanged: + result = true + of hashNotChanged: + result = false + of hashCached: + let newHash = secureHashFile(fileIdx.toFullPath) + result = newHash != gMemCacheData[fileIdx].hash + gMemCacheData[fileIdx].hash = newHash + updateStatus() + of hashNotTaken: + gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath) + result = true + updateStatus() + + proc doHash(fileIdx: int32) = + if gMemCacheData[fileIdx].hashStatus == hashNotTaken: + # echo "FIRST Hash: ", fileIdx.ToFilename + gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath) + + proc resetModule*(fileIdx: int32) = + # echo "HARD RESETTING ", fileIdx.toFilename + if fileIdx <% gMemCacheData.len: + gMemCacheData[fileIdx].needsRecompile = Yes + if fileIdx <% gCompiledModules.len: + gCompiledModules[fileIdx] = nil + if fileIdx <% cgendata.gModules.len: + cgendata.gModules[fileIdx] = nil + + proc resetModule*(module: PSym) = + let conflict = getModule(module.position.int32) + if conflict == nil: return + doAssert conflict == module + resetModule(module.position.int32) + initStrTable(module.tab) + + proc resetAllModules* = + for i in 0..gCompiledModules.high: + if gCompiledModules[i] != nil: + resetModule(i.int32) + resetPackageCache() + # for m in cgenModules(): echo "CGEN MODULE FOUND" + + proc resetAllModulesHard* = + resetPackageCache() + gCompiledModules.setLen 0 + gMemCacheData.setLen 0 + magicsys.resetSysTypes() + # XXX + #gOwners = @[] + + proc checkDepMem(fileIdx: int32): TNeedRecompile = + template markDirty = + resetModule(fileIdx) + return Yes + + if gFuzzyGraphChecking: + if gMemCacheData[fileIdx].needsRecompile != Maybe: + return gMemCacheData[fileIdx].needsRecompile + else: + # cycle detection: We claim that a cycle does no harm. + if gMemCacheData[fileIdx].needsRecompile == Probing: + return No + + if optForceFullMake in gGlobalOptions or hashChanged(fileIdx): + markDirty() - if optForceFullMake in gGlobalOptions or hashChanged(fileIdx): - markDirty() + if gMemCacheData[fileIdx].deps != nil: + gMemCacheData[fileIdx].needsRecompile = Probing + for dep in gMemCacheData[fileIdx].deps: + let d = checkDepMem(dep) + if d in {Yes, Recompiled}: + # echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d + markDirty() - if gMemCacheData[fileIdx].deps != nil: - gMemCacheData[fileIdx].needsRecompile = Probing - for dep in gMemCacheData[fileIdx].deps: - let d = checkDepMem(dep) - if d in {Yes, Recompiled}: - # echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d - markDirty() + gMemCacheData[fileIdx].needsRecompile = No + return No - gMemCacheData[fileIdx].needsRecompile = No - return No +proc resetSystemArtifacts*() = + magicsys.resetSysTypes() -proc newModule(fileIdx: int32): PSym = +proc newModule(graph: ModuleGraph; fileIdx: int32): PSym = # We cannot call ``newSym`` here, because we have to circumvent the ID # mechanism, which we do in order to assign each module a persistent ID. new(result) @@ -143,20 +134,19 @@ proc newModule(fileIdx: int32): PSym = result.info = newLineInfo(fileIdx, 1, 1) let pack = getIdent(getPackageName(filename)) - var packSym = packageSyms.strTableGet(pack) + var packSym = graph.packageSyms.strTableGet(pack) if packSym == nil: let pck = getPackageName(filename) let pck2 = if pck.len > 0: pck else: "unknown" packSym = newSym(skPackage, getIdent(pck2), nil, result.info) initStrTable(packSym.tab) - packageSyms.strTableAdd(packSym) + graph.packageSyms.strTableAdd(packSym) result.owner = packSym result.position = fileIdx - growCache gMemCacheData, fileIdx - growCache gCompiledModules, fileIdx - gCompiledModules[result.position] = result + growCache graph.modules, fileIdx + graph.modules[result.position] = result incl(result.flags, sfUsed) initStrTable(result.tab) @@ -167,57 +157,66 @@ proc newModule(fileIdx: int32): PSym = # strTableIncl() for error corrections: discard strTableIncl(packSym.tab, result) -proc compileModule*(fileIdx: int32, flags: TSymFlags): PSym = - result = getModule(fileIdx) +proc compileModule*(graph: ModuleGraph; fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym = + result = graph.getModule(fileIdx) if result == nil: - growCache gMemCacheData, fileIdx - gMemCacheData[fileIdx].needsRecompile = Probing - result = newModule(fileIdx) - #var rd = handleSymbolFile(result) + #growCache gMemCacheData, fileIdx + #gMemCacheData[fileIdx].needsRecompile = Probing + result = newModule(graph, fileIdx) var rd: PRodReader result.flags = result.flags + flags if sfMainModule in result.flags: gMainPackageId = result.owner.id if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}: - rd = handleSymbolFile(result) + rd = handleSymbolFile(result, cache) if result.id < 0: - internalError("handleSymbolFile should have set the module\'s ID") + internalError("handleSymbolFile should have set the module's ID") return else: result.id = getID() - let validFile = processModule(result, if sfMainModule in flags and gProjectIsStdin: llStreamOpen(stdin) else: nil, rd) - if optCaasEnabled in gGlobalOptions: - gMemCacheData[fileIdx].compiledAt = gLastCmdTime - gMemCacheData[fileIdx].needsRecompile = Recompiled - if validFile: doHash fileIdx - else: - if checkDepMem(fileIdx) == Yes: - result = compileModule(fileIdx, flags) - else: - result = gCompiledModules[fileIdx] - -proc importModule*(s: PSym, fileIdx: int32): PSym {.procvar.} = + discard processModule(graph, result, + if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil, + rd, cache) + #if optCaasEnabled in gGlobalOptions: + # gMemCacheData[fileIdx].needsRecompile = Recompiled + # if validFile: doHash fileIdx + elif graph.isDirty(result): + result.flags.excl sfDirty + # reset module fields: + initStrTable(result.tab) + result.ast = nil + discard processModule(graph, result, + if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil, + nil, cache) + graph.markClientsDirty(fileIdx) + when false: + if checkDepMem(fileIdx) == Yes: + result = compileModule(fileIdx, cache, flags) + else: + result = gCompiledModules[fileIdx] + +proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: int32; + cache: IdentCache): PSym {.procvar.} = # this is called by the semantic checking phase - result = compileModule(fileIdx, {}) - if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx) + result = compileModule(graph, fileIdx, cache, {}) + graph.addDep(s, fileIdx) #if sfSystemModule in result.flags: # localError(result.info, errAttemptToRedefine, result.name.s) # restore the notes for outer module: gNotes = if s.owner.id == gMainPackageId: gMainPackageNotes else: ForeignPackageNotes -proc includeModule*(s: PSym, fileIdx: int32): PNode {.procvar.} = - result = syntaxes.parseFile(fileIdx) - if optCaasEnabled in gGlobalOptions: - growCache gMemCacheData, fileIdx - addDep(s, fileIdx) - doHash(fileIdx) +proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: int32; + cache: IdentCache): PNode {.procvar.} = + result = syntaxes.parseFile(fileIdx, cache) + graph.addDep(s, fileIdx) + graph.addIncludeDep(s.position.int32, fileIdx) -proc compileSystemModule* = +proc compileSystemModule*(graph: ModuleGraph; cache: IdentCache) = if magicsys.systemModule == nil: systemFileIdx = fileInfoIdx(options.libpath/"system.nim") - discard compileModule(systemFileIdx, {sfSystemModule}) + discard graph.compileModule(systemFileIdx, cache, {sfSystemModule}) proc wantMainModule* = if gProjectFull.len == 0: @@ -227,18 +226,20 @@ proc wantMainModule* = passes.gIncludeFile = includeModule passes.gImportModule = importModule -proc compileProject*(projectFileIdx = -1'i32) = +proc compileProject*(graph: ModuleGraph; cache: IdentCache; + projectFileIdx = -1'i32) = wantMainModule() let systemFileIdx = fileInfoIdx(options.libpath / "system.nim") let projectFile = if projectFileIdx < 0: gProjectMainIdx else: projectFileIdx + graph.importStack.add projectFile if projectFile == systemFileIdx: - discard compileModule(projectFile, {sfMainModule, sfSystemModule}) + discard graph.compileModule(projectFile, cache, {sfMainModule, sfSystemModule}) else: - compileSystemModule() - discard compileModule(projectFile, {sfMainModule}) + graph.compileSystemModule(cache) + discard graph.compileModule(projectFile, cache, {sfMainModule}) -proc makeModule*(filename: string): PSym = - result = newModule(fileInfoIdx filename) +proc makeModule*(graph: ModuleGraph; filename: string): PSym = + result = graph.newModule(fileInfoIdx filename) result.id = getID() -proc makeStdinModule*(): PSym = makeModule"stdin" +proc makeStdinModule*(graph: ModuleGraph): PSym = graph.makeModule"stdin" diff --git a/compiler/msgs.nim b/compiler/msgs.nim index fd0aafccb..94b0bee00 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -35,7 +35,7 @@ type errNoneSpeedOrSizeExpectedButXFound, errGuiConsoleOrLibExpectedButXFound, errUnknownOS, errUnknownCPU, errGenOutExpectedButXFound, errArgsNeedRunOption, errInvalidMultipleAsgn, errColonOrEqualsExpected, - errExprExpected, errUndeclaredIdentifier, errUndeclaredField, + errExprExpected, errUndeclaredField, errUndeclaredRoutine, errUseQualifier, errTypeExpected, errSystemNeeds, errExecutionOfProgramFailed, errNotOverloadable, @@ -197,7 +197,6 @@ const errInvalidMultipleAsgn: "multiple assignment is not allowed", errColonOrEqualsExpected: "\':\' or \'=\' expected, but found \'$1\'", errExprExpected: "expression expected, but found \'$1\'", - errUndeclaredIdentifier: "undeclared identifier: \'$1\'", errUndeclaredField: "undeclared field: \'$1\'", errUndeclaredRoutine: "attempting to call undeclared routine: \'$1\'", errUseQualifier: "ambiguous identifier: \'$1\' -- use a qualifier", @@ -215,7 +214,7 @@ const errOrdinalTypeExpected: "ordinal type expected", errOrdinalOrFloatTypeExpected: "ordinal or float type expected", errOverOrUnderflow: "over- or underflow", - errCannotEvalXBecauseIncompletelyDefined: "cannot evalutate '$1' because type is not defined completely", + errCannotEvalXBecauseIncompletelyDefined: "cannot evaluate '$1' because type is not defined completely", errChrExpectsRange0_255: "\'chr\' expects an int in the range 0..255", errDynlibRequiresExportc: "\'dynlib\' requires \'exportc\'", errUndeclaredFieldX: "undeclared field: \'$1\'", @@ -676,9 +675,8 @@ proc getInfoContext*(index: int): TLineInfo = if i >=% L: result = unknownLineInfo() else: result = msgContext[i] -proc toFilename*(fileIdx: int32): string = - if fileIdx < 0: result = "???" - else: result = fileInfos[fileIdx].projPath +template toFilename*(fileIdx: int32): string = + (if fileIdx < 0: "???" else: fileInfos[fileIdx].projPath) proc toFullPath*(fileIdx: int32): string = if fileIdx < 0: result = "???" diff --git a/compiler/nim.nim b/compiler/nim.nim index a58afd593..35afecf20 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -13,10 +13,15 @@ when defined(gcc) and defined(windows): else: {.link: "icons/nim_icon.o".} +when defined(amd64) and defined(windows) and defined(vcc): + {.link: "icons/nim-amd64-windows-vcc.res".} +when defined(i386) and defined(windows) and defined(vcc): + {.link: "icons/nim-i386-windows-vcc.res".} + import commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes, extccomp, strutils, os, osproc, platform, main, parseopt, service, - nodejs, scriptconfig + nodejs, scriptconfig, idents, modulegraphs when hasTinyCBackend: import tccgen @@ -32,7 +37,7 @@ proc prependCurDir(f: string): string = else: result = f -proc handleCmdLine() = +proc handleCmdLine(cache: IdentCache) = if paramCount() == 0: writeCommandLineUsage() else: @@ -41,7 +46,7 @@ proc handleCmdLine() = if gProjectName == "-": gProjectName = "stdinfile" gProjectFull = "stdinfile" - gProjectPath = getCurrentDir() + gProjectPath = canonicalizePath getCurrentDir() gProjectIsStdin = true elif gProjectName != "": try: @@ -49,26 +54,26 @@ proc handleCmdLine() = except OSError: gProjectFull = gProjectName let p = splitFile(gProjectFull) - gProjectPath = p.dir + gProjectPath = canonicalizePath p.dir gProjectName = p.name else: - gProjectPath = getCurrentDir() + gProjectPath = canonicalizePath getCurrentDir() loadConfigs(DefaultConfig) # load all config files let scriptFile = gProjectFull.changeFileExt("nims") if fileExists(scriptFile): - runNimScript(scriptFile, freshDefines=false) + runNimScript(cache, scriptFile, freshDefines=false) # 'nim foo.nims' means to just run the NimScript file and do nothing more: if scriptFile == gProjectFull: return elif fileExists(gProjectPath / "config.nims"): # directory wide NimScript file - runNimScript(gProjectPath / "config.nims", freshDefines=false) + runNimScript(cache, gProjectPath / "config.nims", freshDefines=false) # now process command line arguments again, because some options in the # command line can overwite the config file's settings extccomp.initVars() processCmdLine(passCmd2, "") if options.command == "": rawMessage(errNoCommand, command) - mainCommand() + mainCommand(newModuleGraph(), cache) if optHints in gOptions and hintGCStats in gNotes: echo(GC_getStatistics()) #echo(GC_getStatistics()) if msgs.gErrorCounter == 0: @@ -112,5 +117,5 @@ when compileOption("gc", "v2") or compileOption("gc", "refc"): condsyms.initDefines() when not defined(selftest): - handleCmdLine() + handleCmdLine(newIdentCache()) msgQuit(int8(msgs.gErrorCounter > 0)) diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 496bd0123..4bf2fbc9a 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -156,7 +156,7 @@ proc checkSymbol(L: TLexer, tok: TToken) = lexMessage(L, errIdentifierExpected, tokToStr(tok)) proc parseAssignment(L: var TLexer, tok: var TToken) = - if tok.ident.id == getIdent("-").id or tok.ident.id == getIdent("--").id: + if tok.ident.s == "-" or tok.ident.s == "--": confTok(L, tok) # skip unnecessary prefix var info = getLineInfo(L, tok) # save for later in case of an error checkSymbol(L, tok) @@ -179,14 +179,14 @@ proc parseAssignment(L: var TLexer, tok: var TToken) = if tok.tokType == tkBracketRi: confTok(L, tok) else: lexMessage(L, errTokenExpected, "']'") add(val, ']') - let percent = tok.ident.id == getIdent("%=").id + let percent = tok.ident != nil and tok.ident.s == "%=" if tok.tokType in {tkColon, tkEquals} or percent: if len(val) > 0: add(val, ':') confTok(L, tok) # skip ':' or '=' or '%' checkSymbol(L, tok) add(val, tokToStr(tok)) confTok(L, tok) # skip symbol - while tok.ident != nil and tok.ident.id == getIdent("&").id: + while tok.ident != nil and tok.ident.s == "&": confTok(L, tok) checkSymbol(L, tok) add(val, tokToStr(tok)) @@ -197,7 +197,7 @@ proc parseAssignment(L: var TLexer, tok: var TToken) = else: processSwitch(s, val, passPP, info) -proc readConfigFile(filename: string) = +proc readConfigFile(filename: string; cache: IdentCache) = var L: TLexer tok: TToken @@ -205,7 +205,7 @@ proc readConfigFile(filename: string) = stream = llStreamOpen(filename, fmRead) if stream != nil: initToken(tok) - openLexer(L, filename, stream) + openLexer(L, filename, stream, cache) tok.tokType = tkEof # to avoid a pointless warning confTok(L, tok) # read in the first token while tok.tokType != tkEof: parseAssignment(L, tok) @@ -225,22 +225,22 @@ proc getSystemConfigPath(filename: string): string = if not existsFile(result): result = joinPath([p, "etc", filename]) if not existsFile(result): result = "/etc/" & filename -proc loadConfigs*(cfg: string) = +proc loadConfigs*(cfg: string; cache: IdentCache) = setDefaultLibpath() if optSkipConfigFile notin gGlobalOptions: - readConfigFile(getSystemConfigPath(cfg)) + readConfigFile(getSystemConfigPath(cfg), cache) if optSkipUserConfigFile notin gGlobalOptions: - readConfigFile(getUserConfigPath(cfg)) + readConfigFile(getUserConfigPath(cfg), cache) var pd = if gProjectPath.len > 0: gProjectPath else: getCurrentDir() if optSkipParentConfigFiles notin gGlobalOptions: for dir in parentDirs(pd, fromRoot=true, inclusive=false): - readConfigFile(dir / cfg) + readConfigFile(dir / cfg, cache) if optSkipProjConfigFile notin gGlobalOptions: - readConfigFile(pd / cfg) + readConfigFile(pd / cfg, cache) if gProjectName.len != 0: # new project wide config file: @@ -251,4 +251,8 @@ proc loadConfigs*(cfg: string) = projectConfig = changeFileExt(gProjectFull, "nimrod.cfg") if fileExists(projectConfig): rawMessage(warnDeprecated, projectConfig) - readConfigFile(projectConfig) + readConfigFile(projectConfig, cache) + +proc loadConfigs*(cfg: string) = + # for backwards compatibility only. + loadConfigs(cfg, newIdentCache()) diff --git a/compiler/nimeval.nim b/compiler/nimeval.nim index 2bddb76e7..2872bdade 100644 --- a/compiler/nimeval.nim +++ b/compiler/nimeval.nim @@ -8,10 +8,9 @@ # ## exposes the Nim VM to clients. - import ast, modules, passes, passaux, condsyms, - options, nimconf, lists, sem, semdata, llstream, vm + options, nimconf, lists, sem, semdata, llstream, vm, modulegraphs, idents proc execute*(program: string) = passes.gIncludeFile = includeModule @@ -27,7 +26,9 @@ proc execute*(program: string) = registerPass(evalPass) appendStr(searchPaths, options.libpath) - compileSystemModule() - var m = makeStdinModule() + var graph = newModuleGraph() + var cache = newIdentCache() + var m = makeStdinModule(graph) incl(m.flags, sfMainModule) - processModule(m, llStreamOpen(program), nil) + compileSystemModule(graph,cache) + processModule(graph,m, llStreamOpen(program), nil, cache) diff --git a/compiler/nimfix/nimfix.nim b/compiler/nimfix/nimfix.nim index 39436702f..b4007cdaf 100644 --- a/compiler/nimfix/nimfix.nim +++ b/compiler/nimfix/nimfix.nim @@ -73,7 +73,7 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) = of "auto": gStyleCheck = StyleCheck.Auto else: localError(gCmdLineInfo, errOnOrOffExpected) of "wholeproject": gOnlyMainfile = false - of "besteffort": msgs.gErrorMax = high(int) # dont stop after first error + of "besteffort": msgs.gErrorMax = high(int) # don't stop after first error else: processSwitch(pass, p) of cmdArgument: diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim deleted file mode 100644 index 2be368d68..000000000 --- a/compiler/nimsuggest/nimsuggest.nim +++ /dev/null @@ -1,12 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Nimsuggest has been moved to https://github.com/nim-lang/nimsuggest - -{.error: "This project has moved to the following repo: https://github.com/nim-lang/nimsuggest".} diff --git a/compiler/nversion.nim b/compiler/nversion.nim index d69e1e553..4d4fe6c95 100644 --- a/compiler/nversion.nim +++ b/compiler/nversion.nim @@ -13,5 +13,5 @@ const MaxSetElements* = 1 shl 16 # (2^16) to support unicode character sets? VersionAsString* = system.NimVersion - RodFileVersion* = "1221" # modify this if the rod-format changes! + RodFileVersion* = "1222" # modify this if the rod-format changes! diff --git a/compiler/options.nim b/compiler/options.nim index 7cf707945..9edafb17a 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -34,7 +34,8 @@ type # please make sure we have under 32 options optProfiler, # profiler turned on optImplicitStatic, # optimization: implicit at compile time # evaluation - optPatterns # en/disable pattern matching + optPatterns, # en/disable pattern matching + optMemTracker TOptions* = set[TOption] TGlobalOption* = enum # **keep binary compatible** @@ -231,10 +232,10 @@ proc canonicalizePath*(path: string): string = proc shortenDir*(dir: string): string = ## returns the interesting part of a dir - var prefix = getPrefixDir() & DirSep + var prefix = gProjectPath & DirSep if startsWith(dir, prefix): return substr(dir, len(prefix)) - prefix = gProjectPath & DirSep + prefix = getPrefixDir() & DirSep if startsWith(dir, prefix): return substr(dir, len(prefix)) result = dir diff --git a/compiler/parampatterns.nim b/compiler/parampatterns.nim index f8f1f355c..c51d406ac 100644 --- a/compiler/parampatterns.nim +++ b/compiler/parampatterns.nim @@ -230,6 +230,8 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult # builtin slice keeps lvalue-ness: if getMagic(n) in {mArrGet, mSlice}: result = isAssignable(owner, n.sons[1], isUnsafeAddr) + elif n.typ != nil and n.typ.kind == tyVar: + result = arLValue of nkStmtList, nkStmtListExpr: if n.typ != nil: result = isAssignable(owner, n.lastSon, isUnsafeAddr) diff --git a/compiler/parser.nim b/compiler/parser.nim index 40862eb63..902bf0fcb 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -28,15 +28,15 @@ import llstream, lexer, idents, strutils, ast, astalgo, msgs type - TParser*{.final.} = object # A TParser object represents a module that - # is being parsed - currInd: int # current indentation level + TParser*{.final.} = object # A TParser object represents a file that + # is being parsed + currInd: int # current indentation level firstTok, strongSpaces: bool # Has the first token been read? # Is strongSpaces on? - lex*: TLexer # The lexer that is used for parsing - tok*: TToken # The current token - inPragma: int # Pragma level - inSemiStmtList: int + lex*: TLexer # The lexer that is used for parsing + tok*: TToken # The current token + inPragma*: int # Pragma level + inSemiStmtList*: int proc parseAll*(p: var TParser): PNode proc closeParser*(p: var TParser) @@ -73,18 +73,20 @@ proc getTok(p: var TParser) = rawGetTok(p.lex, p.tok) proc openParser*(p: var TParser, fileIdx: int32, inputStream: PLLStream, + cache: IdentCache; strongSpaces=false) = ## Open a parser, using the given arguments to set up its internal state. ## initToken(p.tok) - openLexer(p.lex, fileIdx, inputStream) + openLexer(p.lex, fileIdx, inputStream, cache) getTok(p) # read the first token p.firstTok = true p.strongSpaces = strongSpaces proc openParser*(p: var TParser, filename: string, inputStream: PLLStream, + cache: IdentCache; strongSpaces=false) = - openParser(p, filename.fileInfoIdx, inputStream, strongSpaces) + openParser(p, filename.fileInfoIdx, inputStream, cache, strongSpaces) proc closeParser(p: var TParser) = ## Close a parser, freeing up its resources. @@ -320,9 +322,9 @@ proc parseSymbol(p: var TParser, allowNil = false): PNode = tkParLe..tkParDotRi}: accm.add(tokToStr(p.tok)) getTok(p) - result.add(newIdentNodeP(getIdent(accm), p)) + result.add(newIdentNodeP(p.lex.cache.getIdent(accm), p)) of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCharLit: - result.add(newIdentNodeP(getIdent(tokToStr(p.tok)), p)) + result.add(newIdentNodeP(p.lex.cache.getIdent(tokToStr(p.tok)), p)) getTok(p) else: parMessage(p, errIdentifierExpected, p.tok) @@ -923,7 +925,7 @@ proc parseParamList(p: var TParser, retColon = true): PNode = optPar(p) eat(p, tkParRi) let hasRet = if retColon: p.tok.tokType == tkColon - else: p.tok.tokType == tkOpr and identEq(p.tok.ident, "->") + else: p.tok.tokType == tkOpr and p.tok.ident.s == "->" if hasRet and p.tok.indent < 0: getTok(p) optInd(p, result) @@ -2023,7 +2025,8 @@ proc parseTopLevelStmt(p: var TParser): PNode = if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) break -proc parseString*(s: string; filename: string = ""; line: int = 0; +proc parseString*(s: string; cache: IdentCache; filename: string = ""; + line: int = 0; errorHandler: TErrorHandler = nil): PNode = ## Parses a string into an AST, returning the top node. ## `filename` and `line`, although optional, provide info so that the @@ -2036,7 +2039,7 @@ proc parseString*(s: string; filename: string = ""; line: int = 0; # XXX for now the builtin 'parseStmt/Expr' functions do not know about strong # spaces... parser.lex.errorHandler = errorHandler - openParser(parser, filename, stream, false) + openParser(parser, filename, stream, cache, false) result = parser.parseAll closeParser(parser) diff --git a/compiler/passaux.nim b/compiler/passaux.nim index d4361a671..eeaf12953 100644 --- a/compiler/passaux.nim +++ b/compiler/passaux.nim @@ -10,9 +10,11 @@ ## implements some little helper passes import - strutils, ast, astalgo, passes, msgs, options, idgen + strutils, ast, astalgo, passes, idents, msgs, options, idgen -proc verboseOpen(s: PSym): PPassContext = +from modulegraphs import ModuleGraph + +proc verboseOpen(graph: ModuleGraph; s: PSym; cache: IdentCache): PPassContext = #MessageOut('compiling ' + s.name.s); result = nil # we don't need a context rawMessage(hintProcessing, s.name.s) diff --git a/compiler/passes.nim b/compiler/passes.nim index b7642e3e4..3cc15147e 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -13,7 +13,7 @@ import strutils, lists, options, ast, astalgo, llstream, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math, magicsys, nversion, - nimsets, syntaxes, times, rodread, idgen + nimsets, syntaxes, times, rodread, idgen, modulegraphs type TPassContext* = object of RootObj # the pass's context @@ -21,9 +21,9 @@ type PPassContext* = ref TPassContext - TPassOpen* = proc (module: PSym): PPassContext {.nimcall.} + TPassOpen* = proc (graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext {.nimcall.} TPassOpenCached* = - proc (module: PSym, rd: PRodReader): PPassContext {.nimcall.} + proc (graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext {.nimcall.} TPassClose* = proc (p: PPassContext, n: PNode): PNode {.nimcall.} TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.} @@ -48,8 +48,8 @@ proc makePass*(open: TPassOpen = nil, # the semantic checker needs these: var - gImportModule*: proc (m: PSym, fileIdx: int32): PSym {.nimcall.} - gIncludeFile*: proc (m: PSym, fileIdx: int32): PNode {.nimcall.} + gImportModule*: proc (graph: ModuleGraph; m: PSym, fileIdx: int32; cache: IdentCache): PSym {.nimcall.} + gIncludeFile*: proc (graph: ModuleGraph; m: PSym, fileIdx: int32; cache: IdentCache): PNode {.nimcall.} # implementation @@ -90,28 +90,32 @@ proc registerPass*(p: TPass) = gPasses[gPassesLen] = p inc(gPassesLen) -proc carryPass*(p: TPass, module: PSym, m: TPassData): TPassData = - var c = p.open(module) +proc carryPass*(g: ModuleGraph; p: TPass, module: PSym; cache: IdentCache; + m: TPassData): TPassData = + var c = p.open(g, module, cache) result.input = p.process(c, m.input) result.closeOutput = if p.close != nil: p.close(c, m.closeOutput) else: m.closeOutput -proc carryPasses*(nodes: PNode, module: PSym, passes: TPasses) = +proc carryPasses*(g: ModuleGraph; nodes: PNode, module: PSym; + cache: IdentCache; passes: TPasses) = var passdata: TPassData passdata.input = nodes for pass in passes: - passdata = carryPass(pass, module, passdata) + passdata = carryPass(g, pass, module, cache, passdata) -proc openPasses(a: var TPassContextArray, module: PSym) = +proc openPasses(g: ModuleGraph; a: var TPassContextArray; + module: PSym; cache: IdentCache) = for i in countup(0, gPassesLen - 1): if not isNil(gPasses[i].open): - a[i] = gPasses[i].open(module) + a[i] = gPasses[i].open(g, module, cache) else: a[i] = nil -proc openPassesCached(a: var TPassContextArray, module: PSym, rd: PRodReader) = +proc openPassesCached(g: ModuleGraph; a: var TPassContextArray, module: PSym, + rd: PRodReader) = for i in countup(0, gPassesLen - 1): if not isNil(gPasses[i].openCached): - a[i] = gPasses[i].openCached(module, rd) + a[i] = gPasses[i].openCached(g, module, rd) if a[i] != nil: a[i].fromCache = true else: @@ -145,24 +149,35 @@ proc closePassesCached(a: var TPassContextArray) = m = gPasses[i].close(a[i], m) a[i] = nil # free the memory here +proc resolveMod(module, relativeTo: string): int32 = + let fullPath = findModule(module, relativeTo) + if fullPath.len == 0: + result = InvalidFileIDX + else: + result = fullPath.fileInfoIdx + proc processImplicits(implicits: seq[string], nodeKind: TNodeKind, - a: var TPassContextArray) = + a: var TPassContextArray; m: PSym) = + # XXX fixme this should actually be relative to the config file! + let relativeTo = m.info.toFullPath for module in items(implicits): - var importStmt = newNodeI(nodeKind, gCmdLineInfo) - var str = newStrNode(nkStrLit, module) - str.info = gCmdLineInfo - importStmt.addSon str - if not processTopLevelStmt(importStmt, a): break - -proc processModule*(module: PSym, stream: PLLStream, - rd: PRodReader): bool {.discardable.} = + # implicit imports should not lead to a module importing itself + if m.position != resolveMod(module, relativeTo): + var importStmt = newNodeI(nodeKind, gCmdLineInfo) + var str = newStrNode(nkStrLit, module) + str.info = gCmdLineInfo + importStmt.addSon str + if not processTopLevelStmt(importStmt, a): break + +proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream, + rd: PRodReader; cache: IdentCache): bool {.discardable.} = var p: TParsers a: TPassContextArray s: PLLStream fileIdx = module.fileIdx if rd == nil: - openPasses(a, module) + openPasses(graph, a, module, cache) if stream == nil: let filename = fileIdx.toFullPathConsiderDirty s = llStreamOpen(filename, fmRead) @@ -172,15 +187,15 @@ proc processModule*(module: PSym, stream: PLLStream, else: s = stream while true: - openParsers(p, fileIdx, s) + openParsers(p, fileIdx, s, cache) if sfSystemModule notin module.flags: # XXX what about caching? no processing then? what if I change the # modules to include between compilation runs? we'd need to track that # in ROD files. I think we should enable this feature only # for the interactive mode. - processImplicits implicitImports, nkImportStmt, a - processImplicits implicitIncludes, nkIncludeStmt, a + processImplicits implicitImports, nkImportStmt, a, module + processImplicits implicitIncludes, nkIncludeStmt, a, module while true: var n = parseTopLevelStmt(p) @@ -202,7 +217,7 @@ proc processModule*(module: PSym, stream: PLLStream, # id synchronization point for more consistent code generation: idSynchronizationPoint(1000) else: - openPassesCached(a, module, rd) + openPassesCached(graph, a, module, rd) var n = loadInitSection(rd) for i in countup(0, sonsLen(n) - 1): processTopLevelStmtCached(n.sons[i], a) closePassesCached(a) diff --git a/compiler/pbraces.nim b/compiler/pbraces.nim index 00f83a11e..df9204be6 100644 --- a/compiler/pbraces.nim +++ b/compiler/pbraces.nim @@ -1,18 +1,1780 @@ # # # The Nim Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2016 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # +# This module implements the parser of the braces Nim syntax. + import - llstream, lexer, parser, idents, strutils, ast, msgs + llstream, lexer, idents, strutils, ast, astalgo, msgs + +from parser import TParser + +proc getTok(p: var TParser) = + ## Get the next token from the parser's lexer, and store it in the parser's + ## `tok` member. + rawGetTok(p.lex, p.tok) + +proc openParser*(p: var TParser, fileIdx: int32, inputStream: PLLStream; + cache: IdentCache) = + ## Open a parser, using the given arguments to set up its internal state. + ## + initToken(p.tok) + openLexer(p.lex, fileIdx, inputStream, cache) + getTok(p) # read the first token + p.lex.allowTabs = true + +proc openParser*(p: var TParser, filename: string, inputStream: PLLStream; + cache: IdentCache) = + openParser(p, filename.fileInfoIdx, inputStream, cache) + +proc closeParser*(p: var TParser) = + ## Close a parser, freeing up its resources. + closeLexer(p.lex) + +proc parMessage(p: TParser, msg: TMsgKind, arg = "") = + ## Produce and emit the parser message `arg` to output. + lexMessageTok(p.lex, msg, p.tok, arg) + +proc parMessage(p: TParser, msg: TMsgKind, tok: TToken) = + ## Produce and emit a parser message to output about the token `tok` + parMessage(p, msg, prettyTok(tok)) + +proc rawSkipComment(p: var TParser, node: PNode) = + if p.tok.tokType == tkComment: + if node != nil: + if node.comment == nil: node.comment = "" + add(node.comment, p.tok.literal) + else: + parMessage(p, errInternal, "skipComment") + getTok(p) + +proc skipComment(p: var TParser, node: PNode) = + rawSkipComment(p, node) + +proc flexComment(p: var TParser, node: PNode) = + rawSkipComment(p, node) + +proc skipInd(p: var TParser) = discard +proc optPar(p: var TParser) = discard + +proc optInd(p: var TParser, n: PNode) = + skipComment(p, n) + +proc getTokNoInd(p: var TParser) = + getTok(p) + +proc expectIdentOrKeyw(p: TParser) = + if p.tok.tokType != tkSymbol and not isKeyword(p.tok.tokType): + lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok)) + +proc expectIdent(p: TParser) = + if p.tok.tokType != tkSymbol: + lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok)) + +proc eat(p: var TParser, tokType: TTokType) = + ## Move the parser to the next token if the current token is of type + ## `tokType`, otherwise error. + if p.tok.tokType == tokType: + getTok(p) + else: + lexMessageTok(p.lex, errTokenExpected, p.tok, TokTypeToStr[tokType]) + +proc parLineInfo(p: TParser): TLineInfo = + ## Retrieve the line information associated with the parser's current state. + result = getLineInfo(p.lex, p.tok) + +proc indAndComment(p: var TParser, n: PNode) = + rawSkipComment(p, n) + +proc newNodeP(kind: TNodeKind, p: TParser): PNode = + result = newNodeI(kind, parLineInfo(p)) + +proc newIntNodeP(kind: TNodeKind, intVal: BiggestInt, p: TParser): PNode = + result = newNodeP(kind, p) + result.intVal = intVal + +proc newFloatNodeP(kind: TNodeKind, floatVal: BiggestFloat, + p: TParser): PNode = + result = newNodeP(kind, p) + result.floatVal = floatVal + +proc newStrNodeP(kind: TNodeKind, strVal: string, p: TParser): PNode = + result = newNodeP(kind, p) + result.strVal = strVal + +proc newIdentNodeP(ident: PIdent, p: TParser): PNode = + result = newNodeP(nkIdent, p) + result.ident = ident + +proc parseExpr(p: var TParser): PNode +proc parseStmt(p: var TParser): PNode +proc parseTypeDesc(p: var TParser): PNode +proc parseDoBlocks(p: var TParser, call: PNode) +proc parseParamList(p: var TParser, retColon = true): PNode +proc parseStmtPragma(p: var TParser): PNode +proc parseCase(p: var TParser): PNode +proc parseTry(p: var TParser): PNode + +proc isSigilLike(tok: TToken): bool {.inline.} = + result = tok.tokType == tkOpr and tok.ident.s[0] == '@' + +proc isAt(tok: TToken): bool {.inline.} = + tok.tokType == tkOpr and tok.ident.s == "@" and tok.strongSpaceB == 0 + +proc isRightAssociative(tok: TToken): bool {.inline.} = + ## Determines whether the token is right assocative. + result = tok.tokType == tkOpr and tok.ident.s[0] == '^' + # or (let L = tok.ident.s.len; L > 1 and tok.ident.s[L-1] == '>')) + +proc getPrecedence(tok: TToken): int = + ## Calculates the precedence of the given token. + template considerStrongSpaces(x): untyped = x + + case tok.tokType + of tkOpr: + let L = tok.ident.s.len + let relevantChar = tok.ident.s[0] + + # arrow like? + if L > 1 and tok.ident.s[L-1] == '>' and + tok.ident.s[L-2] in {'-', '~', '='}: return considerStrongSpaces(1) + + template considerAsgn(value: untyped) = + result = if tok.ident.s[L-1] == '=': 1 else: value + + case relevantChar + of '$', '^': considerAsgn(10) + of '*', '%', '/', '\\': considerAsgn(9) + of '~': result = 8 + of '+', '-', '|': considerAsgn(8) + of '&': considerAsgn(7) + of '=', '<', '>', '!': result = 5 + of '.': considerAsgn(6) + of '?': result = 2 + else: considerAsgn(2) + of tkDiv, tkMod, tkShl, tkShr: result = 9 + of tkIn, tkNotin, tkIs, tkIsnot, tkNot, tkOf, tkAs: result = 5 + of tkDotDot: result = 6 + of tkAnd: result = 4 + of tkOr, tkXor, tkPtr, tkRef: result = 3 + else: return -10 + result = considerStrongSpaces(result) + +proc isOperator(tok: TToken): bool = + ## Determines if the given token is an operator type token. + tok.tokType in {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs, + tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor} + +proc isUnary(p: TParser): bool = + ## Check if the current parser token is a unary operator + if p.tok.tokType in {tkOpr, tkDotDot}: + result = true + +proc checkBinary(p: TParser) {.inline.} = + ## Check if the current parser token is a binary operator. + # we don't check '..' here as that's too annoying + discard + +#| module = stmt ^* (';' / IND{=}) +#| +#| comma = ',' COMMENT? +#| semicolon = ';' COMMENT? +#| colon = ':' COMMENT? +#| colcom = ':' COMMENT? +#| +#| operator = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 +#| | 'or' | 'xor' | 'and' +#| | 'is' | 'isnot' | 'in' | 'notin' | 'of' +#| | 'div' | 'mod' | 'shl' | 'shr' | 'not' | 'static' | '..' +#| +#| prefixOperator = operator +#| +#| optInd = COMMENT? +#| optPar = (IND{>} | IND{=})? +#| +#| simpleExpr = arrowExpr (OP0 optInd arrowExpr)* +#| arrowExpr = assignExpr (OP1 optInd assignExpr)* +#| assignExpr = orExpr (OP2 optInd orExpr)* +#| orExpr = andExpr (OP3 optInd andExpr)* +#| andExpr = cmpExpr (OP4 optInd cmpExpr)* +#| cmpExpr = sliceExpr (OP5 optInd sliceExpr)* +#| sliceExpr = ampExpr (OP6 optInd ampExpr)* +#| ampExpr = plusExpr (OP7 optInd plusExpr)* +#| plusExpr = mulExpr (OP8 optInd mulExpr)* +#| mulExpr = dollarExpr (OP9 optInd dollarExpr)* +#| dollarExpr = primary (OP10 optInd primary)* + +proc colcom(p: var TParser, n: PNode) = + skipComment(p, n) + +proc parseSymbol(p: var TParser, allowNil = false): PNode = + #| symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`' + #| | IDENT | 'addr' | 'type' + case p.tok.tokType + of tkSymbol, tkAddr, tkType: + result = newIdentNodeP(p.tok.ident, p) + getTok(p) + of tkAccent: + result = newNodeP(nkAccQuoted, p) + getTok(p) + while true: + case p.tok.tokType + of tkAccent: + if result.len == 0: + parMessage(p, errIdentifierExpected, p.tok) + break + of tkOpr, tkDot, tkDotDot, tkEquals, tkParLe..tkParDotRi: + var accm = "" + while p.tok.tokType in {tkOpr, tkDot, tkDotDot, tkEquals, + tkParLe..tkParDotRi}: + accm.add(tokToStr(p.tok)) + getTok(p) + result.add(newIdentNodeP(p.lex.cache.getIdent(accm), p)) + of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCharLit: + result.add(newIdentNodeP(p.lex.cache.getIdent(tokToStr(p.tok)), p)) + getTok(p) + else: + parMessage(p, errIdentifierExpected, p.tok) + break + eat(p, tkAccent) + else: + if allowNil and p.tok.tokType == tkNil: + result = newNodeP(nkNilLit, p) + getTok(p) + else: + parMessage(p, errIdentifierExpected, p.tok) + # BUGFIX: We must consume a token here to prevent endless loops! + # But: this really sucks for idetools and keywords, so we don't do it + # if it is a keyword: + if not isKeyword(p.tok.tokType): getTok(p) + result = ast.emptyNode + +proc colonOrEquals(p: var TParser, a: PNode): PNode = + if p.tok.tokType == tkColon: + result = newNodeP(nkExprColonExpr, p) + getTok(p) + #optInd(p, result) + addSon(result, a) + addSon(result, parseExpr(p)) + elif p.tok.tokType == tkEquals: + result = newNodeP(nkExprEqExpr, p) + getTok(p) + #optInd(p, result) + addSon(result, a) + addSon(result, parseExpr(p)) + else: + result = a + +proc exprColonEqExpr(p: var TParser): PNode = + #| exprColonEqExpr = expr (':'|'=' expr)? + var a = parseExpr(p) + result = colonOrEquals(p, a) + +proc exprList(p: var TParser, endTok: TTokType, result: PNode) = + #| exprList = expr ^+ comma + getTok(p) + optInd(p, result) + while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof): + var a = parseExpr(p) + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + +proc dotExpr(p: var TParser, a: PNode): PNode = + #| dotExpr = expr '.' optInd symbol + var info = p.parLineInfo + getTok(p) + result = newNodeI(nkDotExpr, info) + optInd(p, result) + addSon(result, a) + addSon(result, parseSymbol(p)) + +proc qualifiedIdent(p: var TParser): PNode = + #| qualifiedIdent = symbol ('.' optInd symbol)? + result = parseSymbol(p) + if p.tok.tokType == tkDot: result = dotExpr(p, result) + +proc exprColonEqExprListAux(p: var TParser, endTok: TTokType, result: PNode) = + assert(endTok in {tkCurlyLe, tkCurlyRi, tkCurlyDotRi, tkBracketRi, tkParRi}) + getTok(p) + optInd(p, result) + while p.tok.tokType != endTok and p.tok.tokType != tkEof: + var a = exprColonEqExpr(p) + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + skipComment(p, a) + optPar(p) + eat(p, endTok) + +proc exprColonEqExprList(p: var TParser, kind: TNodeKind, + endTok: TTokType): PNode = + #| exprColonEqExprList = exprColonEqExpr (comma exprColonEqExpr)* (comma)? + result = newNodeP(kind, p) + exprColonEqExprListAux(p, endTok, result) + +proc setOrTableConstr(p: var TParser): PNode = + result = newNodeP(nkCurly, p) + getTok(p) + optInd(p, result) + if p.tok.tokType == tkColon: + getTok(p) # skip ':' + result.kind = nkTableConstr + else: + while p.tok.tokType notin {tkBracketDotRi, tkEof}: + var a = exprColonEqExpr(p) + if a.kind == nkExprColonExpr: result.kind = nkTableConstr + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + skipComment(p, a) + optPar(p) + eat(p, tkBracketDotRi) + +proc parseCast(p: var TParser): PNode = + #| castExpr = 'cast' '[' optInd typeDesc optPar ']' '(' optInd expr optPar ')' + result = newNodeP(nkCast, p) + getTok(p) + eat(p, tkBracketLe) + optInd(p, result) + addSon(result, parseTypeDesc(p)) + optPar(p) + eat(p, tkBracketRi) + eat(p, tkParLe) + optInd(p, result) + addSon(result, parseExpr(p)) + optPar(p) + eat(p, tkParRi) + +proc setBaseFlags(n: PNode, base: TNumericalBase) = + case base + of base10: discard + of base2: incl(n.flags, nfBase2) + of base8: incl(n.flags, nfBase8) + of base16: incl(n.flags, nfBase16) + +proc parseGStrLit(p: var TParser, a: PNode): PNode = + case p.tok.tokType + of tkGStrLit: + result = newNodeP(nkCallStrLit, p) + addSon(result, a) + addSon(result, newStrNodeP(nkRStrLit, p.tok.literal, p)) + getTok(p) + of tkGTripleStrLit: + result = newNodeP(nkCallStrLit, p) + addSon(result, a) + addSon(result, newStrNodeP(nkTripleStrLit, p.tok.literal, p)) + getTok(p) + else: + result = a + +type + TPrimaryMode = enum pmNormal, pmTypeDesc, pmTypeDef, pmSkipSuffix + +proc complexOrSimpleStmt(p: var TParser): PNode +proc simpleExpr(p: var TParser, mode = pmNormal): PNode + +proc semiStmtList(p: var TParser, result: PNode) = + inc p.inSemiStmtList + result.add(complexOrSimpleStmt(p)) + while p.tok.tokType == tkSemiColon: + getTok(p) + optInd(p, result) + result.add(complexOrSimpleStmt(p)) + dec p.inSemiStmtList + result.kind = nkStmtListExpr + +proc parsePar(p: var TParser): PNode = + #| parKeyw = 'discard' | 'include' | 'if' | 'while' | 'case' | 'try' + #| | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let' + #| | 'when' | 'var' | 'mixin' + #| par = '(' optInd + #| ( &parKeyw complexOrSimpleStmt ^+ ';' + #| | ';' complexOrSimpleStmt ^+ ';' + #| | pragmaStmt + #| | simpleExpr ( ('=' expr (';' complexOrSimpleStmt ^+ ';' )? ) + #| | (':' expr (',' exprColonEqExpr ^+ ',' )? ) ) ) + #| optPar ')' + # + # unfortunately it's ambiguous: (expr: expr) vs (exprStmt); however a + # leading ';' could be used to enforce a 'stmt' context ... + result = newNodeP(nkPar, p) + getTok(p) + optInd(p, result) + if p.tok.tokType in {tkDiscard, tkInclude, tkIf, tkWhile, tkCase, + tkTry, tkDefer, tkFinally, tkExcept, tkFor, tkBlock, + tkConst, tkLet, tkWhen, tkVar, + tkMixin}: + # XXX 'bind' used to be an expression, so we exclude it here; + # tests/reject/tbind2 fails otherwise. + semiStmtList(p, result) + elif p.tok.tokType == tkSemiColon: + # '(;' enforces 'stmt' context: + getTok(p) + optInd(p, result) + semiStmtList(p, result) + elif p.tok.tokType == tkCurlyDotLe: + result.add(parseStmtPragma(p)) + elif p.tok.tokType != tkParRi: + var a = simpleExpr(p) + if p.tok.tokType == tkEquals: + # special case: allow assignments + getTok(p) + optInd(p, result) + let b = parseExpr(p) + let asgn = newNodeI(nkAsgn, a.info, 2) + asgn.sons[0] = a + asgn.sons[1] = b + result.add(asgn) + if p.tok.tokType == tkSemiColon: + semiStmtList(p, result) + elif p.tok.tokType == tkSemiColon: + # stmt context: + result.add(a) + semiStmtList(p, result) + else: + a = colonOrEquals(p, a) + result.add(a) + if p.tok.tokType == tkComma: + getTok(p) + skipComment(p, a) + while p.tok.tokType != tkParRi and p.tok.tokType != tkEof: + var a = exprColonEqExpr(p) + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + skipComment(p, a) + optPar(p) + eat(p, tkParRi) + +proc identOrLiteral(p: var TParser, mode: TPrimaryMode): PNode = + #| literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT + #| | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT + #| | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT + #| | STR_LIT | RSTR_LIT | TRIPLESTR_LIT + #| | CHAR_LIT + #| | NIL + #| generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT + #| identOrLiteral = generalizedLit | symbol | literal + #| | par | arrayConstr | setOrTableConstr + #| | castExpr + #| tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')' + #| arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']' + case p.tok.tokType + of tkSymbol, tkType, tkAddr: + result = newIdentNodeP(p.tok.ident, p) + getTok(p) + result = parseGStrLit(p, result) + of tkAccent: + result = parseSymbol(p) # literals + of tkIntLit: + result = newIntNodeP(nkIntLit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkInt8Lit: + result = newIntNodeP(nkInt8Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkInt16Lit: + result = newIntNodeP(nkInt16Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkInt32Lit: + result = newIntNodeP(nkInt32Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkInt64Lit: + result = newIntNodeP(nkInt64Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkUIntLit: + result = newIntNodeP(nkUIntLit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkUInt8Lit: + result = newIntNodeP(nkUInt8Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkUInt16Lit: + result = newIntNodeP(nkUInt16Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkUInt32Lit: + result = newIntNodeP(nkUInt32Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkUInt64Lit: + result = newIntNodeP(nkUInt64Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkFloatLit: + result = newFloatNodeP(nkFloatLit, p.tok.fNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkFloat32Lit: + result = newFloatNodeP(nkFloat32Lit, p.tok.fNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkFloat64Lit: + result = newFloatNodeP(nkFloat64Lit, p.tok.fNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkFloat128Lit: + result = newFloatNodeP(nkFloat128Lit, p.tok.fNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkStrLit: + result = newStrNodeP(nkStrLit, p.tok.literal, p) + getTok(p) + of tkRStrLit: + result = newStrNodeP(nkRStrLit, p.tok.literal, p) + getTok(p) + of tkTripleStrLit: + result = newStrNodeP(nkTripleStrLit, p.tok.literal, p) + getTok(p) + of tkCharLit: + result = newIntNodeP(nkCharLit, ord(p.tok.literal[0]), p) + getTok(p) + of tkNil: + result = newNodeP(nkNilLit, p) + getTok(p) + of tkParLe: + # () constructor + if mode in {pmTypeDesc, pmTypeDef}: + result = exprColonEqExprList(p, nkPar, tkParRi) + else: + result = parsePar(p) + of tkBracketDotLe: + # {} constructor + result = setOrTableConstr(p) + of tkBracketLe: + # [] constructor + result = exprColonEqExprList(p, nkBracket, tkBracketRi) + of tkCast: + result = parseCast(p) + else: + parMessage(p, errExprExpected, p.tok) + getTok(p) # we must consume a token here to prevend endless loops! + result = ast.emptyNode + +proc namedParams(p: var TParser, callee: PNode, + kind: TNodeKind, endTok: TTokType): PNode = + let a = callee + result = newNodeP(kind, p) + addSon(result, a) + exprColonEqExprListAux(p, endTok, result) + +proc parseMacroColon(p: var TParser, x: PNode): PNode +proc primarySuffix(p: var TParser, r: PNode): PNode = + #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks? + #| | doBlocks + #| | '.' optInd symbol generalizedLit? + #| | '[' optInd indexExprList optPar ']' + #| | '{' optInd indexExprList optPar '}' + #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax + result = r + + template somePar() = discard + while p.tok.indent < 0: + case p.tok.tokType + of tkParLe: + somePar() + result = namedParams(p, result, nkCall, tkParRi) + if result.len > 1 and result.sons[1].kind == nkExprColonExpr: + result.kind = nkObjConstr + else: + parseDoBlocks(p, result) + of tkDo: + var a = result + result = newNodeP(nkCall, p) + addSon(result, a) + parseDoBlocks(p, result) + of tkDot: + result = dotExpr(p, result) + result = parseGStrLit(p, result) + of tkBracketLe: + somePar() + result = namedParams(p, result, nkBracketExpr, tkBracketRi) + of tkBracketDotLe: + somePar() + result = namedParams(p, result, nkCurlyExpr, tkBracketDotRi) + of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast, tkAddr, tkType: + if p.inPragma == 0: + # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet + # solution, but pragmas.nim can't handle that + let a = result + result = newNodeP(nkCommand, p) + addSon(result, a) + when true: + addSon result, parseExpr(p) + else: + while p.tok.tokType != tkEof: + let x = parseExpr(p) + addSon(result, x) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, x) + if p.tok.tokType == tkDo: + parseDoBlocks(p, result) + else: + result = parseMacroColon(p, result) + break + else: + break + +proc primary(p: var TParser, mode: TPrimaryMode): PNode +proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode + +proc parseOperators(p: var TParser, headNode: PNode, + limit: int, mode: TPrimaryMode): PNode = + result = headNode + # expand while operators have priorities higher than 'limit' + var opPrec = getPrecedence(p.tok) + let modeB = if mode == pmTypeDef: pmTypeDesc else: mode + # the operator itself must not start on a new line: + while opPrec >= limit and p.tok.indent < 0 and not isAt(p.tok): + checkBinary(p) + var leftAssoc = 1-ord(isRightAssociative(p.tok)) + var a = newNodeP(nkInfix, p) + var opNode = newIdentNodeP(p.tok.ident, p) # skip operator: + getTok(p) + optInd(p, a) + # read sub-expression with higher priority: + var b = simpleExprAux(p, opPrec + leftAssoc, modeB) + addSon(a, opNode) + addSon(a, result) + addSon(a, b) + result = a + opPrec = getPrecedence(p.tok) + +proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode = + result = primary(p, mode) + result = parseOperators(p, result, limit, mode) + +proc simpleExpr(p: var TParser, mode = pmNormal): PNode = + result = simpleExprAux(p, -1, mode) + +proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode = + #| condExpr = expr colcom expr optInd + #| ('elif' expr colcom expr optInd)* + #| 'else' colcom expr + #| ifExpr = 'if' condExpr + #| whenExpr = 'when' condExpr + result = newNodeP(kind, p) + while true: + getTok(p) # skip `if`, `elif` + var branch = newNodeP(nkElifExpr, p) + addSon(branch, parseExpr(p)) + colcom(p, branch) + addSon(branch, parseExpr(p)) + optInd(p, branch) + addSon(result, branch) + if p.tok.tokType != tkElif: break + var branch = newNodeP(nkElseExpr, p) + eat(p, tkElse) + colcom(p, branch) + addSon(branch, parseExpr(p)) + addSon(result, branch) + +proc parsePragma(p: var TParser): PNode = + result = newNodeP(nkPragma, p) + inc p.inPragma + if isAt(p.tok): + while isAt(p.tok): + getTok(p) + var a = parseExpr(p) + optInd(p, a) + if a.kind in nkCallKinds and a.len == 2: + let repaired = newNodeI(nkExprColonExpr, a.info) + repaired.add a[0] + repaired.add a[1] + a = repaired + addSon(result, a) + skipComment(p, a) + else: + getTok(p) + optInd(p, result) + while p.tok.tokType notin {tkCurlyDotRi, tkCurlyRi, tkEof}: + var a = exprColonEqExpr(p) + addSon(result, a) + if p.tok.tokType == tkComma: + getTok(p) + skipComment(p, a) + optPar(p) + if p.tok.tokType in {tkCurlyDotRi, tkCurlyRi}: getTok(p) + else: parMessage(p, errTokenExpected, ".}") + dec p.inPragma + +proc identVis(p: var TParser; allowDot=false): PNode = + #| identVis = symbol opr? # postfix position + #| identVisDot = symbol '.' optInd symbol opr? + var a = parseSymbol(p) + if p.tok.tokType == tkOpr: + result = newNodeP(nkPostfix, p) + addSon(result, newIdentNodeP(p.tok.ident, p)) + addSon(result, a) + getTok(p) + elif p.tok.tokType == tkDot and allowDot: + result = dotExpr(p, a) + else: + result = a + +proc identWithPragma(p: var TParser; allowDot=false): PNode = + #| identWithPragma = identVis pragma? + #| identWithPragmaDot = identVisDot pragma? + var a = identVis(p, allowDot) + if p.tok.tokType == tkCurlyDotLe or isAt(p.tok): + result = newNodeP(nkPragmaExpr, p) + addSon(result, a) + addSon(result, parsePragma(p)) + else: + result = a + +type + TDeclaredIdentFlag = enum + withPragma, # identifier may have pragma + withBothOptional # both ':' and '=' parts are optional + TDeclaredIdentFlags = set[TDeclaredIdentFlag] + +proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode = + #| declColonEquals = identWithPragma (comma identWithPragma)* comma? + #| (':' optInd typeDesc)? ('=' optInd expr)? + #| identColonEquals = ident (comma ident)* comma? + #| (':' optInd typeDesc)? ('=' optInd expr)?) + var a: PNode + result = newNodeP(nkIdentDefs, p) + while true: + case p.tok.tokType + of tkSymbol, tkAccent: + if withPragma in flags: a = identWithPragma(p) + else: a = parseSymbol(p) + if a.kind == nkEmpty: return + else: break + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + if p.tok.tokType == tkColon: + getTok(p) + optInd(p, result) + addSon(result, parseTypeDesc(p)) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType != tkEquals and withBothOptional notin flags: + parMessage(p, errColonOrEqualsExpected, p.tok) + if p.tok.tokType == tkEquals: + getTok(p) + optInd(p, result) + addSon(result, parseExpr(p)) + else: + addSon(result, ast.emptyNode) + +proc parseTuple(p: var TParser): PNode = + result = newNodeP(nkTupleTy, p) + getTok(p) + if p.tok.tokType in {tkBracketLe, tkCurlyLe}: + let usedCurly = p.tok.tokType == tkCurlyLe + getTok(p) + optInd(p, result) + while p.tok.tokType in {tkSymbol, tkAccent}: + var a = parseIdentColonEquals(p, {}) + addSon(result, a) + if p.tok.tokType notin {tkComma, tkSemiColon}: break + getTok(p) + skipComment(p, a) + optPar(p) + if usedCurly: eat(p, tkCurlyRi) + else: eat(p, tkBracketRi) + else: + result = newNodeP(nkTupleClassTy, p) + +proc parseParamList(p: var TParser, retColon = true): PNode = + #| paramList = '(' declColonEquals ^* (comma/semicolon) ')' + #| paramListArrow = paramList? ('->' optInd typeDesc)? + #| paramListColon = paramList? (':' optInd typeDesc)? + var a: PNode + result = newNodeP(nkFormalParams, p) + addSon(result, ast.emptyNode) # return type + let hasParLe = p.tok.tokType == tkParLe and p.tok.indent < 0 + if hasParLe: + getTok(p) + optInd(p, result) + while true: + case p.tok.tokType + of tkSymbol, tkAccent: + a = parseIdentColonEquals(p, {withBothOptional, withPragma}) + of tkParRi: + break + else: + parMessage(p, errTokenExpected, ")") + break + addSon(result, a) + if p.tok.tokType notin {tkComma, tkSemiColon}: break + getTok(p) + skipComment(p, a) + optPar(p) + eat(p, tkParRi) + let hasRet = if retColon: p.tok.tokType == tkColon + else: p.tok.tokType == tkOpr and p.tok.ident.s == "->" + if hasRet and p.tok.indent < 0: + getTok(p) + optInd(p, result) + result.sons[0] = parseTypeDesc(p) + elif not retColon and not hasParle: + # Mark as "not there" in order to mark for deprecation in the semantic pass: + result = ast.emptyNode + +proc optPragmas(p: var TParser): PNode = + if p.tok.tokType == tkCurlyDotLe or isAt(p.tok): + result = parsePragma(p) + else: + result = ast.emptyNode + +proc parseDoBlock(p: var TParser): PNode = + #| doBlock = 'do' paramListArrow pragmas? colcom stmt + let info = parLineInfo(p) + getTok(p) + let params = parseParamList(p, retColon=false) + let pragmas = optPragmas(p) + colcom(p, result) + result = newProcNode(nkDo, info, parseStmt(p), + params = params, + pragmas = pragmas) + +proc parseDoBlocks(p: var TParser, call: PNode) = + #| doBlocks = doBlock ^* IND{=} + while p.tok.tokType == tkDo: + addSon(call, parseDoBlock(p)) + +proc parseCurlyStmt(p: var TParser): PNode = + result = newNodeP(nkStmtList, p) + eat(p, tkCurlyLe) + result.add parseStmt(p) + while p.tok.tokType notin {tkEof, tkCurlyRi}: + if p.tok.tokType == tkSemicolon: getTok(p) + elif p.tok.indent < 0: break + result.add parseStmt(p) + eat(p, tkCurlyRi) + +proc parseProcExpr(p: var TParser, isExpr: bool): PNode = + #| procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)? + # either a proc type or a anonymous proc + let info = parLineInfo(p) + getTok(p) + let hasSignature = p.tok.tokType in {tkParLe, tkColon} and p.tok.indent < 0 + let params = parseParamList(p) + let pragmas = optPragmas(p) + if p.tok.tokType == tkCurlyLe and isExpr: + result = newProcNode(nkLambda, info, parseCurlyStmt(p), + params = params, + pragmas = pragmas) + else: + result = newNodeI(nkProcTy, info) + if hasSignature: + addSon(result, params) + addSon(result, pragmas) + +proc isExprStart(p: TParser): bool = + case p.tok.tokType + of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, + tkProc, tkIterator, tkBind, tkAddr, + tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr, + tkTuple, tkObject, tkType, tkWhen, tkCase, tkOut: + result = true + else: result = false + +proc parseSymbolList(p: var TParser, result: PNode, allowNil = false) = + while true: + var s = parseSymbol(p, allowNil) + if s.kind == nkEmpty: break + addSon(result, s) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, s) + +proc parseTypeDescKAux(p: var TParser, kind: TNodeKind, + mode: TPrimaryMode): PNode = + #| distinct = 'distinct' optInd typeDesc + result = newNodeP(kind, p) + getTok(p) + optInd(p, result) + if not isOperator(p.tok) and isExprStart(p): + addSon(result, primary(p, mode)) + if kind == nkDistinctTy and p.tok.tokType in {tkWith, tkWithout}: + let nodeKind = if p.tok.tokType == tkWith: nkWith + else: nkWithout + getTok(p) + let list = newNodeP(nodeKind, p) + result.addSon list + parseSymbolList(p, list, allowNil = true) + +proc parseExpr(p: var TParser): PNode = + #| expr = (ifExpr + #| | whenExpr + #| | caseExpr + #| | tryExpr) + #| / simpleExpr + case p.tok.tokType: + of tkIf: result = parseIfExpr(p, nkIfExpr) + of tkWhen: result = parseIfExpr(p, nkWhenExpr) + of tkCase: result = parseCase(p) + of tkTry: result = parseTry(p) + else: result = simpleExpr(p) + +proc parseEnum(p: var TParser): PNode +proc parseObject(p: var TParser): PNode +proc parseTypeClass(p: var TParser): PNode + +proc primary(p: var TParser, mode: TPrimaryMode): PNode = + #| typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple' + #| | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum' + #| primary = typeKeyw typeDescK + #| / prefixOperator* identOrLiteral primarySuffix* + #| / 'static' primary + #| / 'bind' primary + if isOperator(p.tok): + let isSigil = isSigilLike(p.tok) + result = newNodeP(nkPrefix, p) + var a = newIdentNodeP(p.tok.ident, p) + addSon(result, a) + getTok(p) + optInd(p, a) + if isSigil: + #XXX prefix operators + addSon(result, primary(p, pmSkipSuffix)) + result = primarySuffix(p, result) + else: + addSon(result, primary(p, pmNormal)) + return + + case p.tok.tokType: + of tkTuple: result = parseTuple(p) + of tkProc: result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}) + of tkIterator: + result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}) + if result.kind == nkLambda: result.kind = nkIteratorDef + else: result.kind = nkIteratorTy + of tkEnum: + if mode == pmTypeDef: + result = parseEnum(p) + else: + result = newNodeP(nkEnumTy, p) + getTok(p) + of tkObject: + if mode == pmTypeDef: + result = parseObject(p) + else: + result = newNodeP(nkObjectTy, p) + getTok(p) + of tkConcept: + if mode == pmTypeDef: + result = parseTypeClass(p) + else: + parMessage(p, errInvalidToken, p.tok) + of tkStatic: + let info = parLineInfo(p) + getTokNoInd(p) + let next = primary(p, pmNormal) + if next.kind == nkBracket and next.sonsLen == 1: + result = newNode(nkStaticTy, info, @[next.sons[0]]) + else: + result = newNode(nkStaticExpr, info, @[next]) + of tkBind: + result = newNodeP(nkBind, p) + getTok(p) + optInd(p, result) + addSon(result, primary(p, pmNormal)) + of tkVar: result = parseTypeDescKAux(p, nkVarTy, mode) + of tkOut: result = parseTypeDescKAux(p, nkVarTy, mode) + of tkRef: result = parseTypeDescKAux(p, nkRefTy, mode) + of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, mode) + of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, mode) + else: + result = identOrLiteral(p, mode) + if mode != pmSkipSuffix: + result = primarySuffix(p, result) + +proc parseTypeDesc(p: var TParser): PNode = + #| typeDesc = simpleExpr + result = simpleExpr(p, pmTypeDesc) + +proc parseTypeDefAux(p: var TParser): PNode = + #| typeDefAux = simpleExpr + #| | 'concept' typeClass + result = simpleExpr(p, pmTypeDef) + +proc makeCall(n: PNode): PNode = + ## Creates a call if the given node isn't already a call. + if n.kind in nkCallKinds: + result = n + else: + result = newNodeI(nkCall, n.info) + result.add n + +proc parseMacroColon(p: var TParser, x: PNode): PNode = + #| macroColon = ':' stmt? ( IND{=} 'of' exprList ':' stmt + #| | IND{=} 'elif' expr ':' stmt + #| | IND{=} 'except' exprList ':' stmt + #| | IND{=} 'else' ':' stmt )* + result = x + if p.tok.tokType == tkColon and p.tok.indent < 0: + result = makeCall(result) + getTok(p) + skipComment(p, result) + let stmtList = newNodeP(nkStmtList, p) + if p.tok.tokType notin {tkOf, tkElif, tkElse, tkExcept}: + let body = parseStmt(p) + stmtList.add body + #addSon(result, makeStmtList(body)) + while true: + var b: PNode + case p.tok.tokType + of tkOf: + b = newNodeP(nkOfBranch, p) + exprList(p, tkCurlyLe, b) + of tkElif: + b = newNodeP(nkElifBranch, p) + getTok(p) + optInd(p, b) + addSon(b, parseExpr(p)) + of tkExcept: + b = newNodeP(nkExceptBranch, p) + exprList(p, tkCurlyLe, b) + of tkElse: + b = newNodeP(nkElse, p) + getTok(p) + else: break + addSon(b, parseCurlyStmt(p)) + addSon(stmtList, b) + if b.kind == nkElse: break + if stmtList.len == 1 and stmtList[0].kind == nkStmtList: + # to keep backwards compatibility (see tests/vm/tstringnil) + result.add stmtList[0] + else: + result.add stmtList + +proc parseExprStmt(p: var TParser): PNode = + #| exprStmt = simpleExpr + #| (( '=' optInd expr ) + #| / ( expr ^+ comma + #| doBlocks + #| / macroColon + #| ))? + var a = simpleExpr(p) + if p.tok.tokType == tkEquals: + getTok(p) + optInd(p, result) + var b = parseExpr(p) + result = newNodeI(nkAsgn, a.info) + addSon(result, a) + addSon(result, b) + else: + # simpleExpr parsed 'p a' from 'p a, b'? + if p.tok.indent < 0 and p.tok.tokType == tkComma and a.kind == nkCommand: + result = a + while true: + getTok(p) + optInd(p, result) + var e = parseExpr(p) + addSon(result, e) + if p.tok.tokType != tkComma: break + elif p.tok.indent < 0 and isExprStart(p): + if a.kind == nkCommand: + result = a + else: + result = newNode(nkCommand, a.info, @[a]) + while true: + var e = parseExpr(p) + addSon(result, e) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, result) + else: + result = a + if p.tok.tokType == tkDo and p.tok.indent < 0: + result = makeCall(result) + parseDoBlocks(p, result) + return result + result = parseMacroColon(p, result) + +proc parseModuleName(p: var TParser, kind: TNodeKind): PNode = + result = parseExpr(p) + +proc parseImport(p: var TParser, kind: TNodeKind): PNode = + #| importStmt = 'import' optInd expr + #| ((comma expr)* + #| / 'except' optInd (expr ^+ comma)) + result = newNodeP(kind, p) + getTok(p) # skip `import` or `export` + optInd(p, result) + var a = parseModuleName(p, kind) + addSon(result, a) + if p.tok.tokType in {tkComma, tkExcept}: + if p.tok.tokType == tkExcept: + result.kind = succ(kind) + getTok(p) + optInd(p, result) + while true: + # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}: + a = parseModuleName(p, kind) + if a.kind == nkEmpty: break + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + #expectNl(p) + +proc parseIncludeStmt(p: var TParser): PNode = + #| includeStmt = 'include' optInd expr ^+ comma + result = newNodeP(nkIncludeStmt, p) + getTok(p) # skip `import` or `include` + optInd(p, result) + while true: + # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}: + var a = parseExpr(p) + if a.kind == nkEmpty: break + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + #expectNl(p) + +proc parseFromStmt(p: var TParser): PNode = + #| fromStmt = 'from' moduleName 'import' optInd expr (comma expr)* + result = newNodeP(nkFromStmt, p) + getTok(p) # skip `from` + optInd(p, result) + var a = parseModuleName(p, nkImportStmt) + addSon(result, a) #optInd(p, a); + eat(p, tkImport) + optInd(p, result) + while true: + # p.tok.tokType notin {tkEof, tkSad, tkDed}: + a = parseExpr(p) + if a.kind == nkEmpty: break + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + #expectNl(p) + +proc parseReturnOrRaise(p: var TParser, kind: TNodeKind): PNode = + #| returnStmt = 'return' optInd expr? + #| raiseStmt = 'raise' optInd expr? + #| yieldStmt = 'yield' optInd expr? + #| discardStmt = 'discard' optInd expr? + #| breakStmt = 'break' optInd expr? + #| continueStmt = 'break' optInd expr? + result = newNodeP(kind, p) + getTok(p) + if p.tok.tokType == tkComment: + skipComment(p, result) + addSon(result, ast.emptyNode) + elif p.tok.indent >= 0 or not isExprStart(p): + # NL terminates: + addSon(result, ast.emptyNode) + else: + addSon(result, parseExpr(p)) + +proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode = + #| condStmt = expr colcom stmt COMMENT? + #| (IND{=} 'elif' expr colcom stmt)* + #| (IND{=} 'else' colcom stmt)? + #| ifStmt = 'if' condStmt + #| whenStmt = 'when' condStmt + result = newNodeP(kind, p) + while true: + getTok(p) # skip `if`, `when`, `elif` + var branch = newNodeP(nkElifBranch, p) + optInd(p, branch) + addSon(branch, parseExpr(p)) + colcom(p, branch) + addSon(branch, parseCurlyStmt(p)) + skipComment(p, branch) + addSon(result, branch) + if p.tok.tokType != tkElif: break + if p.tok.tokType == tkElse: + var branch = newNodeP(nkElse, p) + eat(p, tkElse) + addSon(branch, parseCurlyStmt(p)) + addSon(result, branch) + +proc parseWhile(p: var TParser): PNode = + #| whileStmt = 'while' expr colcom stmt + result = newNodeP(nkWhileStmt, p) + getTok(p) + optInd(p, result) + addSon(result, parseExpr(p)) + colcom(p, result) + addSon(result, parseCurlyStmt(p)) + +proc parseCase(p: var TParser): PNode = + #| ofBranch = 'of' exprList colcom stmt + #| ofBranches = ofBranch (IND{=} ofBranch)* + #| (IND{=} 'elif' expr colcom stmt)* + #| (IND{=} 'else' colcom stmt)? + #| caseStmt = 'case' expr ':'? COMMENT? + #| (IND{>} ofBranches DED + #| | IND{=} ofBranches) + var + b: PNode + inElif= false + result = newNodeP(nkCaseStmt, p) + getTok(p) + addSon(result, parseExpr(p)) + eat(p, tkCurlyLe) + skipComment(p, result) + + while true: + case p.tok.tokType + of tkOf: + if inElif: break + b = newNodeP(nkOfBranch, p) + exprList(p, tkCurlyLe, b) + of tkElif: + inElif = true + b = newNodeP(nkElifBranch, p) + getTok(p) + optInd(p, b) + addSon(b, parseExpr(p)) + of tkElse: + b = newNodeP(nkElse, p) + getTok(p) + else: break + skipComment(p, b) + addSon(b, parseCurlyStmt(p)) + addSon(result, b) + if b.kind == nkElse: break + eat(p, tkCurlyRi) + +proc parseTry(p: var TParser): PNode = + #| tryStmt = 'try' colcom stmt &(IND{=}? 'except'|'finally') + #| (IND{=}? 'except' exprList colcom stmt)* + #| (IND{=}? 'finally' colcom stmt)? + #| tryExpr = 'try' colcom stmt &(optInd 'except'|'finally') + #| (optInd 'except' exprList colcom stmt)* + #| (optInd 'finally' colcom stmt)? + result = newNodeP(nkTryStmt, p) + getTok(p) + colcom(p, result) + addSon(result, parseCurlyStmt(p)) + var b: PNode = nil + while true: + case p.tok.tokType + of tkExcept: + b = newNodeP(nkExceptBranch, p) + exprList(p, tkCurlyLe, b) + of tkFinally: + b = newNodeP(nkFinally, p) + getTok(p) + else: break + skipComment(p, b) + addSon(b, parseCurlyStmt(p)) + addSon(result, b) + if b.kind == nkFinally: break + if b == nil: parMessage(p, errTokenExpected, "except") + +proc parseFor(p: var TParser): PNode = + #| forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt + result = newNodeP(nkForStmt, p) + getTokNoInd(p) + var a = identWithPragma(p) + addSon(result, a) + while p.tok.tokType == tkComma: + getTok(p) + optInd(p, a) + a = identWithPragma(p) + addSon(result, a) + eat(p, tkIn) + addSon(result, parseExpr(p)) + colcom(p, result) + addSon(result, parseCurlyStmt(p)) + +proc parseBlock(p: var TParser): PNode = + #| blockStmt = 'block' symbol? colcom stmt + result = newNodeP(nkBlockStmt, p) + getTokNoInd(p) + if p.tok.tokType == tkCurlyLe: addSon(result, ast.emptyNode) + else: addSon(result, parseSymbol(p)) + colcom(p, result) + addSon(result, parseCurlyStmt(p)) + +proc parseStaticOrDefer(p: var TParser; k: TNodeKind): PNode = + #| staticStmt = 'static' colcom stmt + #| deferStmt = 'defer' colcom stmt + result = newNodeP(k, p) + getTok(p) + colcom(p, result) + addSon(result, parseCurlyStmt(p)) + +proc parseAsm(p: var TParser): PNode = + #| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLE_STR_LIT) + result = newNodeP(nkAsmStmt, p) + getTokNoInd(p) + if p.tok.tokType == tkCurlyDotLe or isAt(p.tok): addSon(result, parsePragma(p)) + else: addSon(result, ast.emptyNode) + case p.tok.tokType + of tkStrLit: addSon(result, newStrNodeP(nkStrLit, p.tok.literal, p)) + of tkRStrLit: addSon(result, newStrNodeP(nkRStrLit, p.tok.literal, p)) + of tkTripleStrLit: addSon(result, + newStrNodeP(nkTripleStrLit, p.tok.literal, p)) + else: + parMessage(p, errStringLiteralExpected) + addSon(result, ast.emptyNode) + return + getTok(p) + +proc parseGenericParam(p: var TParser): PNode = + #| genericParam = symbol (comma symbol)* (colon expr)? ('=' optInd expr)? + var a: PNode + result = newNodeP(nkIdentDefs, p) + while true: + case p.tok.tokType + of tkSymbol, tkAccent: + a = parseSymbol(p) + if a.kind == nkEmpty: return + else: break + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + if p.tok.tokType == tkColon: + getTok(p) + optInd(p, result) + addSon(result, parseExpr(p)) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkEquals: + getTok(p) + optInd(p, result) + addSon(result, parseExpr(p)) + else: + addSon(result, ast.emptyNode) + +proc parseGenericParamList(p: var TParser): PNode = + #| genericParamList = '[' optInd + #| genericParam ^* (comma/semicolon) optPar ']' + result = newNodeP(nkGenericParams, p) + getTok(p) + optInd(p, result) + while p.tok.tokType in {tkSymbol, tkAccent}: + var a = parseGenericParam(p) + addSon(result, a) + if p.tok.tokType notin {tkComma, tkSemiColon}: break + getTok(p) + skipComment(p, a) + optPar(p) + eat(p, tkBracketRi) + +proc parsePattern(p: var TParser): PNode = + eat(p, tkBracketDotLe) + result = parseStmt(p) + eat(p, tkBracketDotRi) + +proc validInd(p: TParser): bool = p.tok.indent < 0 + +proc parseRoutine(p: var TParser, kind: TNodeKind): PNode = + #| indAndComment = (IND{>} COMMENT)? | COMMENT? + #| routine = optInd identVis pattern? genericParamList? + #| paramListColon pragma? ('=' COMMENT? stmt)? indAndComment + result = newNodeP(kind, p) + getTok(p) + optInd(p, result) + addSon(result, identVis(p)) + if p.tok.tokType == tkBracketDotLe and p.validInd: + addSon(result, p.parsePattern) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkBracketLe and p.validInd: + result.add(p.parseGenericParamList) + else: + addSon(result, ast.emptyNode) + addSon(result, p.parseParamList) + if (p.tok.tokType == tkCurlyDotLe or isAt(p.tok)) and p.validInd: + addSon(result, p.parsePragma) + else: + addSon(result, ast.emptyNode) + # empty exception tracking: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkCurlyLe: + addSon(result, parseCurlyStmt(p)) + else: + addSon(result, ast.emptyNode) + indAndComment(p, result) + +proc newCommentStmt(p: var TParser): PNode = + #| commentStmt = COMMENT + result = newNodeP(nkCommentStmt, p) + result.comment = p.tok.literal + getTok(p) + +type + TDefParser = proc (p: var TParser): PNode {.nimcall.} + +proc parseSection(p: var TParser, kind: TNodeKind, + defparser: TDefParser): PNode = + #| section(p) = COMMENT? p / (IND{>} (p / COMMENT)^+IND{=} DED) + result = newNodeP(kind, p) + if kind != nkTypeSection: getTok(p) + skipComment(p, result) + if p.tok.tokType == tkParLe: + getTok(p) + skipComment(p, result) + while true: + case p.tok.tokType + of tkSymbol, tkAccent, tkParLe: + var a = defparser(p) + skipComment(p, a) + addSon(result, a) + of tkComment: + var a = newCommentStmt(p) + addSon(result, a) + of tkParRi: break + else: + parMessage(p, errIdentifierExpected, p.tok) + break + eat(p, tkParRi) + if result.len == 0: parMessage(p, errIdentifierExpected, p.tok) + elif p.tok.tokType in {tkSymbol, tkAccent, tkBracketLe}: + # tkBracketLe is allowed for ``var [x, y] = ...`` tuple parsing + addSon(result, defparser(p)) + else: + parMessage(p, errIdentifierExpected, p.tok) + +proc parseConstant(p: var TParser): PNode = + #| constant = identWithPragma (colon typedesc)? '=' optInd expr indAndComment + result = newNodeP(nkConstDef, p) + addSon(result, identWithPragma(p)) + if p.tok.tokType == tkColon: + getTok(p) + optInd(p, result) + addSon(result, parseTypeDesc(p)) + else: + addSon(result, ast.emptyNode) + eat(p, tkEquals) + optInd(p, result) + addSon(result, parseExpr(p)) + indAndComment(p, result) + +proc parseEnum(p: var TParser): PNode = + #| enum = 'enum' optInd (symbol optInd ('=' optInd expr COMMENT?)? comma?)+ + result = newNodeP(nkEnumTy, p) + getTok(p) + addSon(result, ast.emptyNode) + optInd(p, result) + flexComment(p, result) + eat(p, tkCurlyLe) + optInd(p, result) + while p.tok.tokType notin {tkEof, tkCurlyRi}: + var a = parseSymbol(p) + if a.kind == nkEmpty: return + if p.tok.tokType == tkEquals: + getTok(p) + optInd(p, a) + var b = a + a = newNodeP(nkEnumFieldDef, p) + addSon(a, b) + addSon(a, parseExpr(p)) + if p.tok.indent < 0: + rawSkipComment(p, a) + if p.tok.tokType == tkComma: + getTok(p) + rawSkipComment(p, a) + addSon(result, a) + eat(p, tkCurlyRi) + if result.len <= 1: + lexMessageTok(p.lex, errIdentifierExpected, p.tok, prettyTok(p.tok)) + +proc parseObjectPart(p: var TParser; needsCurly: bool): PNode +proc parseObjectWhen(p: var TParser): PNode = + result = newNodeP(nkRecWhen, p) + while true: + getTok(p) # skip `when`, `elif` + var branch = newNodeP(nkElifBranch, p) + optInd(p, branch) + addSon(branch, parseExpr(p)) + colcom(p, branch) + addSon(branch, parseObjectPart(p, true)) + flexComment(p, branch) + addSon(result, branch) + if p.tok.tokType != tkElif: break + if p.tok.tokType == tkElse: + var branch = newNodeP(nkElse, p) + eat(p, tkElse) + colcom(p, branch) + addSon(branch, parseObjectPart(p, true)) + flexComment(p, branch) + addSon(result, branch) + +proc parseObjectCase(p: var TParser): PNode = + result = newNodeP(nkRecCase, p) + getTokNoInd(p) + var a = newNodeP(nkIdentDefs, p) + addSon(a, identWithPragma(p)) + eat(p, tkColon) + addSon(a, parseTypeDesc(p)) + addSon(a, ast.emptyNode) + addSon(result, a) + eat(p, tkCurlyLe) + flexComment(p, result) + while true: + var b: PNode + case p.tok.tokType + of tkOf: + b = newNodeP(nkOfBranch, p) + exprList(p, tkColon, b) + of tkElse: + b = newNodeP(nkElse, p) + getTok(p) + else: break + colcom(p, b) + var fields = parseObjectPart(p, true) + if fields.kind == nkEmpty: + parMessage(p, errIdentifierExpected, p.tok) + fields = newNodeP(nkNilLit, p) # don't break further semantic checking + addSon(b, fields) + addSon(result, b) + if b.kind == nkElse: break + eat(p, tkCurlyRi) + +proc parseObjectPart(p: var TParser; needsCurly: bool): PNode = + if p.tok.tokType == tkCurlyLe: + result = newNodeP(nkRecList, p) + getTok(p) + rawSkipComment(p, result) + while true: + case p.tok.tokType + of tkCase, tkWhen, tkSymbol, tkAccent, tkNil, tkDiscard: + addSon(result, parseObjectPart(p, false)) + of tkCurlyRi: break + else: + parMessage(p, errIdentifierExpected, p.tok) + break + eat(p, tkCurlyRi) + else: + if needsCurly: + parMessage(p, errTokenExpected, "{") + case p.tok.tokType + of tkWhen: + result = parseObjectWhen(p) + of tkCase: + result = parseObjectCase(p) + of tkSymbol, tkAccent: + result = parseIdentColonEquals(p, {withPragma}) + if p.tok.indent < 0: rawSkipComment(p, result) + of tkNil, tkDiscard: + result = newNodeP(nkNilLit, p) + getTok(p) + else: + result = ast.emptyNode + +proc parseObject(p: var TParser): PNode = + result = newNodeP(nkObjectTy, p) + getTok(p) + if (p.tok.tokType == tkCurlyDotLe or isAt(p.tok)) and p.validInd: + addSon(result, parsePragma(p)) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkOf and p.tok.indent < 0: + var a = newNodeP(nkOfInherit, p) + getTok(p) + addSon(a, parseTypeDesc(p)) + addSon(result, a) + else: + addSon(result, ast.emptyNode) + skipComment(p, result) + # an initial IND{>} HAS to follow: + addSon(result, parseObjectPart(p, true)) + +proc parseTypeClassParam(p: var TParser): PNode = + if p.tok.tokType in {tkOut, tkVar}: + result = newNodeP(nkVarTy, p) + getTok(p) + result.addSon(p.parseSymbol) + else: + result = p.parseSymbol + +proc parseTypeClass(p: var TParser): PNode = + #| typeClassParam = ('var' | 'out')? symbol + #| typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? + #| &IND{>} stmt + result = newNodeP(nkTypeClassTy, p) + getTok(p) + var args = newNodeP(nkArgList, p) + addSon(result, args) + addSon(args, p.parseTypeClassParam) + while p.tok.tokType == tkComma: + getTok(p) + addSon(args, p.parseTypeClassParam) + if (p.tok.tokType == tkCurlyDotLe or isAt(p.tok)) and p.validInd: + addSon(result, parsePragma(p)) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkOf and p.tok.indent < 0: + var a = newNodeP(nkOfInherit, p) + getTok(p) + while true: + addSon(a, parseTypeDesc(p)) + if p.tok.tokType != tkComma: break + getTok(p) + addSon(result, a) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkComment: + skipComment(p, result) + addSon(result, parseCurlyStmt(p)) + +proc parseTypeDef(p: var TParser): PNode = + #| + #| typeDef = identWithPragmaDot genericParamList? '=' optInd typeDefAux + #| indAndComment? + result = newNodeP(nkTypeDef, p) + addSon(result, identWithPragma(p, allowDot=true)) + if p.tok.tokType == tkBracketLe and p.validInd: + addSon(result, parseGenericParamList(p)) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkEquals: + getTok(p) + optInd(p, result) + addSon(result, parseTypeDefAux(p)) + else: + addSon(result, ast.emptyNode) + indAndComment(p, result) # special extension! + +proc parseVarTuple(p: var TParser): PNode = + #| varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr + result = newNodeP(nkVarTuple, p) + getTok(p) # skip '(' + optInd(p, result) + while p.tok.tokType in {tkSymbol, tkAccent}: + var a = identWithPragma(p) + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + skipComment(p, a) + addSon(result, ast.emptyNode) # no type desc + optPar(p) + eat(p, tkBracketRi) + eat(p, tkEquals) + optInd(p, result) + addSon(result, parseExpr(p)) + +proc parseVariable(p: var TParser): PNode = + #| variable = (varTuple / identColonEquals) indAndComment + if p.tok.tokType == tkBracketLe: result = parseVarTuple(p) + else: result = parseIdentColonEquals(p, {withPragma}) + indAndComment(p, result) + +proc parseBind(p: var TParser, k: TNodeKind): PNode = + #| bindStmt = 'bind' optInd qualifiedIdent ^+ comma + #| mixinStmt = 'mixin' optInd qualifiedIdent ^+ comma + result = newNodeP(k, p) + getTok(p) + optInd(p, result) + while true: + var a = qualifiedIdent(p) + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + +proc parseStmtPragma(p: var TParser): PNode = + result = parsePragma(p) + if p.tok.tokType == tkCurlyLe: + let a = result + result = newNodeI(nkPragmaBlock, a.info) + getTok(p) + skipComment(p, result) + result.add a + result.add parseStmt(p) + eat(p, tkCurlyRi) + +proc simpleStmt(p: var TParser): PNode = + case p.tok.tokType + of tkReturn: result = parseReturnOrRaise(p, nkReturnStmt) + of tkRaise: result = parseReturnOrRaise(p, nkRaiseStmt) + of tkYield: result = parseReturnOrRaise(p, nkYieldStmt) + of tkDiscard: result = parseReturnOrRaise(p, nkDiscardStmt) + of tkBreak: result = parseReturnOrRaise(p, nkBreakStmt) + of tkContinue: result = parseReturnOrRaise(p, nkContinueStmt) + of tkCurlyDotLe: result = parseStmtPragma(p) + of tkImport: result = parseImport(p, nkImportStmt) + of tkExport: result = parseImport(p, nkExportStmt) + of tkFrom: result = parseFromStmt(p) + of tkInclude: result = parseIncludeStmt(p) + of tkComment: result = newCommentStmt(p) + else: + if isExprStart(p): result = parseExprStmt(p) + else: result = ast.emptyNode + if result.kind notin {nkEmpty, nkCommentStmt}: skipComment(p, result) + +proc complexOrSimpleStmt(p: var TParser): PNode = + case p.tok.tokType + of tkIf: result = parseIfOrWhen(p, nkIfStmt) + of tkWhile: result = parseWhile(p) + of tkCase: result = parseCase(p) + of tkTry: result = parseTry(p) + of tkFor: result = parseFor(p) + of tkBlock: result = parseBlock(p) + of tkStatic: result = parseStaticOrDefer(p, nkStaticStmt) + of tkDefer: result = parseStaticOrDefer(p, nkDefer) + of tkAsm: result = parseAsm(p) + of tkProc: result = parseRoutine(p, nkProcDef) + of tkMethod: result = parseRoutine(p, nkMethodDef) + of tkIterator: result = parseRoutine(p, nkIteratorDef) + of tkMacro: result = parseRoutine(p, nkMacroDef) + of tkTemplate: result = parseRoutine(p, nkTemplateDef) + of tkConverter: result = parseRoutine(p, nkConverterDef) + of tkType: + getTok(p) + if p.tok.tokType == tkBracketLe: + getTok(p) + result = newNodeP(nkTypeOfExpr, p) + result.addSon(primary(p, pmTypeDesc)) + eat(p, tkBracketRi) + result = parseOperators(p, result, -1, pmNormal) + else: + result = parseSection(p, nkTypeSection, parseTypeDef) + of tkConst: result = parseSection(p, nkConstSection, parseConstant) + of tkLet: result = parseSection(p, nkLetSection, parseVariable) + of tkWhen: result = parseIfOrWhen(p, nkWhenStmt) + of tkVar: result = parseSection(p, nkVarSection, parseVariable) + of tkBind: result = parseBind(p, nkBindStmt) + of tkMixin: result = parseBind(p, nkMixinStmt) + of tkUsing: result = parseSection(p, nkUsingStmt, parseVariable) + else: result = simpleStmt(p) + +proc parseStmt(p: var TParser): PNode = + result = complexOrSimpleStmt(p) proc parseAll*(p: var TParser): PNode = - result = nil + ## Parses the rest of the input stream held by the parser into a PNode. + result = newNodeP(nkStmtList, p) + while p.tok.tokType != tkEof: + var a = complexOrSimpleStmt(p) + if a.kind != nkEmpty: + addSon(result, a) + else: + parMessage(p, errExprExpected, p.tok) + # bugfix: consume a token here to prevent an endless loop: + getTok(p) proc parseTopLevelStmt*(p: var TParser): PNode = - result = nil - + ## Implements an iterator which, when called repeatedly, returns the next + ## top-level statement or emptyNode if end of stream. + result = ast.emptyNode + while true: + case p.tok.tokType + of tkSemiColon: getTok(p) + of tkEof: break + else: + result = complexOrSimpleStmt(p) + if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) + break diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index f4109b26d..e11a8d08b 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -323,7 +323,8 @@ proc processOption(c: PContext, n: PNode): bool = of wStacktrace: onOff(c, n, {optStackTrace}) of wLinetrace: onOff(c, n, {optLineTrace}) of wDebugger: onOff(c, n, {optEndb}) - of wProfiler: onOff(c, n, {optProfiler}) + of wProfiler: onOff(c, n, {optProfiler, optMemTracker}) + of wMemTracker: onOff(c, n, {optMemTracker}) of wByRef: onOff(c, n, {optByRef}) of wDynlib: processDynLib(c, n, nil) of wOptimization: diff --git a/compiler/renderer.nim b/compiler/renderer.nim index a116a8afe..926e67743 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -54,8 +54,6 @@ proc isKeyword*(i: PIdent): bool = (i.id <= ord(tokKeywordHigh) - ord(tkSymbol)): result = true -proc isKeyword*(s: string): bool = isKeyword(getIdent(s)) - proc renderDefinitionName*(s: PSym, noQuotes = false): string = ## Returns the definition name of the symbol. ## @@ -289,8 +287,7 @@ proc lsub(n: PNode): int proc litAux(n: PNode, x: BiggestInt, size: int): string = proc skip(t: PType): PType = result = t - while result.kind in {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal, - tyConst, tyMutable}: + while result.kind in {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal}: result = lastSon(result) if n.typ != nil and n.typ.skip.kind in {tyBool, tyEnum}: let enumfields = n.typ.skip.n diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 679e7ba15..f7e5a0f84 100644 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -142,6 +142,7 @@ type methods*: TSymSeq origFile: string inViewMode: bool + cache*: IdentCache PRodReader* = ref TRodReader @@ -219,7 +220,7 @@ proc decodeNodeLazyBody(r: PRodReader, fInfo: TLineInfo, if r.s[r.pos] == '!': inc(r.pos) var fl = decodeStr(r.s, r.pos) - result.ident = getIdent(fl) + result.ident = r.cache.getIdent(fl) else: internalError(result.info, "decodeNode: nkIdent") of nkSym: @@ -401,7 +402,7 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym = internalError(info, "decodeSym: no id") if r.s[r.pos] == '&': inc(r.pos) - ident = getIdent(decodeStr(r.s, r.pos)) + ident = r.cache.getIdent(decodeStr(r.s, r.pos)) else: internalError(info, "decodeSym: no ident") #echo "decoding: {", ident.s @@ -519,7 +520,7 @@ proc newStub(r: PRodReader, name: string, id: int): PSym = new(result) result.kind = skStub result.id = id - result.name = getIdent(name) + result.name = r.cache.getIdent(name) result.position = r.readerIndex setId(id) #MessageOut(result.name.s); if debugIds: registerID(result) @@ -632,7 +633,7 @@ proc processRodFile(r: PRodReader, hash: SecureHash) = while r.s[r.pos] > '\x0A': w = decodeStr(r.s, r.pos) inc(d) - if not condsyms.isDefined(getIdent(w)): + if not condsyms.isDefined(r.cache.getIdent(w)): r.reason = rrDefines #MessageOut('not defined, but should: ' + w); if r.s[r.pos] == ' ': inc(r.pos) if d != countDefinedSymbols(): r.reason = rrDefines @@ -707,8 +708,9 @@ proc startsWith(buf: cstring, token: string, pos = 0): bool = result = s == token.len proc newRodReader(modfilename: string, hash: SecureHash, - readerIndex: int): PRodReader = + readerIndex: int; cache: IdentCache): PRodReader = new(result) + result.cache = cache try: result.memfile = memfiles.open(modfilename) except OSError: @@ -866,7 +868,7 @@ proc getHash*(fileIdx: int32): SecureHash = template growCache*(cache, pos) = if cache.len <= pos: cache.setLen(pos+1) -proc checkDep(fileIdx: int32): TReasonForRecompile = +proc checkDep(fileIdx: int32; cache: IdentCache): TReasonForRecompile = assert fileIdx != InvalidFileIDX growCache gMods, fileIdx if gMods[fileIdx].reason != rrEmpty: @@ -877,7 +879,7 @@ proc checkDep(fileIdx: int32): TReasonForRecompile = gMods[fileIdx].reason = rrNone # we need to set it here to avoid cycles result = rrNone var rodfile = toGeneratedFile(filename.withPackageName, RodExt) - var r = newRodReader(rodfile, hash, fileIdx) + var r = newRodReader(rodfile, hash, fileIdx, cache) if r == nil: result = (if existsFile(rodfile): rrRodInvalid else: rrRodDoesNotExist) else: @@ -888,10 +890,10 @@ proc checkDep(fileIdx: int32): TReasonForRecompile = # NOTE: we need to process the entire module graph so that no ID will # be used twice! However, compilation speed does not suffer much from # this, since results are cached. - var res = checkDep(systemFileIdx) + var res = checkDep(systemFileIdx, cache) if res != rrNone: result = rrModDeps for i in countup(0, high(r.modDeps)): - res = checkDep(r.modDeps[i]) + res = checkDep(r.modDeps[i], cache) if res != rrNone: result = rrModDeps # we cannot break here, because of side-effects of `checkDep` @@ -904,14 +906,14 @@ proc checkDep(fileIdx: int32): TReasonForRecompile = gMods[fileIdx].rd = r gMods[fileIdx].reason = result # now we know better -proc handleSymbolFile*(module: PSym): PRodReader = +proc handleSymbolFile*(module: PSym; cache: IdentCache): PRodReader = let fileIdx = module.fileIdx if optSymbolFiles notin gGlobalOptions: module.id = getID() return nil idgen.loadMaxIds(options.gProjectPath / options.gProjectName) - discard checkDep(fileIdx) + discard checkDep(fileIdx, cache) if gMods[fileIdx].reason == rrEmpty: internalError("handleSymbolFile") result = gMods[fileIdx].rd if result != nil: @@ -1078,7 +1080,7 @@ proc writeType(f: File; t: PType) = f.write("]\n") proc viewFile(rodfile: string) = - var r = newRodReader(rodfile, secureHash(""), 0) + var r = newRodReader(rodfile, secureHash(""), 0, newIdentCache()) if r == nil: rawMessage(errGenerated, "cannot open file (or maybe wrong version):" & rodfile) diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index addbdade6..f2422f947 100644 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -16,6 +16,8 @@ import condsyms, ropes, idents, securehash, rodread, passes, importer, idgen, rodutils +from modulegraphs import ModuleGraph + type TRodWriter = object of TPassContext module: PSym @@ -34,6 +36,7 @@ type tstack: TTypeSeq # a stack of types to process files: TStringSeq origFile: string + cache: IdentCache PRodWriter = ref TRodWriter @@ -54,7 +57,7 @@ proc fileIdx(w: PRodWriter, filename: string): int = template filename*(w: PRodWriter): string = w.module.filename -proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter = +proc newRodWriter(hash: SecureHash, module: PSym; cache: IdentCache): PRodWriter = new(result) result.sstack = @[] result.tstack = @[] @@ -76,6 +79,7 @@ proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter = result.init = "" result.origFile = module.info.toFullPath result.data = newStringOfCap(12_000) + result.cache = cache proc addModDep(w: PRodWriter, dep: string; info: TLineInfo) = if w.modDeps.len != 0: add(w.modDeps, ' ') @@ -575,15 +579,25 @@ proc process(c: PPassContext, n: PNode): PNode = for i in countup(0, sonsLen(n) - 1): discard process(c, n.sons[i]) #var s = n.sons[namePos].sym #addInterfaceSym(w, s) - of nkProcDef, nkMethodDef, nkIteratorDef, nkConverterDef, + of nkProcDef, nkIteratorDef, nkConverterDef, nkTemplateDef, nkMacroDef: - var s = n.sons[namePos].sym + let s = n.sons[namePos].sym if s == nil: internalError(n.info, "rodwrite.process") if n.sons[bodyPos] == nil: internalError(n.info, "rodwrite.process: body is nil") if n.sons[bodyPos].kind != nkEmpty or s.magic != mNone or sfForward notin s.flags: addInterfaceSym(w, s) + of nkMethodDef: + let s = n.sons[namePos].sym + if s == nil: internalError(n.info, "rodwrite.process") + if n.sons[bodyPos] == nil: + internalError(n.info, "rodwrite.process: body is nil") + if n.sons[bodyPos].kind != nkEmpty or s.magic != mNone or + sfForward notin s.flags: + pushSym(w, s) + processStacks(w, false) + of nkVarSection, nkLetSection, nkConstSection: for i in countup(0, sonsLen(n) - 1): var a = n.sons[i] @@ -621,9 +635,9 @@ proc process(c: PPassContext, n: PNode): PNode = else: discard -proc myOpen(module: PSym): PPassContext = +proc myOpen(g: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = if module.id < 0: internalError("rodwrite: module ID not set") - var w = newRodWriter(module.fileIdx.getHash, module) + var w = newRodWriter(module.fileIdx.getHash, module, cache) rawAddInterfaceSym(w, module) result = w diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index f04cef0ee..1105d3b67 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -11,9 +11,9 @@ ## language. import - ast, modules, passes, passaux, condsyms, + ast, modules, idents, passes, passaux, condsyms, options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs, - os, times, osproc, wordrecg, strtabs + os, times, osproc, wordrecg, strtabs, modulegraphs # we support 'cmpIgnoreStyle' natively for efficiency: from strutils import cmpIgnoreStyle, contains @@ -25,9 +25,9 @@ proc listDirs(a: VmArgs, filter: set[PathComponent]) = if kind in filter: result.add path setResult(a, result) -proc setupVM*(module: PSym; scriptName: string): PEvalContext = +proc setupVM*(module: PSym; cache: IdentCache; scriptName: string): PEvalContext = # For Nimble we need to export 'setupVM'. - result = newCtx(module) + result = newCtx(module, cache) result.mode = emRepl registerAdditionalOps(result) @@ -134,9 +134,11 @@ proc setupVM*(module: PSym; scriptName: string): PEvalContext = cbconf selfExe: setResult(a, os.getAppFilename()) -proc runNimScript*(scriptName: string; freshDefines=true) = +proc runNimScript*(cache: IdentCache; scriptName: string; + freshDefines=true) = passes.gIncludeFile = includeModule passes.gImportModule = importModule + let graph = newModuleGraph() if freshDefines: initDefines() defineSymbol("nimscript") @@ -146,15 +148,15 @@ proc runNimScript*(scriptName: string; freshDefines=true) = appendStr(searchPaths, options.libpath) - var m = makeModule(scriptName) + var m = graph.makeModule(scriptName) incl(m.flags, sfMainModule) - vm.globalCtx = setupVM(m, scriptName) + vm.globalCtx = setupVM(m, cache, scriptName) - compileSystemModule() - discard processModule(m, llStreamOpen(scriptName, fmRead), nil) + graph.compileSystemModule(cache) + discard graph.processModule(m, llStreamOpen(scriptName, fmRead), nil, cache) # ensure we load 'system.nim' again for the real non-config stuff! - resetAllModulesHard() + resetSystemArtifacts() vm.globalCtx = nil # do not remove the defined symbols #initDefines() diff --git a/compiler/sem.nim b/compiler/sem.nim index 7768833b3..02c779ef0 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -18,6 +18,8 @@ import evaltempl, patterns, parampatterns, sempass2, nimfix.pretty, semmacrosanity, semparallel, lowerings, pluginsupport, plugins.active +from modulegraphs import ModuleGraph + when defined(nimfix): import nimfix.prettybase @@ -272,7 +274,7 @@ proc tryConstExpr(c: PContext, n: PNode): PNode = msgs.gErrorMax = high(int) try: - result = evalConstExpr(c.module, e) + result = evalConstExpr(c.module, c.cache, e) if result == nil or result.kind == nkEmpty: result = nil else: @@ -293,7 +295,7 @@ proc semConstExpr(c: PContext, n: PNode): PNode = result = getConstExpr(c.module, e) if result == nil: #if e.kind == nkEmpty: globalError(n.info, errConstExprExpected) - result = evalConstExpr(c.module, e) + result = evalConstExpr(c.module, c.cache, e) if result == nil or result.kind == nkEmpty: if e.info != n.info: pushInfoContext(n.info) @@ -364,7 +366,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym, #if c.evalContext == nil: # c.evalContext = c.createEvalContext(emStatic) - result = evalMacroCall(c.module, n, nOrig, sym) + result = evalMacroCall(c.module, c.cache, n, nOrig, sym) if efNoSemCheck notin flags: result = semAfterMacroCall(c, result, sym, flags) popInfoContext() @@ -398,8 +400,8 @@ proc addCodeForGenerics(c: PContext, n: PNode) = addSon(n, prc.ast) c.lastGenericIdx = c.generics.len -proc myOpen(module: PSym): PPassContext = - var c = newContext(module) +proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = + var c = newContext(graph, module, cache) if c.p != nil: internalError(module.info, "sem.myOpen") c.semConstExpr = semConstExpr c.semExpr = semExpr @@ -428,8 +430,8 @@ proc myOpen(module: PSym): PPassContext = gNotes = ForeignPackageNotes result = c -proc myOpenCached(module: PSym, rd: PRodReader): PPassContext = - result = myOpen(module) +proc myOpenCached(graph: ModuleGraph; module: PSym; rd: PRodReader): PPassContext = + result = myOpen(graph, module, rd.cache) for m in items(rd.methods): methodDef(m, true) proc isImportSystemStmt(n: PNode): bool = diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index 2e925e386..c4116a814 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -221,14 +221,15 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add newAsgnStmt(x, call) of tyVarargs, tyOpenArray: localError(c.info, errGenerated, "cannot copy openArray") - of tyFromExpr, tyIter, tyProxy, tyBuiltInTypeClass, tyUserTypeClass, + of tyFromExpr, tyProxy, tyBuiltInTypeClass, tyUserTypeClass, tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything, - tyMutable, tyGenericParam, tyGenericBody, tyNil, tyExpr, tyStmt, - tyTypeDesc, tyGenericInvocation, tyBigNum, tyConst, tyForward: + tyGenericParam, tyGenericBody, tyNil, tyExpr, tyStmt, + tyTypeDesc, tyGenericInvocation, tyForward: internalError(c.info, "assignment requested for type: " & typeToString(t)) of tyOrdinal, tyRange, tyGenericInst, tyFieldAccessor, tyStatic, tyVar: liftBodyAux(c, lastSon(t), body, x, y) + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("liftBodyAux") proc newProcType(info: TLineInfo; owner: PSym): PType = result = newType(tyProc, owner) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 3037a6ecc..ca9b5effb 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -462,7 +462,7 @@ proc searchForBorrowProc(c: PContext, startScope: PScope, fn: PSym): PSym = call.add(newIdentNode(fn.name, fn.info)) for i in 1.. <fn.typ.n.len: let param = fn.typ.n.sons[i] - let t = skipTypes(param.typ, abstractVar-{tyTypeDesc}) + let t = skipTypes(param.typ, abstractVar-{tyTypeDesc, tyDistinct}) if t.kind == tyDistinct or param.typ.kind == tyDistinct: hasDistinct = true var x: PType if param.typ.kind == tyVar: diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 30b6e261d..2fec8c757 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -1,7 +1,7 @@ # # # The Nim Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2016 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -13,7 +13,8 @@ import strutils, lists, intsets, options, lexer, ast, astalgo, trees, treetab, wordrecg, ropes, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math, - magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef + magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef, + modulegraphs type TOptionEntry* = object of lists.TListEntry # entries to put on a @@ -106,7 +107,10 @@ type instTypeBoundOp*: proc (c: PContext; dc: PSym; t: PType; info: TLineInfo; op: TTypeAttachedOp; col: int): PSym {.nimcall.} selfName*: PIdent + cache*: IdentCache + graph*: ModuleGraph signatures*: TStrTable + recursiveDep*: string proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair = result.genericSym = s @@ -116,29 +120,13 @@ proc filename*(c: PContext): string = # the module's filename return c.module.filename -proc newContext*(module: PSym): PContext - -proc lastOptionEntry*(c: PContext): POptionEntry -proc newOptionEntry*(): POptionEntry -proc newLib*(kind: TLibKind): PLib -proc addToLib*(lib: PLib, sym: PSym) -proc makePtrType*(c: PContext, baseType: PType): PType -proc newTypeS*(kind: TTypeKind, c: PContext): PType -proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext) - proc scopeDepth*(c: PContext): int {.inline.} = result = if c.currentScope != nil: c.currentScope.depthLevel else: 0 -# owner handling: -proc getCurrOwner*(): PSym -proc pushOwner*(owner: PSym) -proc popOwner*() -# implementation - var gOwners*: seq[PSym] = @[] -proc getCurrOwner(): PSym = +proc getCurrOwner*(): PSym = # owner stack (used for initializing the # owner field of syms) # the documentation comment always gets @@ -146,27 +134,27 @@ proc getCurrOwner(): PSym = # BUGFIX: global array is needed! result = gOwners[high(gOwners)] -proc pushOwner(owner: PSym) = +proc pushOwner*(owner: PSym) = add(gOwners, owner) -proc popOwner() = +proc popOwner*() = var length = len(gOwners) if length > 0: setLen(gOwners, length - 1) else: internalError("popOwner") -proc lastOptionEntry(c: PContext): POptionEntry = +proc lastOptionEntry*(c: PContext): POptionEntry = result = POptionEntry(c.optionStack.tail) proc popProcCon*(c: PContext) {.inline.} = c.p = c.p.next -proc newOptionEntry(): POptionEntry = +proc newOptionEntry*(): POptionEntry = new(result) result.options = gOptions result.defaultCC = ccDefault result.dynlib = nil result.notes = gNotes -proc newContext(module: PSym): PContext = +proc newContext*(graph: ModuleGraph; module: PSym; cache: IdentCache): PContext = new(result) result.ambiguousSymbols = initIntSet() initLinkedList(result.optionStack) @@ -180,6 +168,8 @@ proc newContext(module: PSym): PContext = initStrTable(result.userPragmas) result.generics = @[] result.unknownIdents = initIntSet() + result.cache = cache + result.graph = graph initStrTable(result.signatures) @@ -196,16 +186,19 @@ proc addConverter*(c: PContext, conv: PSym) = proc addPattern*(c: PContext, p: PSym) = inclSym(c.patterns, p) -proc newLib(kind: TLibKind): PLib = +proc newLib*(kind: TLibKind): PLib = new(result) result.kind = kind #initObjectSet(result.syms) -proc addToLib(lib: PLib, sym: PSym) = +proc addToLib*(lib: PLib, sym: PSym) = #if sym.annex != nil and not isGenericRoutine(sym): # LocalError(sym.info, errInvalidPragma) sym.annex = lib -proc makePtrType(c: PContext, baseType: PType): PType = +proc newTypeS*(kind: TTypeKind, c: PContext): PType = + result = newType(kind, getCurrOwner()) + +proc makePtrType*(c: PContext, baseType: PType): PType = result = newTypeS(tyPtr, c) addSonSkipIntLit(result, baseType.assertNotNil) @@ -222,7 +215,7 @@ proc makeTypeDesc*(c: PContext, typ: PType): PType = proc makeTypeSymNode*(c: PContext, typ: PType, info: TLineInfo): PNode = let typedesc = makeTypeDesc(c, typ) - let sym = newSym(skType, idAnon, getCurrOwner(), info).linkTo(typedesc) + let sym = newSym(skType, c.cache.idAnon, getCurrOwner(), info).linkTo(typedesc) return newSymNode(sym, info) proc makeTypeFromExpr*(c: PContext, n: PNode): PType = @@ -284,9 +277,6 @@ template rangeHasStaticIf*(t: PType): bool = template getStaticTypeFromRange*(t: PType): PType = t.n[1][0][1].typ -proc newTypeS(kind: TTypeKind, c: PContext): PType = - result = newType(kind, getCurrOwner()) - proc errorType*(c: PContext): PType = ## creates a type representing an error state result = newTypeS(tyError, c) @@ -295,7 +285,7 @@ proc errorNode*(c: PContext, n: PNode): PNode = result = newNodeI(nkEmpty, n.info) result.typ = errorType(c) -proc fillTypeS(dest: PType, kind: TTypeKind, c: PContext) = +proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext) = dest.kind = kind dest.owner = getCurrOwner() dest.size = - 1 diff --git a/compiler/semdestruct.nim b/compiler/semdestruct.nim index 9ea581f3a..85d106056 100644 --- a/compiler/semdestruct.nim +++ b/compiler/semdestruct.nim @@ -124,7 +124,7 @@ proc instantiateDestructor(c: PContext, typ: PType): PType = # destructor that must be used for the varialbe. # The destructor is either user-defined or automatically # generated by the compiler in a member-wise fashion. - var t = skipTypes(typ, {tyConst, tyMutable}).skipGenericAlias + var t = typ.skipGenericAlias let typeHoldingUserDefinition = if t.kind == tyGenericInst: t.base else: t if typeHoldingUserDefinition.destructor != nil: diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index fbbaaf483..8aaf4f9d8 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -607,12 +607,12 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode = call.add(a) #echo "NOW evaluating at compile time: ", call.renderTree if sfCompileTime in callee.flags: - result = evalStaticExpr(c.module, call, c.p.owner) + result = evalStaticExpr(c.module, c.cache, call, c.p.owner) if result.isNil: localError(n.info, errCannotInterpretNodeX, renderTree(call)) else: result = fixupTypeAfterEval(c, result, n) else: - result = evalConstExpr(c.module, call) + result = evalConstExpr(c.module, c.cache, call) if result.isNil: result = n else: result = fixupTypeAfterEval(c, result, n) #if result != n: @@ -620,7 +620,7 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode = proc semStaticExpr(c: PContext, n: PNode): PNode = let a = semExpr(c, n.sons[0]) - result = evalStaticExpr(c.module, a, c.p.owner) + result = evalStaticExpr(c.module, c.cache, a, c.p.owner) if result.isNil: localError(n.info, errCannotInterpretNodeX, renderTree(n)) result = emptyNode @@ -695,6 +695,22 @@ proc semBracketedMacro(c: PContext; outer, inner: PNode; s: PSym; else: assert(false) return +proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode = + result = n + let callee = result.sons[0].sym + case callee.kind + of skMacro: result = semMacroExpr(c, result, orig, callee, flags) + of skTemplate: result = semTemplateExpr(c, result, callee, flags) + else: + semFinishOperands(c, result) + activate(c, result) + fixAbstractType(c, result) + analyseIfAddressTakenInCall(c, result) + if callee.magic != mNone: + result = magicsAfterOverloadResolution(c, result, flags) + if c.inTypeClass == 0: + result = evalAtCompileTime(c, result) + proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = result = nil checkMinSonsLen(n, 1) @@ -773,27 +789,11 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = # See bug #904 of how to trigger it: return result #result = afterCallActions(c, result, nOrig, flags) - fixAbstractType(c, result) - analyseIfAddressTakenInCall(c, result) - if result.sons[0].kind == nkSym and result.sons[0].sym.magic != mNone: - result = magicsAfterOverloadResolution(c, result, flags) - result = evalAtCompileTime(c, result) - -proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode = - result = n - let callee = result.sons[0].sym - case callee.kind - of skMacro: result = semMacroExpr(c, result, orig, callee, flags) - of skTemplate: result = semTemplateExpr(c, result, callee, flags) + if result.sons[0].kind == nkSym: + result = afterCallActions(c, result, nOrig, flags) else: - semFinishOperands(c, result) - activate(c, result) fixAbstractType(c, result) analyseIfAddressTakenInCall(c, result) - if callee.magic != mNone: - result = magicsAfterOverloadResolution(c, result, flags) - if c.inTypeClass == 0: - result = evalAtCompileTime(c, result) proc semDirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = # this seems to be a hotspot in the compiler! @@ -861,7 +861,7 @@ proc lookupInRecordAndBuildCheck(c: PContext, n, r: PNode, field: PIdent, s = newNodeIT(nkCurly, n.info, setType) for j in countup(0, sonsLen(it) - 2): addSon(s, copyTree(it.sons[j])) var inExpr = newNodeIT(nkCall, n.info, getSysType(tyBool)) - addSon(inExpr, newSymNode(ast.opContains, n.info)) + addSon(inExpr, newSymNode(opContains, n.info)) addSon(inExpr, s) addSon(inExpr, copyTree(r.sons[0])) addSon(check, inExpr) @@ -874,11 +874,11 @@ proc lookupInRecordAndBuildCheck(c: PContext, n, r: PNode, field: PIdent, check = newNodeI(nkCheckedFieldExpr, n.info) addSon(check, ast.emptyNode) # make space for access node var inExpr = newNodeIT(nkCall, n.info, getSysType(tyBool)) - addSon(inExpr, newSymNode(ast.opContains, n.info)) + addSon(inExpr, newSymNode(opContains, n.info)) addSon(inExpr, s) addSon(inExpr, copyTree(r.sons[0])) var notExpr = newNodeIT(nkCall, n.info, getSysType(tyBool)) - addSon(notExpr, newSymNode(ast.opNot, n.info)) + addSon(notExpr, newSymNode(opNot, n.info)) addSon(notExpr, inExpr) addSon(check, notExpr) return @@ -1551,7 +1551,7 @@ proc expectMacroOrTemplateCall(c: PContext, n: PNode): PSym = if isCallExpr(n): var expandedSym = qualifiedLookUp(c, n[0], {checkUndeclared}) if expandedSym == nil: - localError(n.info, errUndeclaredIdentifier, n[0].renderTree) + errorUndeclaredIdentifier(c, n.info, n[0].renderTree) return errorSym(c, n[0]) if expandedSym.kind notin {skMacro, skTemplate}: @@ -1574,9 +1574,9 @@ proc getMagicSym(magic: TMagic): PSym = result = newSym(skProc, getIdent($magic), systemModule, gCodegenLineInfo) result.magic = magic -proc newAnonSym(kind: TSymKind, info: TLineInfo, +proc newAnonSym(c: PContext; kind: TSymKind, info: TLineInfo, owner = getCurrOwner()): PSym = - result = newSym(kind, idAnon, owner, info) + result = newSym(kind, c.cache.idAnon, owner, info) result.flags = {sfGenSym} proc semExpandToAst(c: PContext, n: PNode): PNode = @@ -1648,7 +1648,7 @@ proc semQuoteAst(c: PContext, n: PNode): PNode = processQuotations(doBlk.sons[bodyPos], op, quotes, ids) - doBlk.sons[namePos] = newAnonSym(skTemplate, n.info).newSymNode + doBlk.sons[namePos] = newAnonSym(c, skTemplate, n.info).newSymNode if ids.len > 0: doBlk.sons[paramsPos] = newNodeI(nkFormalParams, n.info) doBlk[paramsPos].add getSysSym("stmt").newSymNode # return type diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index b8451865e..ab0ce7c4c 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -107,7 +107,7 @@ proc lookup(c: PContext, n: PNode, flags: TSemGenericFlags, var s = searchInScopes(c, ident).skipAlias(n) if s == nil: if ident.id notin ctx.toMixin and withinMixin notin flags: - localError(n.info, errUndeclaredIdentifier, ident.s) + errorUndeclaredIdentifier(c, n.info, ident.s) else: if withinBind in flags: result = symChoice(c, n, s, scClosed) @@ -195,7 +195,7 @@ proc semGenericStmt(c: PContext, n: PNode, if s == nil and withinMixin notin flags and fn.kind in {nkIdent, nkAccQuoted} and considerQuotedIdent(fn).id notin ctx.toMixin: - localError(n.info, errUndeclaredIdentifier, fn.renderTree) + errorUndeclaredIdentifier(c, n.info, fn.renderTree) var first = 0 var mixinContext = false diff --git a/compiler/seminst.nim b/compiler/seminst.nim index d7cad6a2f..e1a65da74 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -58,7 +58,7 @@ iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable): PSym for i, a in n.pairs: internalAssert a.kind == nkSym var q = a.sym - if q.typ.kind notin {tyTypeDesc, tyGenericParam, tyStatic, tyIter}+tyTypeClasses: + if q.typ.kind notin {tyTypeDesc, tyGenericParam, tyStatic}+tyTypeClasses: continue let symKind = if q.typ.kind == tyStatic: skConst else: skType var s = newSym(symKind, q.name, getCurrOwner(), q.info) @@ -99,7 +99,8 @@ proc genericCacheGet(genericSym: PSym, entry: TInstantiation; proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) = # we need to create a fresh set of gensym'ed symbols: - if n.kind == nkSym and sfGenSym in n.sym.flags and n.sym.owner == orig: + if n.kind == nkSym and sfGenSym in n.sym.flags and + (n.sym.owner == orig or n.sym.owner.kind == skPackage): let s = n.sym var x = PSym(idTableGet(symMap, s)) if x == nil: diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 806b00db6..e72172c81 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -24,7 +24,7 @@ proc semTypeOf(c: PContext; n: PNode): PNode = result = newNodeI(nkTypeOfExpr, n.info) let typExpr = semExprWithType(c, n, {efInTypeof}) result.add typExpr - result.typ = makeTypeDesc(c, typExpr.typ.skipTypes({tyTypeDesc, tyIter})) + result.typ = makeTypeDesc(c, typExpr.typ.skipTypes({tyTypeDesc})) type SemAsgnMode = enum asgnNormal, noOverloadedSubscript, noOverloadedAsgn @@ -143,7 +143,7 @@ proc semBindSym(c: PContext, n: PNode): PNode = var sc = symChoice(c, id, s, TSymChoiceRule(isMixin.intVal)) result.add(sc) else: - localError(n.sons[1].info, errUndeclaredIdentifier, sl.strVal) + errorUndeclaredIdentifier(c, n.sons[1].info, sl.strVal) proc semShallowCopy(c: PContext, n: PNode, flags: TExprFlags): PNode diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index df9b3f69c..8aa8f15c8 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -204,17 +204,22 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool; cycleCheck: var IntSet) = let u = s.gcUnsafetyReason if u != nil and not cycleCheck.containsOrIncl(u.id): let msgKind = if onlyWarning: warnGcUnsafe2 else: errGenerated - if u.kind in {skLet, skVar}: + case u.kind + of skLet, skVar: message(s.info, msgKind, ("'$#' is not GC-safe as it accesses '$#'" & " which is a global using GC'ed memory") % [s.name.s, u.name.s]) - elif u.kind in routineKinds: + of routineKinds: # recursive call *always* produces only a warning so the full error # message is printed: listGcUnsafety(u, true, cycleCheck) message(s.info, msgKind, "'$#' is not GC-safe as it calls '$#'" % [s.name.s, u.name.s]) + of skParam: + message(s.info, msgKind, + "'$#' is not GC-safe as it performs an indirect call via '$#'" % + [s.name.s, u.name.s]) else: internalAssert u.kind == skUnknown message(u.info, msgKind, @@ -721,7 +726,8 @@ proc track(tracked: PEffects, n: PNode) = # and it's not a recursive call: if not (a.kind == nkSym and a.sym == tracked.owner): markSideEffect(tracked, a) - for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i)) + if a.kind != nkSym or a.sym.magic != mNBindSym: + for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i)) if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, mNewSeq}: # may not look like an assignment, but it is: let arg = n.sons[1] diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index ebcff643f..0c6f6848e 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -524,7 +524,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = addDefer(c, result, v) checkNilable(v) if sfCompileTime in v.flags: hasCompileTime = true - if hasCompileTime: vm.setupCompileTimeVar(c.module, result) + if hasCompileTime: vm.setupCompileTimeVar(c.module, c.cache, result) proc semConst(c: PContext, n: PNode): PNode = result = copyNode(n) @@ -820,7 +820,7 @@ proc semAllTypeSections(c: PContext; n: PNode): PNode = if containsOrIncl(c.includedFiles, f): localError(n.info, errRecursiveDependencyX, f.toFilename) else: - let code = gIncludeFile(c.module, f) + let code = gIncludeFile(c.graph, c.module, f, c.cache) gatherStmts c, code, result excl(c.includedFiles, f) of nkStmtList: @@ -922,7 +922,7 @@ proc semProcAnnotation(c: PContext, prc: PNode; if m == nil: if key.kind == nkIdent and key.ident.id == ord(wDelegator): if considerQuotedIdent(prc.sons[namePos]).s == "()": - prc.sons[namePos] = newIdentNode(idDelegator, prc.info) + prc.sons[namePos] = newIdentNode(c.cache.idDelegator, prc.info) prc.sons[pragmasPos] = copyExcept(n, i) else: localError(prc.info, errOnlyACallOpCanBeDelegator) @@ -965,7 +965,7 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode = checkSonsLen(n, bodyPos + 1) var s: PSym if n[namePos].kind != nkSym: - s = newSym(skProc, idAnon, getCurrOwner(), n.info) + s = newSym(skProc, c.cache.idAnon, getCurrOwner(), n.info) s.ast = n n.sons[namePos] = newSymNode(s) else: @@ -1159,7 +1159,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, assert phase == stepRegisterSymbol if n[namePos].kind == nkEmpty: - s = newSym(kind, idAnon, getCurrOwner(), n.info) + s = newSym(kind, c.cache.idAnon, getCurrOwner(), n.info) incl(s.flags, sfUsed) isAnon = true else: @@ -1418,7 +1418,7 @@ proc evalInclude(c: PContext, n: PNode): PNode = if containsOrIncl(c.includedFiles, f): localError(n.info, errRecursiveDependencyX, f.toFilename) else: - addSon(result, semStmt(c, gIncludeFile(c.module, f))) + addSon(result, semStmt(c, gIncludeFile(c.graph, c.module, f, c.cache))) excl(c.includedFiles, f) proc setLine(n: PNode, info: TLineInfo) = @@ -1445,7 +1445,7 @@ proc semStaticStmt(c: PContext, n: PNode): PNode = #writeStackTrace() let a = semStmt(c, n.sons[0]) n.sons[0] = a - evalStaticStmt(c.module, a, c.p.owner) + evalStaticStmt(c.module, c.cache, a, c.p.owner) result = newNodeI(nkDiscardStmt, n.info, 1) result.sons[0] = emptyNode when false: diff --git a/compiler/service.nim b/compiler/service.nim index 640dd2010..ac04b7860 100644 --- a/compiler/service.nim +++ b/compiler/service.nim @@ -11,7 +11,7 @@ import times, commands, options, msgs, nimconf, - extccomp, strutils, os, platform, parseopt + extccomp, strutils, os, platform, parseopt, idents when useCaas: import net @@ -45,11 +45,11 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) = if optRun notin gGlobalOptions and arguments != "" and options.command.normalize != "run": rawMessage(errArgsNeedRunOption, []) -proc serve*(action: proc (){.nimcall.}) = +proc serve*(cache: IdentCache; action: proc (cache: IdentCache){.nimcall.}) = template execute(cmd) = curCaasCmd = cmd processCmdLine(passCmd2, cmd) - action() + action(cache) gErrorCounter = 0 let typ = getConfigVar("server.type") diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index df5a76a57..15171874f 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -549,8 +549,6 @@ proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation = of tyNil: result = f.allowsNil - of tyIter: - if tfIterator in f.flags: result = typeRel(c, f.base, a.base) else: discard proc typeRangeRel(f, a: PType): TTypeRelation {.noinline.} = @@ -1021,11 +1019,9 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = var fskip = skippedNone let aobj = x.skipToObject(askip) let fobj = genericBody.lastSon.skipToObject(fskip) + var depth = -1 if fobj != nil and aobj != nil and askip == fskip: - let depth = isObjectSubtype(c, aobj, fobj, f) - if depth >= 0: - c.inheritancePenalty += depth - return if depth == 0: isGeneric else: isSubtype + depth = isObjectSubtype(c, aobj, fobj, f) result = typeRel(c, genericBody, x) if result != isNone: # see tests/generics/tgeneric3.nim for an example that triggers this @@ -1047,6 +1043,11 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = else: put(c, f.sons[i], x) + if depth >= 0: + c.inheritancePenalty += depth + # bug #4863: We still need to bind generic alias crap, so + # we cannot return immediately: + result = if depth == 0: isGeneric else: isSubtype of tyAnd: considerPreviousT: result = isEqual @@ -1220,13 +1221,6 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = else: result = isNone - of tyIter: - if a.kind == tyIter or - (a.kind == tyProc and tfIterator in a.flags): - result = typeRel(c, f.base, a.base) - else: - result = isNone - of tyStmt: if aOrig != nil and tfOldSchoolExprStmt notin f.flags: put(c, f, aOrig) @@ -1584,8 +1578,9 @@ proc prepareOperand(c: PContext; formal: PType; a: PNode): PNode = elif a.typ.isNil: # XXX This is unsound! 'formal' can differ from overloaded routine to # overloaded routine! - let flags = if formal.kind == tyIter: {efDetermineType, efWantIterator} - else: {efDetermineType, efAllowStmt} + let flags = {efDetermineType, efAllowStmt} + #if formal.kind == tyIter: {efDetermineType, efWantIterator} + #else: {efDetermineType, efAllowStmt} #elif formal.kind == tyStmt: {efDetermineType, efWantStmt} #else: {efDetermineType} result = c.semOperand(c, a, flags) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 52f00550b..39689099a 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -233,7 +233,7 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) = # error: no known module name: typ = nil else: - let m = gImportModule(c.module, fullpath.fileInfoIdx) + let m = gImportModule(c.graph, c.module, fullpath.fileInfoIdx, c.cache) if m == nil: typ = nil else: for it in items(n.sym.tab): diff --git a/compiler/syntaxes.nim b/compiler/syntaxes.nim index 37ea6e2db..4745b1ac7 100644 --- a/compiler/syntaxes.nim +++ b/compiler/syntaxes.nim @@ -26,34 +26,11 @@ const "strip"] type - TParsers*{.final.} = object + TParsers* = object skin*: TParserKind parser*: TParser - -proc parseFile*(fileIdx: int32): PNode{.procvar.} -proc openParsers*(p: var TParsers, fileIdx: int32, inputstream: PLLStream) -proc closeParsers*(p: var TParsers) -proc parseAll*(p: var TParsers): PNode -proc parseTopLevelStmt*(p: var TParsers): PNode - # implements an iterator. Returns the next top-level statement or nil if end - # of stream. - -# implementation - -proc parseFile(fileIdx: int32): PNode = - var - p: TParsers - f: File - let filename = fileIdx.toFullPathConsiderDirty - if not open(f, filename): - rawMessage(errCannotOpenFile, filename) - return - openParsers(p, fileIdx, llStreamOpen(f)) - result = parseAll(p) - closeParsers(p) - -proc parseAll(p: var TParsers): PNode = +proc parseAll*(p: var TParsers): PNode = case p.skin of skinStandard, skinStrongSpaces: result = parser.parseAll(p.parser) @@ -63,7 +40,7 @@ proc parseAll(p: var TParsers): PNode = internalError("parser to implement") result = ast.emptyNode -proc parseTopLevelStmt(p: var TParsers): PNode = +proc parseTopLevelStmt*(p: var TParsers): PNode = case p.skin of skinStandard, skinStrongSpaces: result = parser.parseTopLevelStmt(p.parser) @@ -74,18 +51,18 @@ proc parseTopLevelStmt(p: var TParsers): PNode = result = ast.emptyNode proc utf8Bom(s: string): int = - if (s[0] == '\xEF') and (s[1] == '\xBB') and (s[2] == '\xBF'): + if s[0] == '\xEF' and s[1] == '\xBB' and s[2] == '\xBF': result = 3 else: result = 0 proc containsShebang(s: string, i: int): bool = - if (s[i] == '#') and (s[i + 1] == '!'): + if s[i] == '#' and s[i+1] == '!': var j = i + 2 while s[j] in Whitespace: inc(j) result = s[j] == '/' -proc parsePipe(filename: string, inputStream: PLLStream): PNode = +proc parsePipe(filename: string, inputStream: PLLStream; cache: IdentCache): PNode = result = ast.emptyNode var s = llStreamOpen(filename, fmRead) if s != nil: @@ -101,20 +78,20 @@ proc parsePipe(filename: string, inputStream: PLLStream): PNode = inc(i, 2) while line[i] in Whitespace: inc(i) var q: TParser - openParser(q, filename, llStreamOpen(substr(line, i))) + parser.openParser(q, filename, llStreamOpen(substr(line, i)), cache) result = parser.parseAll(q) - closeParser(q) + parser.closeParser(q) llStreamClose(s) proc getFilter(ident: PIdent): TFilterKind = for i in countup(low(TFilterKind), high(TFilterKind)): - if identEq(ident, filterNames[i]): + if cmpIgnoreStyle(ident.s, filterNames[i]) == 0: return i result = filtNone proc getParser(ident: PIdent): TParserKind = for i in countup(low(TParserKind), high(TParserKind)): - if identEq(ident, parserNames[i]): + if cmpIgnoreStyle(ident.s, parserNames[i]) == 0: return i rawMessage(errInvalidDirectiveX, ident.s) @@ -150,8 +127,7 @@ proc evalPipe(p: var TParsers, n: PNode, filename: string, start: PLLStream): PLLStream = result = start if n.kind == nkEmpty: return - if n.kind == nkInfix and n.sons[0].kind == nkIdent and - identEq(n.sons[0].ident, "|"): + if n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.s == "|": for i in countup(1, 2): if n.sons[i].kind == nkInfix: result = evalPipe(p, n.sons[i], filename, result) @@ -162,18 +138,31 @@ proc evalPipe(p: var TParsers, n: PNode, filename: string, else: result = applyFilter(p, n, filename, result) -proc openParsers(p: var TParsers, fileIdx: int32, inputstream: PLLStream) = +proc openParsers*(p: var TParsers, fileIdx: int32, inputstream: PLLStream; + cache: IdentCache) = var s: PLLStream p.skin = skinStandard let filename = fileIdx.toFullPathConsiderDirty - var pipe = parsePipe(filename, inputstream) + var pipe = parsePipe(filename, inputstream, cache) if pipe != nil: s = evalPipe(p, pipe, filename, inputstream) else: s = inputstream case p.skin of skinStandard, skinBraces, skinEndX: - parser.openParser(p.parser, fileIdx, s, false) + parser.openParser(p.parser, fileIdx, s, cache, false) of skinStrongSpaces: - parser.openParser(p.parser, fileIdx, s, true) + parser.openParser(p.parser, fileIdx, s, cache, true) -proc closeParsers(p: var TParsers) = +proc closeParsers*(p: var TParsers) = parser.closeParser(p.parser) + +proc parseFile*(fileIdx: int32; cache: IdentCache): PNode {.procvar.} = + var + p: TParsers + f: File + let filename = fileIdx.toFullPathConsiderDirty + if not open(f, filename): + rawMessage(errCannotOpenFile, filename) + return + openParsers(p, fileIdx, llStreamOpen(f), cache) + result = parseAll(p) + closeParsers(p) diff --git a/compiler/trees.nim b/compiler/trees.nim index a629b3834..08a1a8c1f 100644 --- a/compiler/trees.nim +++ b/compiler/trees.nim @@ -10,7 +10,7 @@ # tree helper routines import - ast, astalgo, lexer, msgs, strutils, wordrecg + ast, astalgo, lexer, msgs, strutils, wordrecg, idents proc cyclicTreeAux(n: PNode, visited: var seq[PNode]): bool = if n == nil: return diff --git a/compiler/types.nim b/compiler/types.nim index 3db0c4507..fc50449ec 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -51,18 +51,14 @@ const # TODO: Remove tyTypeDesc from each abstractX and (where necessary) # replace with typedescX abstractPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyDistinct, tyOrdinal, - tyConst, tyMutable, tyTypeDesc} - abstractVar* = {tyVar, tyGenericInst, tyDistinct, tyOrdinal, - tyConst, tyMutable, tyTypeDesc} - abstractRange* = {tyGenericInst, tyRange, tyDistinct, tyOrdinal, - tyConst, tyMutable, tyTypeDesc} - abstractVarRange* = {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal, - tyConst, tyMutable, tyTypeDesc} - abstractInst* = {tyGenericInst, tyDistinct, tyConst, tyMutable, tyOrdinal, tyTypeDesc} + abstractVar* = {tyVar, tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc} + abstractRange* = {tyGenericInst, tyRange, tyDistinct, tyOrdinal, tyTypeDesc} + abstractVarRange* = {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal, + tyTypeDesc} + abstractInst* = {tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc} - skipPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyConst, tyMutable, - tyTypeDesc} + skipPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyTypeDesc} # typedescX is used if we're sure tyTypeDesc should be included (or skipped) typedescPtrs* = abstractPtrs + {tyTypeDesc} typedescInst* = abstractInst + {tyTypeDesc} @@ -116,8 +112,7 @@ proc isFloatLit*(t: PType): bool {.inline.} = proc isCompatibleToCString(a: PType): bool = if a.kind == tyArray: if (firstOrd(a.sons[0]) == 0) and - (skipTypes(a.sons[0], {tyRange, tyConst, - tyMutable, tyGenericInst}).kind in + (skipTypes(a.sons[0], {tyRange, tyGenericInst}).kind in {tyInt..tyInt64, tyUInt..tyUInt64}) and (a.sons[1].kind == tyChar): result = true @@ -151,13 +146,12 @@ proc isOrdinalType(t: PType): bool = const # caution: uint, uint64 are no ordinal types! baseKinds = {tyChar,tyInt..tyInt64,tyUInt8..tyUInt32,tyBool,tyEnum} - parentKinds = {tyRange, tyOrdinal, tyConst, tyMutable, tyGenericInst, - tyDistinct} + parentKinds = {tyRange, tyOrdinal, tyGenericInst, tyDistinct} t.kind in baseKinds or (t.kind in parentKinds and isOrdinalType(t.sons[0])) proc enumHasHoles(t: PType): bool = var b = t - while b.kind in {tyConst, tyMutable, tyRange, tyGenericInst}: b = b.sons[0] + while b.kind in {tyRange, tyGenericInst}: b = b.sons[0] result = b.kind == tyEnum and tfEnumHasHoles in b.flags proc iterOverTypeAux(marker: var IntSet, t: PType, iter: TTypeIter, @@ -275,7 +269,7 @@ proc analyseObjectWithTypeFieldAux(t: PType, if res == frHeader: result = frHeader if result == frNone: if isObjectWithTypeFieldPredicate(t): result = frHeader - of tyGenericInst, tyDistinct, tyConst, tyMutable: + of tyGenericInst, tyDistinct: result = analyseObjectWithTypeFieldAux(lastSon(t), marker) of tyArray, tyArrayConstr, tyTuple: for i in countup(0, sonsLen(t) - 1): @@ -408,8 +402,8 @@ const "int", "int8", "int16", "int32", "int64", "float", "float32", "float64", "float128", "uint", "uint8", "uint16", "uint32", "uint64", - "bignum", "const ", - "!", "varargs[$1]", "iter[$1]", "Error Type", + "unused0", "unused1", + "unused2", "varargs[$1]", "unused", "Error Type", "BuiltInTypeClass", "UserTypeClass", "UserTypeClassInst", "CompositeTypeClass", "and", "or", "not", "any", "static", "TypeFromExpr", "FieldAccessor", @@ -534,7 +528,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = add(result, typeToString(t.sons[i])) if i < sonsLen(t) - 1: add(result, ", ") add(result, ')') - of tyPtr, tyRef, tyVar, tyMutable, tyConst: + of tyPtr, tyRef, tyVar: result = typeToStr[t.kind] if t.len >= 2: setLen(result, result.len-1) @@ -572,7 +566,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = addSep(prag) add(prag, "locks: " & $t.lockLevel) if len(prag) != 0: add(result, "{." & prag & ".}") - of tyVarargs, tyIter: + of tyVarargs: result = typeToStr[t.kind] % typeToString(t.sons[0]) else: result = typeToStr[t.kind] @@ -606,7 +600,7 @@ proc firstOrd(t: PType): BiggestInt = else: assert(t.n.sons[0].kind == nkSym) result = t.n.sons[0].sym.position - of tyGenericInst, tyDistinct, tyConst, tyMutable, tyTypeDesc, tyFieldAccessor: + of tyGenericInst, tyDistinct, tyTypeDesc, tyFieldAccessor: result = firstOrd(lastSon(t)) of tyOrdinal: if t.len > 0: result = firstOrd(lastSon(t)) @@ -642,8 +636,7 @@ proc lastOrd(t: PType): BiggestInt = of tyEnum: assert(t.n.sons[sonsLen(t.n) - 1].kind == nkSym) result = t.n.sons[sonsLen(t.n) - 1].sym.position - of tyGenericInst, tyDistinct, tyConst, tyMutable, - tyTypeDesc, tyFieldAccessor: + of tyGenericInst, tyDistinct, tyTypeDesc, tyFieldAccessor: result = lastOrd(lastSon(t)) of tyProxy: result = 0 of tyOrdinal: @@ -656,7 +649,7 @@ proc lastOrd(t: PType): BiggestInt = proc lengthOrd(t: PType): BiggestInt = case t.kind of tyInt64, tyInt32, tyInt: result = lastOrd(t) - of tyDistinct, tyConst, tyMutable: result = lengthOrd(t.sons[0]) + of tyDistinct: result = lengthOrd(t.sons[0]) else: let last = lastOrd t let first = firstOrd t @@ -925,7 +918,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = case a.kind of tyEmpty, tyChar, tyBool, tyNil, tyPointer, tyString, tyCString, - tyInt..tyBigNum, tyStmt, tyExpr, tyVoid: + tyInt..tyUInt64, tyStmt, tyExpr, tyVoid: result = sameFlags(a, b) of tyStatic, tyFromExpr: result = exprStructuralEquivalent(a.n, b.n) and sameFlags(a, b) @@ -965,8 +958,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = result = a.sym.position == b.sym.position of tyGenericInvocation, tyGenericBody, tySequence, tyOpenArray, tySet, tyRef, tyPtr, tyVar, tyArrayConstr, - tyArray, tyProc, tyConst, tyMutable, tyVarargs, tyIter, - tyOrdinal, tyTypeClasses, tyFieldAccessor: + tyArray, tyProc, tyVarargs, tyOrdinal, tyTypeClasses, tyFieldAccessor: cycleCheck() if a.kind == tyUserTypeClass and a.n != nil: return a.n == b.n result = sameChildrenAux(a, b, c) and sameFlags(a, b) @@ -980,6 +972,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = sameValue(a.n.sons[1], b.n.sons[1]) of tyGenericInst: discard of tyNone: result = false + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("sameFlags") proc sameBackendType*(x, y: PType): bool = var c = initSameTypeClosure() @@ -1120,7 +1113,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = t of tyNil: if kind != skConst: result = t - of tyString, tyBool, tyChar, tyEnum, tyInt..tyBigNum, tyCString, tyPointer: + of tyString, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString, tyPointer: result = nil of tyOrdinal: if kind != skParam: result = t @@ -1143,7 +1136,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, else: result = typeAllowedAux(marker, t.lastSon, skVar, flags+{taHeap}) of tyPtr: result = typeAllowedAux(marker, t.lastSon, skVar, flags+{taHeap}) - of tyArrayConstr, tySet, tyConst, tyMutable, tyIter: + of tyArrayConstr, tySet: for i in countup(0, sonsLen(t) - 1): result = typeAllowedAux(marker, t.sons[i], kind, flags) if result != nil: break @@ -1160,6 +1153,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, # for now same as error node; we say it's a valid type as it should # prevent cascading errors: result = nil + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("typeAllowedAux") proc typeAllowed*(t: PType, kind: TSymKind): PType = # returns 'nil' on success and otherwise the part of the type that is @@ -1250,8 +1244,7 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt = if typ.callConv == ccClosure: result = 2 * ptrSize else: result = ptrSize a = ptrSize - of tyNil, tyCString, tyString, tySequence, tyPtr, tyRef, tyVar, tyOpenArray, - tyBigNum: + of tyNil, tyCString, tyString, tySequence, tyPtr, tyRef, tyVar, tyOpenArray: let base = typ.lastSon if base == typ or (base.kind == tyTuple and base.size==szIllegalRecursion): result = szIllegalRecursion @@ -1311,7 +1304,7 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt = if result < 0: return if a < maxAlign: a = maxAlign result = align(result, a) - of tyGenericInst, tyDistinct, tyGenericBody, tyMutable, tyConst, tyIter: + of tyGenericInst, tyDistinct, tyGenericBody: result = computeSizeAux(lastSon(typ), a) of tyTypeDesc: result = computeSizeAux(typ.base, a) @@ -1374,7 +1367,7 @@ proc safeInheritanceDiff*(a, b: PType): int = if a.kind == tyError or b.kind == tyError: result = -1 else: - result = inheritanceDiff(a, b) + result = inheritanceDiff(a.skipTypes(skipPtrs), b.skipTypes(skipPtrs)) proc compatibleEffectsAux(se, re: PNode): bool = if re.isNil: return false diff --git a/compiler/typesrenderer.nim b/compiler/typesrenderer.nim index 438744b1c..e9c27ac9d 100644 --- a/compiler/typesrenderer.nim +++ b/compiler/typesrenderer.nim @@ -37,14 +37,20 @@ proc renderType(n: PNode): string = of nkIdent: result = n.ident.s of nkSym: result = typeToString(n.sym.typ) of nkVarTy: - assert len(n) == 1 - result = renderType(n[0]) + if n.len == 1: + result = renderType(n[0]) + else: + result = "var" of nkRefTy: - assert len(n) == 1 - result = "ref." & renderType(n[0]) + if n.len == 1: + result = "ref." & renderType(n[0]) + else: + result = "ref" of nkPtrTy: - assert len(n) == 1 - result = "ptr." & renderType(n[0]) + if n.len == 1: + result = "ptr." & renderType(n[0]) + else: + result = "ptr" of nkProcTy: assert len(n) > 1 let params = n[0] diff --git a/compiler/vm.nim b/compiler/vm.nim index efcc55c59..1bb440c6c 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -24,6 +24,8 @@ import from semfold import leValueConv, ordinalValToString from evaltempl import evalTemplate +from modulegraphs import ModuleGraph + when hasFFI: import evalffi @@ -76,12 +78,13 @@ proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) = msgWriteln(s) proc stackTrace(c: PCtx, tos: PStackFrame, pc: int, - msg: TMsgKind, arg = "") = + msg: TMsgKind, arg = "", n: PNode = nil) = msgWriteln("stack trace: (most recent call last)") stackTraceAux(c, tos, pc) # XXX test if we want 'globalError' for every mode - if c.mode == emRepl: globalError(c.debug[pc], msg, arg) - else: localError(c.debug[pc], msg, arg) + let lineInfo = if n == nil: c.debug[pc] else: n.info + if c.mode == emRepl: globalError(lineInfo, msg, arg) + else: localError(lineInfo, msg, arg) proc bailOut(c: PCtx; tos: PStackFrame) = stackTrace(c, tos, c.exceptionInstr, errUnhandledExceptionX, @@ -900,7 +903,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = if newPc < pc: handleJmpBack() #echo "new pc ", newPc, " calling: ", prc.name.s var newFrame = PStackFrame(prc: prc, comesFrom: pc, next: tos) - newSeq(newFrame.slots, prc.offset) + newSeq(newFrame.slots, prc.offset+ord(isClosure)) if not isEmptyType(prc.typ.sons[0]) or prc.kind == skMacro: putIntoReg(newFrame.slots[0], getNullValue(prc.typ.sons[0], prc.info)) for i in 1 .. rc-1: @@ -1242,9 +1245,13 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = createStr regs[ra] regs[ra].node.strVal = opGorge(regs[rb].node.strVal, - regs[rc].node.strVal, regs[rd].node.strVal) + regs[rc].node.strVal, regs[rd].node.strVal, + c.debug[pc]) of opcNError: - stackTrace(c, tos, pc, errUser, regs[ra].node.strVal) + decodeB(rkNode) + let a = regs[ra].node + let b = regs[rb].node + stackTrace(c, tos, pc, errUser, a.strVal, if b.kind == nkNilLit: nil else: b) of opcNWarning: message(c.debug[pc], warnUser, regs[ra].node.strVal) of opcNHint: @@ -1253,7 +1260,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = decodeB(rkNode) # c.debug[pc].line.int - countLines(regs[rb].strVal) ? var error: string - let ast = parseString(regs[rb].node.strVal, c.debug[pc].toFullPath, + let ast = parseString(regs[rb].node.strVal, c.cache, c.debug[pc].toFullPath, c.debug[pc].line.int, proc (info: TLineInfo; msg: TMsgKind; arg: string) = if error.isNil and msg <= msgs.errMax: @@ -1267,7 +1274,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of opcParseStmtToAst: decodeB(rkNode) var error: string - let ast = parseString(regs[rb].node.strVal, c.debug[pc].toFullPath, + let ast = parseString(regs[rb].node.strVal, c.cache, c.debug[pc].toFullPath, c.debug[pc].line.int, proc (info: TLineInfo; msg: TMsgKind; arg: string) = if error.isNil and msg <= msgs.errMax: @@ -1421,7 +1428,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = else: regs[rc].node.strVal if k < 0 or k > ord(high(TSymKind)): internalError(c.debug[pc], "request to create symbol of invalid kind") - var sym = newSym(k.TSymKind, name.getIdent, c.module, c.debug[pc]) + var sym = newSym(k.TSymKind, name.getIdent, c.module.owner, c.debug[pc]) incl(sym.flags, sfGenSym) regs[ra].node = newSymNode(sym) of opcTypeTrait: @@ -1504,20 +1511,20 @@ include vmops var globalCtx*: PCtx -proc setupGlobalCtx(module: PSym) = +proc setupGlobalCtx(module: PSym; cache: IdentCache) = if globalCtx.isNil: - globalCtx = newCtx(module) + globalCtx = newCtx(module, cache) registerAdditionalOps(globalCtx) else: refresh(globalCtx, module) -proc myOpen(module: PSym): PPassContext = +proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = #var c = newEvalContext(module, emRepl) #c.features = {allowCast, allowFFI, allowInfiniteLoops} #pushStackFrame(c, newStackFrame()) # XXX produce a new 'globals' environment here: - setupGlobalCtx(module) + setupGlobalCtx(module, cache) result = globalCtx when hasFFI: globalCtx.features = {allowFFI, allowCast} @@ -1535,9 +1542,10 @@ proc myProcess(c: PPassContext, n: PNode): PNode = const evalPass* = makePass(myOpen, nil, myProcess, myProcess) -proc evalConstExprAux(module, prc: PSym, n: PNode, mode: TEvalMode): PNode = +proc evalConstExprAux(module: PSym; cache: IdentCache; prc: PSym, n: PNode, + mode: TEvalMode): PNode = let n = transformExpr(module, n) - setupGlobalCtx(module) + setupGlobalCtx(module, cache) var c = globalCtx let oldMode = c.mode defer: c.mode = oldMode @@ -1552,17 +1560,17 @@ proc evalConstExprAux(module, prc: PSym, n: PNode, mode: TEvalMode): PNode = result = rawExecute(c, start, tos).regToNode if result.info.line < 0: result.info = n.info -proc evalConstExpr*(module: PSym, e: PNode): PNode = - result = evalConstExprAux(module, nil, e, emConst) +proc evalConstExpr*(module: PSym; cache: IdentCache, e: PNode): PNode = + result = evalConstExprAux(module, cache, nil, e, emConst) -proc evalStaticExpr*(module: PSym, e: PNode, prc: PSym): PNode = - result = evalConstExprAux(module, prc, e, emStaticExpr) +proc evalStaticExpr*(module: PSym; cache: IdentCache, e: PNode, prc: PSym): PNode = + result = evalConstExprAux(module, cache, prc, e, emStaticExpr) -proc evalStaticStmt*(module: PSym, e: PNode, prc: PSym) = - discard evalConstExprAux(module, prc, e, emStaticStmt) +proc evalStaticStmt*(module: PSym; cache: IdentCache, e: PNode, prc: PSym) = + discard evalConstExprAux(module, cache, prc, e, emStaticStmt) -proc setupCompileTimeVar*(module: PSym, n: PNode) = - discard evalConstExprAux(module, nil, n, emStaticStmt) +proc setupCompileTimeVar*(module: PSym; cache: IdentCache, n: PNode) = + discard evalConstExprAux(module, cache, nil, n, emStaticStmt) proc setupMacroParam(x: PNode, typ: PType): TFullReg = case typ.kind @@ -1581,7 +1589,8 @@ proc setupMacroParam(x: PNode, typ: PType): TFullReg = var evalMacroCounter: int -proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode = +proc evalMacroCall*(module: PSym; cache: IdentCache, n, nOrig: PNode, + sym: PSym): PNode = # XXX globalError() is ugly here, but I don't know a better solution for now inc(evalMacroCounter) if evalMacroCounter > 100: @@ -1594,7 +1603,7 @@ proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode = n.renderTree, $ <n.safeLen, $ <sym.typ.len]) - setupGlobalCtx(module) + setupGlobalCtx(module, cache) var c = globalCtx c.callsite = nOrig diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 83c1dbf43..7fb35e890 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -10,7 +10,7 @@ ## This module contains the type definitions for the new evaluation engine. ## An instruction is 1-3 int32s in memory, it is a register based VM. -import ast, passes, msgs, intsets +import ast, passes, msgs, idents, intsets const byteExcess* = 128 # we use excess-K for immediates @@ -203,16 +203,18 @@ type comesFromHeuristic*: TLineInfo # Heuristic for better macro stack traces callbacks*: seq[tuple[key: string, value: VmCallback]] errorFlag*: string + cache*: IdentCache TPosition* = distinct int PEvalContext* = PCtx -proc newCtx*(module: PSym): PCtx = +proc newCtx*(module: PSym; cache: IdentCache): PCtx = PCtx(code: @[], debug: @[], globals: newNode(nkStmtListExpr), constants: newNode(nkStmtList), types: @[], prc: PProc(blocks: @[]), module: module, loopIterations: MaxLoopIterations, - comesFromHeuristic: unknownLineInfo(), callbacks: @[], errorFlag: "") + comesFromHeuristic: unknownLineInfo(), callbacks: @[], errorFlag: "", + cache: cache) proc refresh*(c: PCtx, module: PSym) = c.module = module diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index b40ca1058..bd6908722 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -19,7 +19,8 @@ proc readOutput(p: Process): string = result.setLen(result.len - "\n".len) discard p.waitForExit -proc opGorge*(cmd, input, cache: string): string = +proc opGorge*(cmd, input, cache: string, info: TLineInfo): string = + let workingDir = parentDir(info.toFullPath) if cache.len > 0:# and optForceFullMake notin gGlobalOptions: let h = secureHash(cmd & "\t" & input & "\t" & cache) let filename = options.toGeneratedFile("gorge_" & $h, "txt") @@ -30,7 +31,8 @@ proc opGorge*(cmd, input, cache: string): string = return var readSuccessful = false try: - var p = startProcess(cmd, options={poEvalCommand, poStderrToStdout}) + var p = startProcess(cmd, workingDir, + options={poEvalCommand, poStderrToStdout}) if input.len != 0: p.inputStream.write(input) p.inputStream.close() @@ -41,7 +43,8 @@ proc opGorge*(cmd, input, cache: string): string = if not readSuccessful: result = "" else: try: - var p = startProcess(cmd, options={poEvalCommand, poStderrToStdout}) + var p = startProcess(cmd, workingDir, + options={poEvalCommand, poStderrToStdout}) if input.len != 0: p.inputStream.write(input) p.inputStream.close() @@ -266,11 +269,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; of tyUInt16: result = atomicType("uint16", mUint16) of tyUInt32: result = atomicType("uint32", mUint32) of tyUInt64: result = atomicType("uint64", mUint64) - of tyBigNum: result = atomicType("bignum", mNone) - of tyConst: result = mapTypeToBracket("const", mNone, t, info) - of tyMutable: result = mapTypeToBracket("mutable", mNone, t, info) of tyVarargs: result = mapTypeToBracket("varargs", mVarargs, t, info) - of tyIter: result = mapTypeToBracket("iter", mNone, t, info) of tyProxy: result = atomicType("error", mNone) of tyBuiltInTypeClass: result = mapTypeToBracket("builtinTypeClass", mNone, t, info) @@ -292,6 +291,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; result.add atomicType("static", mNone) if t.n != nil: result.add t.n.copyTree + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("mapTypeToAstX") proc opMapTypeToAst*(t: PType; info: TLineInfo): PNode = result = mapTypeToAstX(t, info, false, true) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 6bfc33f00..ed8f3f338 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1068,7 +1068,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = else: # setter unused(n, dest) - genUnaryStmt(c, n, opcNError) + genBinaryStmt(c, n, opcNError) of mNCallSite: if dest < 0: dest = c.getTemp(n.typ) c.gABC(n, opcCallSite, dest) diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index b5ffd51c2..cf66b6358 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -13,8 +13,7 @@ # does not support strings. Without this the code would # be slow and unreadable. -import - hashes, strutils, idents +from strutils import cmpIgnoreStyle # Keywords must be kept sorted and within a range @@ -35,7 +34,7 @@ type wColon, wColonColon, wEquals, wDot, wDotDot, wStar, wMinus, - wMagic, wThread, wFinal, wProfiler, wObjChecks, + wMagic, wThread, wFinal, wProfiler, wMemTracker, wObjChecks, wIntDefine, wStrDefine, wDestroy, @@ -122,7 +121,7 @@ const ":", "::", "=", ".", "..", "*", "-", - "magic", "thread", "final", "profiler", "objchecks", "intdefine", "strdefine", + "magic", "thread", "final", "profiler", "memtracker", "objchecks", "intdefine", "strdefine", "destroy", @@ -180,17 +179,3 @@ proc findStr*(a: openArray[string], s: string): int = if cmpIgnoreStyle(a[i], s) == 0: return i result = - 1 - -proc whichKeyword*(id: PIdent): TSpecialWord = - if id.id < 0: result = wInvalid - else: result = TSpecialWord(id.id) - -proc whichKeyword*(id: string): TSpecialWord = - result = whichKeyword(getIdent(id)) - -proc initSpecials() = - # initialize the keywords: - for s in countup(succ(low(specialWords)), high(specialWords)): - getIdent(specialWords[s], hashIgnoreStyle(specialWords[s])).id = ord(s) - -initSpecials() |