diff options
Diffstat (limited to 'compiler')
31 files changed, 900 insertions, 954 deletions
diff --git a/compiler/ast.nim b/compiler/ast.nim index 8054e9248..2ce0afcc3 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -500,8 +500,7 @@ type skResult, # special 'result' variable skProc, # a proc skMethod, # a method - skIterator, # an inline iterator - skClosureIterator, # a resumable closure iterator + skIterator, # an iterator skConverter, # a type converter skMacro, # a macro skTemplate, # a template; currently also misused for user-defined @@ -518,7 +517,7 @@ type TSymKinds* = set[TSymKind] const - routineKinds* = {skProc, skMethod, skIterator, skClosureIterator, + routineKinds* = {skProc, skMethod, skIterator, skConverter, skMacro, skTemplate} tfIncompleteStruct* = tfVarargs tfUncheckedArray* = tfVarargs @@ -905,7 +904,7 @@ type # the poor naming choices in the standard library. const - OverloadableSyms* = {skProc, skMethod, skIterator, skClosureIterator, + OverloadableSyms* = {skProc, skMethod, skIterator, skConverter, skModule, skTemplate, skMacro} GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody, @@ -929,11 +928,11 @@ const NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, tySequence, tyProc, tyString, tyError} ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType, - skIterator, skClosureIterator, + skIterator, skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias} PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfDotSetter, nfDotField, - nfIsRef, nfIsCursor} + nfIsRef, nfIsCursor, nfLL} namePos* = 0 patternPos* = 1 # empty except for term rewriting macros genericParamsPos* = 2 @@ -958,11 +957,9 @@ const nkStrKinds* = {nkStrLit..nkTripleStrLit} skLocalVars* = {skVar, skLet, skForVar, skParam, skResult} - skProcKinds* = {skProc, skTemplate, skMacro, skIterator, skClosureIterator, + skProcKinds* = {skProc, skTemplate, skMacro, skIterator, skMethod, skConverter} - skIterators* = {skIterator, skClosureIterator} - var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things proc isCallExpr*(n: PNode): bool = @@ -1558,12 +1555,13 @@ proc isGenericRoutine*(s: PSym): bool = else: discard proc skipGenericOwner*(s: PSym): PSym = - internalAssert s.kind in skProcKinds ## Generic instantiations are owned by their originating generic ## symbol. This proc skips such owners and goes straight to the owner ## of the generic itself (the module or the enclosing proc). - result = if sfFromGeneric in s.flags: s.owner.owner - else: s.owner + result = if s.kind in skProcKinds and sfFromGeneric in s.flags: + s.owner.owner + else: + s.owner proc originatingModule*(s: PSym): PSym = result = s.owner diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 86ecc9db8..bd17f85e4 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -118,6 +118,14 @@ proc openArrayLoc(p: BProc, n: PNode): Rope = result = "$1->data, $1->$2" % [a.rdLoc, lenField(p)] of tyArray, tyArrayConstr: result = "$1, $2" % [rdLoc(a), rope(lengthOrd(a.t))] + of tyPtr, tyRef: + case lastSon(a.t).kind + of tyString, tySequence: + result = "(*$1)->data, (*$1)->$2" % [a.rdLoc, lenField(p)] + of tyArray, tyArrayConstr: + result = "$1, $2" % [rdLoc(a), rope(lengthOrd(lastSon(a.t)))] + else: + internalError("openArrayLoc: " & typeToString(a.t)) else: internalError("openArrayLoc: " & typeToString(a.t)) proc genArgStringToCString(p: BProc, n: PNode): Rope {.inline.} = @@ -515,7 +523,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) = line(p, cpsStmts, pl) proc genCall(p: BProc, e: PNode, d: var TLoc) = - if e.sons[0].typ.callConv == ccClosure: + if e.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure: genClosureCall(p, nil, e, d) elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags: genInfixCall(p, nil, e, d) @@ -528,7 +536,7 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) = if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d) proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = - if ri.sons[0].typ.callConv == ccClosure: + if ri.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure: genClosureCall(p, le, ri, d) elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags: genInfixCall(p, le, ri, d) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index df3b655e3..6f513c165 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1502,7 +1502,7 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = else: case op of mIncl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] |=(1U<<($2&7U));$n") - of mExcl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3]] &= ~(1U<<($2&7U));$n") + of mExcl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] &= ~(1U<<($2&7U));$n") of mCard: unaryExprChar(p, e, d, "#cardSet($1, " & $size & ')') of mLtSet, mLeSet: getTemp(p, getSysType(tyInt), i) # our counter @@ -1838,9 +1838,9 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) = assert n.kind == nkClosure if isConstClosure(n): - inc(p.labels) - var tmp = "LOC" & rope(p.labels) - addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n", + inc(p.module.labels) + var tmp = "CNSTCLOSURE" & rope(p.module.labels) + addf(p.module.s[cfsData], "static NIM_CONST $1 $2 = $3;$n", [getTypeDesc(p.module, n.typ), tmp, genConstExpr(p, n)]) putIntoDest(p, d, n.typ, tmp, OnStatic) else: @@ -1965,7 +1965,9 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = else: genProc(p.module, sym) putLocIntoDest(p, d, sym.loc) - of skProc, skConverter, skIterators: + of skProc, skConverter, skIterator: + #if sym.kind == skIterator: + # echo renderTree(sym.getBody, {renderIds}) if sfCompileTime in sym.flags: localError(n.info, "request to generate code for .compileTime proc: " & sym.name.s) @@ -1987,6 +1989,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = if sfGlobal in sym.flags: genVarPrototype(p.module, sym) if sym.loc.r == nil or sym.loc.t == nil: #echo "FAILED FOR PRCO ", p.prc.name.s + #echo renderTree(p.prc.ast, {renderIds}) internalError n.info, "expr: var not init " & sym.name.s & "_" & $sym.id if sfThread in sym.flags: accessThreadLocalVar(p, sym) @@ -2004,9 +2007,9 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = putLocIntoDest(p, d, sym.loc) of skParam: if sym.loc.r == nil or sym.loc.t == nil: - #echo "FAILED FOR PRCO ", p.prc.name.s - #debug p.prc.typ.n - #echo renderTree(p.prc.ast, {renderIds}) + # echo "FAILED FOR PRCO ", p.prc.name.s + # debug p.prc.typ.n + # echo renderTree(p.prc.ast, {renderIds}) internalError(n.info, "expr: param not init " & sym.name.s & "_" & $sym.id) putLocIntoDest(p, d, sym.loc) else: internalError(n.info, "expr(" & $sym.kind & "); unknown symbol") diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 73497bded..529e59273 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -21,8 +21,8 @@ proc registerGcRoot(p: BProc, v: PSym) = # we register a specialized marked proc here; this has the advantage # that it works out of the box for thread local storage then :-) let prc = genTraverseProcForGlobal(p.module, v) - linefmt(p.module.initProc, cpsStmts, - "#nimRegisterGlobalMarker($1);$n", prc) + appcg(p.module, p.module.initProc.procSec(cpsStmts), + "#nimRegisterGlobalMarker($1);$n", [prc]) proc isAssignedImmediately(n: PNode): bool {.inline.} = if n.kind == nkEmpty: return false @@ -955,7 +955,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope = res.add(t.sons[i].strVal) of nkSym: var sym = t.sons[i].sym - if sym.kind in {skProc, skIterator, skClosureIterator, skMethod}: + if sym.kind in {skProc, skIterator, skMethod}: var a: TLoc initLocExpr(p, t.sons[i], a) res.add($rdLoc(a)) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index f63134b66..db376821c 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -594,7 +594,7 @@ proc cgsym(m: BModule, name: string): Rope = var sym = magicsys.getCompilerProc(name) if sym != nil: case sym.kind - of skProc, skMethod, skConverter, skIterators: genProc(m, sym) + of skProc, skMethod, skConverter, skIterator: genProc(m, sym) of skVar, skResult, skLet: genVarPrototype(m, sym) of skType: discard getTypeDesc(m, sym.typ) else: internalError("cgsym: " & name & ": " & $sym.kind) diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index ade202dc1..312afec1a 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -19,10 +19,7 @@ proc genConv(n: PNode, d: PType, downcast: bool): PNode = if (source.kind == tyObject) and (dest.kind == tyObject): var diff = inheritanceDiff(dest, source) if diff == high(int): - # see bug #3550 which triggers it. XXX This is a hack but I don't know yet - # how the real fix looks like: - localError(n.info, "there is no subtype relation between " & - typeToString(d) & " and " & typeToString(n.typ)) + # no subtype relation, nothing to do result = n elif diff < 0: result = newNodeIT(nkObjUpConv, n.info, d) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 8536cc619..8555ec4f0 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -149,10 +149,10 @@ proc ropeFormatNamedVars(frmt: FormatStr, varnames: openArray[string], proc genComment(d: PDoc, n: PNode): string = result = "" var dummyHasToc: bool - if n.comment != nil and startsWith(n.comment, "##"): + if n.comment != nil: renderRstToOut(d[], parseRst(n.comment, toFilename(n.info), toLinenumber(n.info), toColumn(n.info), - dummyHasToc, d.options + {roSkipPounds}), result) + dummyHasToc, d.options), result) proc genRecComment(d: PDoc, n: PNode): Rope = if n == nil: return nil @@ -537,7 +537,7 @@ proc generateJson(d: PDoc, n: PNode, jArray: JsonNode = nil): JsonNode = proc genSection(d: PDoc, kind: TSymKind) = const sectionNames: array[skModule..skTemplate, string] = [ "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Methods", - "Iterators", "Iterators", "Converters", "Macros", "Templates" + "Iterators", "Converters", "Macros", "Templates" ] if d.section[kind] == nil: return var title = sectionNames[kind].rope diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim index 863aa696e..a5a132005 100644 --- a/compiler/evaltempl.nim +++ b/compiler/evaltempl.nim @@ -38,7 +38,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) = if s.owner.id == c.owner.id: if s.kind == skParam and sfGenSym notin s.flags: handleParam actual.sons[s.position] - elif s.kind == skGenericParam: + elif s.kind == skGenericParam or + s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam: handleParam actual.sons[s.owner.typ.len + s.position - 1] else: internalAssert sfGenSym in s.flags diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index f8bf35ed6..14cdf0174 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -931,7 +931,7 @@ proc isIndirect(v: PSym): bool = result = {sfAddrTaken, sfGlobal} * v.flags != {} and #(mapType(v.typ) != etyObject) and {sfImportc, sfVolatile, sfExportc} * v.flags == {} and - v.kind notin {skProc, skConverter, skMethod, skIterator, skClosureIterator, + v.kind notin {skProc, skConverter, skMethod, skIterator, skConst, skTemp, skLet} proc genAddr(p: PProc, n: PNode, r: var TCompRes) = @@ -1636,7 +1636,10 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = of nkSym: genSym(p, n, r) of nkCharLit..nkInt64Lit: - r.res = rope(n.intVal) + if n.typ.kind == tyBool: + r.res = if n.intVal == 0: rope"false" else: rope"true" + else: + r.res = rope(n.intVal) r.kind = resExpr of nkNilLit: if isEmptyType(n.typ): diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index be1631af0..9265d09e3 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -11,7 +11,7 @@ import intsets, strutils, lists, options, ast, astalgo, trees, treetab, msgs, os, - idents, renderer, types, magicsys, rodread, lowerings + idents, renderer, types, magicsys, rodread, lowerings, tables discard """ The basic approach is that captured vars need to be put on the heap and @@ -113,44 +113,19 @@ discard """ # local storage requirements for efficiency. This means closure iterators # have slightly different semantics from ordinary closures. +# ---------------- essential helpers ------------------------------------- const upName* = ":up" # field name for the 'up' reference paramName* = ":envP" envName* = ":env" -type - POuterContext = ref TOuterContext - - TIter = object - fn, closureParam, state, resultSym: PSym # most are only valid if - # fn.kind == skClosureIterator - obj: PType - isIterator: bool - - PEnv = ref TEnv - TEnv {.final.} = object of RootObj - attachedNode, replacementNode: PNode - createdVar: PNode # if != nil it is a used environment; for closure - # iterators this can be 'envParam.env' - createdVarComesFromIter: bool - capturedVars: seq[PSym] # captured variables in this environment - up, next: PEnv # outer scope and next to keep all in a list - upField: PSym # if != nil the dependency to the outer scope is used - obj: PType - fn: PSym # function that belongs to this scope; - # if up.fn != fn then we cross function boundaries. - # This is an important case to consider. - vars: IntSet # variables belonging to this environment - - TOuterContext = object - fn: PSym # may also be a module! - head: PEnv - capturedVars, processed: IntSet - localsToAccess: TIdNodeTable - lambdasToEnv: TIdTable # PSym->PEnv mapping - -proc getStateType(iter: PSym): PType = +proc newCall(a: PSym, b: PNode): PNode = + result = newNodeI(nkCall, a.info) + result.add newSymNode(a) + result.add b + +proc createStateType(iter: PSym): PType = var n = newNodeI(nkRange, iter.info) addSon(n, newIntNode(nkIntLit, -1)) addSon(n, newIntNode(nkIntLit, 0)) @@ -162,7 +137,7 @@ proc getStateType(iter: PSym): PType = proc createStateField(iter: PSym): PSym = result = newSym(skField, getIdent(":state"), iter, iter.info) - result.typ = getStateType(iter) + result.typ = createStateType(iter) proc createEnvObj(owner: PSym): PType = # YYY meh, just add the state field for every closure for now, it's too @@ -170,7 +145,7 @@ proc createEnvObj(owner: PSym): PType = result = createObj(owner, owner.info) rawAddField(result, createStateField(owner)) -proc newIterResult(iter: PSym): PSym = +proc getIterResult(iter: PSym): PSym = if resultPos < iter.ast.len: result = iter.ast.sons[resultPos].sym else: @@ -187,16 +162,20 @@ proc addHiddenParam(routine: PSym, param: PSym) = # some nkEffect node: param.position = routine.typ.n.len-1 addSon(params, newSymNode(param)) - incl(routine.typ.flags, tfCapturesEnv) + #incl(routine.typ.flags, tfCapturesEnv) assert sfFromGeneric in param.flags - #echo "produced environment: ", param.id, " for ", routine.name.s + #echo "produced environment: ", param.id, " for ", routine.id proc getHiddenParam(routine: PSym): PSym = let params = routine.ast.sons[paramsPos] let hidden = lastSon(params) - internalAssert hidden.kind == nkSym and hidden.sym.kind == skParam - result = hidden.sym - assert sfFromGeneric in result.flags + if hidden.kind == nkSym and hidden.sym.kind == skParam and hidden.sym.name.s == paramName: + result = hidden.sym + assert sfFromGeneric in result.flags + else: + # writeStackTrace() + localError(routine.info, "internal error: could not find env param for " & routine.name.s) + result = routine proc getEnvParam*(routine: PSym): PSym = let params = routine.ast.sons[paramsPos] @@ -205,501 +184,423 @@ proc getEnvParam*(routine: PSym): PSym = result = hidden.sym assert sfFromGeneric in result.flags -proc initIter(iter: PSym; ptrType: PType = nil): TIter = - result.fn = iter - result.isIterator = ptrType != nil or iter.kind == skClosureIterator - #echo "fuck you ", ptrType != nil - if result.isIterator: - var cp = getEnvParam(iter) - if cp == nil: - result.obj = if ptrType != nil: ptrType.lastSon else: createEnvObj(iter) - - cp = newSym(skParam, getIdent(paramName), iter, iter.info) - incl(cp.flags, sfFromGeneric) - if ptrType != nil: - cp.typ = ptrType - else: - cp.typ = newType(tyRef, iter) - rawAddSon(cp.typ, result.obj) - addHiddenParam(iter, cp) - else: - result.obj = cp.typ.sons[0] - assert result.obj.kind == tyObject - internalAssert result.obj.n.len > 0 - result.state = result.obj.n[0].sym - result.closureParam = cp - if iter.typ.sons[0] != nil: - result.resultSym = newIterResult(iter) - #iter.ast.add(newSymNode(c.resultSym)) - -proc newOuterContext(fn: PSym): POuterContext = - new(result) - result.fn = fn - result.capturedVars = initIntSet() - result.processed = initIntSet() - initIdNodeTable(result.localsToAccess) - initIdTable(result.lambdasToEnv) - -proc newEnv(o: POuterContext; up: PEnv, n: PNode; owner: PSym): PEnv = - new(result) - result.capturedVars = @[] - result.up = up - result.attachedNode = n - result.fn = owner - result.vars = initIntSet() - result.next = o.head - o.head = result - if owner.kind != skModule and (up == nil or up.fn != owner): - let param = getEnvParam(owner) - if param != nil: - result.obj = param.typ.sons[0] - assert result.obj.kind == tyObject - if result.obj.isNil: - result.obj = createEnvObj(owner) - -proc addCapturedVar(e: PEnv, v: PSym) = - for x in e.capturedVars: - if x == v: return - e.capturedVars.add(v) - addField(e.obj, v) - -proc newCall(a: PSym, b: PNode): PNode = - result = newNodeI(nkCall, a.info) - result.add newSymNode(a) - result.add b - -proc isInnerProc(s, outerProc: PSym): bool = - if s.kind in {skProc, skMethod, skConverter, skClosureIterator}: - var owner = s.skipGenericOwner - while true: - if owner.isNil: return false - if owner == outerProc: return true - owner = owner.owner - #s.typ.callConv == ccClosure - -proc addClosureParam(fn: PSym; e: PEnv) = - var cp = getEnvParam(fn) - if cp == nil: - cp = newSym(skParam, getIdent(paramName), fn, fn.info) - incl(cp.flags, sfFromGeneric) - cp.typ = newType(tyRef, fn) - rawAddSon(cp.typ, e.obj) - addHiddenParam(fn, cp) - #else: - #cp.typ.sons[0] = e.obj - #assert e.obj.kind == tyObject +proc interestingVar(s: PSym): bool {.inline.} = + result = s.kind in {skVar, skLet, skTemp, skForVar, skParam, skResult} and + sfGlobal notin s.flags proc illegalCapture(s: PSym): bool {.inline.} = result = skipTypes(s.typ, abstractInst).kind in {tyVar, tyOpenArray, tyVarargs} or s.kind == skResult -proc interestingVar(s: PSym): bool {.inline.} = - result = s.kind in {skVar, skLet, skTemp, skForVar, skParam, skResult} and - sfGlobal notin s.flags +proc isInnerProc(s: PSym): bool = + if s.kind in {skProc, skMethod, skConverter, skIterator} and s.magic == mNone: + result = s.skipGenericOwner.kind in routineKinds -proc nestedAccess(top: PEnv; local: PSym): PNode = - # Parts after the transformation are in []: - # - # proc main = - # var [:env.]foo = 23 - # proc outer(:paramO) = - # [var :envO; createClosure(:envO); :envO.up = paramO] - # proc inner(:paramI) = - # echo [:paramI.up.]foo - # inner([:envO]) - # outer([:env]) - if not interestingVar(local) or top.fn == local.owner: - return nil - # check it's in fact a captured variable: - var it = top - while it != nil: - if it.vars.contains(local.id): break - it = it.up - if it == nil: return nil - let envParam = top.fn.getEnvParam - internalAssert(not envParam.isNil) - var access = newSymNode(envParam) - it = top.up - while it != nil: - if it.vars.contains(local.id): - access = indirectAccess(access, local, local.info) - return access - internalAssert it.upField != nil - access = indirectAccess(access, it.upField, local.info) - it = it.up - when false: - # Type based expression construction works too, but turned out to hide - # other bugs: - while true: - let obj = access.typ.sons[0] - let field = getFieldFromObj(obj, local) - if field != nil: - return rawIndirectAccess(access, field, local.info) - let upField = lookupInRecord(obj.n, getIdent(upName)) - if upField == nil: break - access = rawIndirectAccess(access, upField, local.info) - return nil - -proc createUpField(obj, fieldType: PType): PSym = - let pos = obj.n.len - result = newSym(skField, getIdent(upName), obj.owner, obj.owner.info) - result.typ = newType(tyRef, obj.owner) - result.position = pos - rawAddSon(result.typ, fieldType) - #rawAddField(obj, result) - addField(obj, result) - -proc captureVar(o: POuterContext; top: PEnv; local: PSym; - info: TLineInfo): bool = - # first check if we should be concerned at all: - var it = top - while it != nil: - if it.vars.contains(local.id): break - it = it.up - if it == nil: return false - # yes, so mark every 'up' pointer as taken: - if illegalCapture(local) or top.fn.typ.callConv notin {ccClosure, ccDefault}: - localError(info, errIllegalCaptureX, local.name.s) - it = top - while it != nil: - if it.vars.contains(local.id): break - # keep in mind that the first element of the chain belong to top.fn itself - # and these don't need any upFields - if it.upField == nil and it.up != nil and it.fn != top.fn: - it.upField = createUpField(it.obj, it.up.obj) - - if it.fn != local.owner: - it.fn.typ.callConv = ccClosure - incl(it.fn.typ.flags, tfCapturesEnv) - - var u = it.up - while u != nil and u.fn == it.fn: u = u.up - addClosureParam(it.fn, u) - - if idTableGet(o.lambdasToEnv, it.fn) == nil: - if u != nil: idTablePut(o.lambdasToEnv, it.fn, u) - - it = it.up - # don't do this: 'top' might not require a closure: - #if idTableGet(o.lambdasToEnv, it.fn) == nil: - # idTablePut(o.lambdasToEnv, it.fn, top) - - # mark as captured: - #if top.iter != nil: - # if not containsOrIncl(o.capturedVars, local.id): - # #addField(top.iter.obj, local) - # addCapturedVar(it, local) - #else: - incl(o.capturedVars, local.id) - addCapturedVar(it, local) - result = true - -proc semCaptureSym*(s, owner: PSym) = - if interestingVar(s) and owner.id != s.owner.id and s.kind != skResult: - if owner.typ != nil and not isGenericRoutine(owner): - # XXX: is this really safe? - # if we capture a var from another generic routine, - # it won't be consider captured. - owner.typ.callConv = ccClosure - #echo "semCaptureSym ", owner.name.s, owner.id, " ", s.name.s, s.id - # since the analysis is not entirely correct, we don't set 'tfCapturesEnv' - # here +proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode = + # Bugfix: unfortunately we cannot use 'nkFastAsgn' here as that would + # mean to be able to capture string literals which have no GC header. + # However this can only happen if the capture happens through a parameter, + # which is however the only case when we generate an assignment in the first + # place. + result = newNodeI(nkAsgn, info, 2) + result.sons[0] = le + result.sons[1] = ri -proc gatherVars(o: POuterContext; e: PEnv; n: PNode): int = - # gather used vars for closure generation; returns number of captured vars - if n == nil: return 0 - case n.kind - of nkSym: - var s = n.sym - if interestingVar(s) and e.fn != s.owner: - if captureVar(o, e, s, n.info): result = 1 - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkClosure, nkProcDef, - nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, nkTypeSection: - discard - else: - for k in countup(0, sonsLen(n) - 1): - result += gatherVars(o, e, n.sons[k]) - -proc generateThunk(prc: PNode, dest: PType): PNode = - ## Converts 'prc' into '(thunk, nil)' so that it's compatible with - ## a closure. - - # we cannot generate a proper thunk here for GC-safety reasons (see internal - # documentation): - if gCmd == cmdCompileToJS: return prc - result = newNodeIT(nkClosure, prc.info, dest) - var conv = newNodeIT(nkHiddenStdConv, prc.info, dest) - conv.add(emptyNode) - conv.add(prc) - result.add(conv) - result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil))) - -proc transformOuterConv(n: PNode): PNode = - # numeric types need range checks: - var dest = skipTypes(n.typ, abstractVarRange) - var source = skipTypes(n.sons[1].typ, abstractVarRange) - if dest.kind == tyProc: - if dest.callConv == ccClosure and source.callConv == ccDefault: - result = generateThunk(n.sons[1], dest) - -proc makeClosure(prc: PSym; env: PNode; info: TLineInfo): PNode = +proc makeClosure*(prc: PSym; env: PNode; info: TLineInfo): PNode = result = newNodeIT(nkClosure, info, prc.typ) result.add(newSymNode(prc)) if env == nil: result.add(newNodeIT(nkNilLit, info, getSysType(tyNil))) else: + if env.kind == nkClosure: + localError(info, "internal error: taking closure of closure") result.add(env) -proc newClosureCreationVar(e: PEnv): PNode = - var v = newSym(skVar, getIdent(envName), e.fn, e.attachedNode.info) - incl(v.flags, sfShadowed) - v.typ = newType(tyRef, e.fn) - v.typ.rawAddSon(e.obj) - if e.fn.kind == skClosureIterator: - let it = initIter(e.fn) - addUniqueField(it.obj, v) - result = indirectAccess(newSymNode(it.closureParam), v, v.info) +proc interestingIterVar(s: PSym): bool {.inline.} = + # XXX optimization: Only lift the variable if it lives across + # yield/return boundaries! This can potentially speed up + # closure iterators quite a bit. + result = s.kind in {skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags + +template isIterator*(owner: PSym): bool = + owner.kind == skIterator and owner.typ.callConv == ccClosure + +proc liftIterSym*(n: PNode; owner: PSym): PNode = + # transforms (iter) to (let env = newClosure[iter](); (iter, env)) + let iter = n.sym + assert iter.isIterator + + result = newNodeIT(nkStmtListExpr, n.info, n.typ) + + let hp = getHiddenParam(iter) + let env = newSym(skLet, iter.name, owner, n.info) + env.typ = hp.typ + env.flags = hp.flags + var v = newNodeI(nkVarSection, n.info) + addVar(v, newSymNode(env)) + result.add(v) + # add 'new' statement: + let envAsNode = env.newSymNode + result.add newCall(getSysSym"internalNew", envAsNode) + result.add makeClosure(iter, envAsNode, n.info) + +proc freshVarForClosureIter*(s, owner: PSym): PNode = + let envParam = getHiddenParam(owner) + let obj = envParam.typ.lastSon + addField(obj, s) + + var access = newSymNode(envParam) + assert obj.kind == tyObject + let field = getFieldFromObj(obj, s) + if field != nil: + result = rawIndirectAccess(access, field, s.info) else: - result = newSymNode(v) + localError(s.info, "internal error: cannot generate fresh variable") + result = access + +# ------------------ new stuff ------------------------------------------- + +proc markAsClosure(owner: PSym; n: PNode) = + let s = n.sym + if illegalCapture(s) or owner.typ.callConv notin {ccClosure, ccDefault}: + localError(n.info, errIllegalCaptureX, s.name.s) + incl(owner.typ.flags, tfCapturesEnv) + owner.typ.callConv = ccClosure -proc getClosureVar(e: PEnv): PNode = - if e.createdVar == nil: - result = newClosureCreationVar(e) - e.createdVar = result +type + DetectionPass = object + processed, capturedVars: IntSet + ownerToType: Table[int, PType] + somethingToDo: bool + +proc initDetectionPass(fn: PSym): DetectionPass = + result.processed = initIntSet() + result.capturedVars = initIntSet() + result.ownerToType = initTable[int, PType]() + result.processed.incl(fn.id) + +discard """ +proc outer = + var a, b: int + proc innerA = use(a) + proc innerB = use(b); innerA() +# --> innerA and innerB need to *share* the closure type! +This is why need to store the 'ownerToType' table and use it +during .closure'fication. +""" + +proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym): PType = + result = c.ownerToType.getOrDefault(owner.id) + if result.isNil: + result = newType(tyRef, owner) + let obj = createEnvObj(owner) + rawAddSon(result, obj) + c.ownerToType[owner.id] = result + +proc createUpField(c: var DetectionPass; dest, dep: PSym) = + let refObj = c.getEnvTypeForOwner(dest) # getHiddenParam(dest).typ + let obj = refObj.lastSon + let fieldType = c.getEnvTypeForOwner(dep) #getHiddenParam(dep).typ + if refObj == fieldType: + localError(dep.info, "internal error: invalid up reference computed") + + let upIdent = getIdent(upName) + let upField = lookupInRecord(obj.n, upIdent) + if upField != nil: + if upField.typ != fieldType: + localError(dep.info, "internal error: up references do not agree") else: - result = e.createdVar - -proc findEnv(o: POuterContext; s: PSym): PEnv = - var env = o.head - while env != nil: - if env.fn == s: break - env = env.next - internalAssert env != nil and env.up != nil - result = env.up - while result.fn == s: result = result.up - -proc transformInnerProc(o: POuterContext; e: PEnv, n: PNode): PNode = + let result = newSym(skField, upIdent, obj.owner, obj.owner.info) + result.typ = fieldType + rawAddField(obj, result) + +discard """ +There are a couple of possibilities of how to implement closure +iterators that capture outer variables in a traditional sense +(aka closure closure iterators). + +1. Transform iter() to iter(state, capturedEnv). So use 2 hidden + parameters. +2. Add the captured vars directly to 'state'. +3. Make capturedEnv an up-reference of 'state'. + +We do (3) here because (2) is obviously wrong and (1) is wrong too. +Consider: + + proc outer = + var xx = 9 + + iterator foo() = + var someState = 3 + + proc bar = echo someState + proc baz = someState = 0 + baz() + bar() + +""" + +proc addClosureParam(c: var DetectionPass; fn: PSym) = + var cp = getEnvParam(fn) + let owner = if fn.kind == skIterator: fn else: fn.skipGenericOwner + let t = c.getEnvTypeForOwner(owner) + if cp == nil: + cp = newSym(skParam, getIdent(paramName), fn, fn.info) + incl(cp.flags, sfFromGeneric) + cp.typ = t + addHiddenParam(fn, cp) + elif cp.typ != t and fn.kind != skIterator: + localError(fn.info, "internal error: inconsistent environment type") + #echo "adding closure to ", fn.name.s + +proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = case n.kind - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard of nkSym: let s = n.sym - if s == e.fn: - # recursive calls go through (lambda, hiddenParam): - result = makeClosure(s, getEnvParam(s).newSymNode, n.info) - elif isInnerProc(s, o.fn) and s.typ.callConv == ccClosure: - # ugh: call to some other inner proc; - result = makeClosure(s, findEnv(o, s).getClosureVar, n.info) - else: - # captured symbol? - result = nestedAccess(e, n.sym) - #result = idNodeTableGet(i.localsToAccess, n.sym) - #of nkLambdaKinds, nkIteratorDef: - # if n.typ != nil: - # result = transformInnerProc(o, e, n.sons[namePos]) - #of nkClosure: - # let x = transformInnerProc(o, e, n.sons[0]) - # if x != nil: n.sons[0] = x - of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, - nkLambdaKinds, nkIteratorDef, nkClosure: - # don't recurse here: + if s.kind in {skProc, skMethod, skConverter, skIterator} and s.typ != nil and s.typ.callConv == ccClosure: + # this handles the case that the inner proc was declared as + # .closure but does not actually capture anything: + addClosureParam(c, s) + c.somethingToDo = true + + let innerProc = isInnerProc(s) + if innerProc: + if s.isIterator: c.somethingToDo = true + if not c.processed.containsOrIncl(s.id): + detectCapturedVars(s.getBody, s, c) + let ow = s.skipGenericOwner + if ow == owner: + if owner.isIterator: + c.somethingToDo = true + addClosureParam(c, owner) + if interestingIterVar(s): + if not c.capturedVars.containsOrIncl(s.id): + let obj = getHiddenParam(owner).typ.lastSon + #let obj = c.getEnvTypeForOwner(s.owner).lastSon + addField(obj, s) + # but always return because the rest of the proc is only relevant when + # ow != owner: + return + # direct or indirect dependency: + if (innerProc and s.typ.callConv == ccClosure) or interestingVar(s): + discard """ + proc outer() = + var x: int + proc inner() = + proc innerInner() = + echo x + innerInner() + inner() + # inner() takes a closure too! + """ + # mark 'owner' as taking a closure: + c.somethingToDo = true + markAsClosure(owner, n) + addClosureParam(c, owner) + #echo "capturing ", n.info + # variable 's' is actually captured: + if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id): + let obj = c.getEnvTypeForOwner(ow).lastSon + #getHiddenParam(owner).typ.lastSon + addField(obj, s) + # create required upFields: + var w = owner.skipGenericOwner + if isInnerProc(w) or owner.isIterator: + if owner.isIterator: w = owner + let last = if ow.isIterator: ow.skipGenericOwner else: ow + while w != nil and w.kind != skModule and last != w: + discard """ + proc outer = + var a, b: int + proc outerB = + proc innerA = use(a) + proc innerB = use(b); innerA() + # --> make outerB of calling convention .closure and + # give it the same env type that outer's env var gets: + """ + let up = w.skipGenericOwner + #echo "up for ", w.name.s, " up ", up.name.s + markAsClosure(w, n) + addClosureParam(c, w) # , ow + createUpField(c, w, up) + w = up + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, + nkTemplateDef, nkTypeSection: discard - else: - for j in countup(0, sonsLen(n) - 1): - let x = transformInnerProc(o, e, n.sons[j]) - if x != nil: n.sons[j] = x - -proc closureCreationPoint(n: PNode): PNode = - if n.kind == nkStmtList and n.len >= 1 and n[0].kind == nkEmpty: - # we already have a free slot - result = n - else: - result = newNodeI(nkStmtList, n.info) - result.add(emptyNode) - result.add(n) - #result.flags.incl nfLL - -proc addParamsToEnv(fn: PSym; env: PEnv) = - let params = fn.typ.n - for i in 1.. <params.len: - if params.sons[i].kind != nkSym: - internalError(params.info, "liftLambdas: strange params") - let param = params.sons[i].sym - env.vars.incl(param.id) - # put the 'result' into the environment so it can be captured: - let ast = fn.ast - if resultPos < sonsLen(ast) and ast.sons[resultPos].kind == nkSym: - env.vars.incl(ast.sons[resultPos].sym.id) - -proc searchForInnerProcs(o: POuterContext, n: PNode, env: PEnv) = - if n == nil: return - case n.kind - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: + of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef: discard - of nkSym: - let fn = n.sym - if isInnerProc(fn, o.fn) and not containsOrIncl(o.processed, fn.id): - let body = fn.getBody - if nfLL in body.flags: return - - # handle deeply nested captures: - let ex = closureCreationPoint(body) - let envB = newEnv(o, env, ex, fn) - addParamsToEnv(fn, envB) - searchForInnerProcs(o, body, envB) - fn.ast.sons[bodyPos] = ex - - let capturedCounter = gatherVars(o, envB, body) - # dummy closure param needed? - if capturedCounter == 0 and fn.typ.callConv == ccClosure: - #assert tfCapturesEnv notin n.sym.typ.flags - if idTableGet(o.lambdasToEnv, fn) == nil: - idTablePut(o.lambdasToEnv, fn, env) - addClosureParam(fn, env) - - elif fn.getEnvParam != nil: - # only transform if it really needs a closure: - let ti = transformInnerProc(o, envB, body) - if ti != nil: fn.ast.sons[bodyPos] = ti of nkLambdaKinds, nkIteratorDef: if n.typ != nil: - searchForInnerProcs(o, n.sons[namePos], env) - of nkWhileStmt, nkForStmt, nkParForStmt, nkBlockStmt: - # some nodes open a new scope, so they are candidates for the insertion - # of closure creation; however for simplicity we merge closures between - # branches, in fact, only loop bodies are of interest here as only they - # yield observable changes in semantics. For Zahary we also - # include ``nkBlock``. We don't do this for closure iterators because - # 'yield' can produce wrong code otherwise (XXX show example): - if env.fn.kind != skClosureIterator: - var body = n.len-1 - for i in countup(0, body - 1): searchForInnerProcs(o, n.sons[i], env) - # special handling for the loop body: - let ex = closureCreationPoint(n.sons[body]) - searchForInnerProcs(o, n.sons[body], newEnv(o, env, ex, env.fn)) - n.sons[body] = ex - else: - for i in countup(0, sonsLen(n) - 1): - searchForInnerProcs(o, n.sons[i], env) - of nkVarSection, nkLetSection: - # we need to compute a mapping var->declaredBlock. Note: The definition - # counts, not the block where it is captured! - for i in countup(0, sonsLen(n) - 1): - var it = n.sons[i] - if it.kind == nkCommentStmt: discard - elif it.kind == nkIdentDefs: - var L = sonsLen(it) - if it.sons[0].kind == nkSym: - # this can be false for recursive invocations that already - # transformed it into 'env.varName': - env.vars.incl(it.sons[0].sym.id) - searchForInnerProcs(o, it.sons[L-1], env) - elif it.kind == nkVarTuple: - var L = sonsLen(it) - for j in countup(0, L-3): - #echo "set: ", it.sons[j].sym.name.s, " ", o.currentBlock == nil - if it.sons[j].kind == nkSym: - env.vars.incl(it.sons[j].sym.id) - searchForInnerProcs(o, it.sons[L-1], env) - else: - internalError(it.info, "searchForInnerProcs") - of nkClosure: - searchForInnerProcs(o, n.sons[0], env) - of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, - nkTypeSection: - # don't recurse here: - discard + detectCapturedVars(n[namePos], owner, c) else: - for i in countup(0, sonsLen(n) - 1): - searchForInnerProcs(o, n.sons[i], env) + for i in 0..<n.len: + detectCapturedVars(n[i], owner, c) -proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode = - # Bugfix: unfortunately we cannot use 'nkFastAsgn' here as that would - # mean to be able to capture string literals which have no GC header. - # However this can only happen if the capture happens through a parameter, - # which is however the only case when we generate an assignment in the first - # place. - result = newNodeI(nkAsgn, info, 2) - result.sons[0] = le - result.sons[1] = ri +type + LiftingPass = object + processed: IntSet + envVars: Table[int, PNode] -proc rawClosureCreation(o: POuterContext, scope: PEnv; env: PNode): PNode = - result = newNodeI(nkStmtList, env.info) - if env.kind == nkSym: - var v = newNodeI(nkVarSection, env.info) - addVar(v, env) - result.add(v) - # add 'new' statement: - result.add(newCall(getSysSym"internalNew", env)) - - # add assignment statements: - for local in scope.capturedVars: - let fieldAccess = indirectAccess(env, local, env.info) - if local.kind == skParam: - # maybe later: (sfByCopy in local.flags) - # add ``env.param = param`` - result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info)) - # it can happen that we already captured 'local' in some other environment - # then we capture by copy for now. This is not entirely correct but better - # than nothing: - let existing = idNodeTableGet(o.localsToAccess, local) - if existing.isNil: - idNodeTablePut(o.localsToAccess, local, fieldAccess) +proc initLiftingPass(fn: PSym): LiftingPass = + result.processed = initIntSet() + result.processed.incl(fn.id) + result.envVars = initTable[int, PNode]() + +proc accessViaEnvParam(n: PNode; owner: PSym): PNode = + let s = n.sym + # Type based expression construction for simplicity: + let envParam = getHiddenParam(owner) + if not envParam.isNil: + var access = newSymNode(envParam) + while true: + let obj = access.typ.sons[0] + assert obj.kind == tyObject + let field = getFieldFromObj(obj, s) + if field != nil: + return rawIndirectAccess(access, field, n.info) + let upField = lookupInRecord(obj.n, getIdent(upName)) + if upField == nil: break + access = rawIndirectAccess(access, upField, n.info) + localError(n.info, "internal error: environment misses: " & s.name.s) + result = n + +proc newEnvVar(owner: PSym; typ: PType): PNode = + var v = newSym(skVar, getIdent(envName), owner, owner.info) + incl(v.flags, sfShadowed) + v.typ = typ + result = newSymNode(v) + when false: + if owner.kind == skIterator and owner.typ.callConv == ccClosure: + let it = getHiddenParam(owner) + addUniqueField(it.typ.sons[0], v) + result = indirectAccess(newSymNode(it), v, v.info) + else: + result = newSymNode(v) + +proc setupEnvVar(owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + if owner.isIterator: + return getHiddenParam(owner).newSymNode + result = c.envvars.getOrDefault(owner.id) + if result.isNil: + let envVarType = d.ownerToType.getOrDefault(owner.id) + if envVarType.isNil: + localError owner.info, "internal error: could not determine closure type" + result = newEnvVar(owner, envVarType) + c.envVars[owner.id] = result + +proc getUpViaParam(owner: PSym): PNode = + let p = getHiddenParam(owner) + result = p.newSymNode + if owner.isIterator: + let upField = lookupInRecord(p.typ.lastSon.n, getIdent(upName)) + if upField == nil: + localError(owner.info, "could not find up reference for closure iter") else: - result.add(newAsgnStmt(fieldAccess, existing, env.info)) - if scope.upField != nil: - # "up" chain has been used: - if scope.up.fn != scope.fn: - # crosses function boundary: - result.add(newAsgnStmt(indirectAccess(env, scope.upField, env.info), - newSymNode(getEnvParam(scope.fn)), env.info)) + result = rawIndirectAccess(result, upField, p.info) + +proc rawClosureCreation(owner: PSym; + d: DetectionPass; c: var LiftingPass): PNode = + result = newNodeI(nkStmtList, owner.info) + + var env: PNode + if owner.isIterator: + env = getHiddenParam(owner).newSymNode + else: + env = setupEnvVar(owner, d, c) + if env.kind == nkSym: + var v = newNodeI(nkVarSection, env.info) + addVar(v, env) + result.add(v) + # add 'new' statement: + result.add(newCall(getSysSym"internalNew", env)) + # add assignment statements for captured parameters: + for i in 1..<owner.typ.n.len: + let local = owner.typ.n[i].sym + if local.id in d.capturedVars: + let fieldAccess = indirectAccess(env, local, env.info) + # add ``env.param = param`` + result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info)) + + let upField = lookupInRecord(env.typ.lastSon.n, getIdent(upName)) + if upField != nil: + let up = getUpViaParam(owner) + if up != nil and upField.typ == up.typ: + result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info), + up, env.info)) + #elif oldenv != nil and oldenv.typ == upField.typ: + # result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info), + # oldenv, env.info)) else: - result.add(newAsgnStmt(indirectAccess(env, scope.upField, env.info), - getClosureVar(scope.up), env.info)) - -proc generateClosureCreation(o: POuterContext, scope: PEnv): PNode = - var env = getClosureVar(scope) - result = rawClosureCreation(o, scope, env) - -proc generateIterClosureCreation(o: POuterContext; env: PEnv; - scope: PNode): PNode = - if env.createdVarComesFromIter or env.createdVar.isNil: - # we have to create a new closure: - result = newClosureCreationVar(env) - let cc = rawClosureCreation(o, env, result) - var insertPoint = scope.sons[0] - if insertPoint.kind == nkEmpty: scope.sons[0] = cc + localError(env.info, "internal error: cannot create up reference") + +proc closureCreationForIter(iter: PNode; + d: DetectionPass; c: var LiftingPass): PNode = + result = newNodeIT(nkStmtListExpr, iter.info, iter.sym.typ) + let owner = iter.sym.skipGenericOwner + var v = newSym(skVar, getIdent(envName), owner, iter.info) + incl(v.flags, sfShadowed) + v.typ = getHiddenParam(iter.sym).typ + var vnode: PNode + if owner.isIterator: + let it = getHiddenParam(owner) + addUniqueField(it.typ.sons[0], v) + vnode = indirectAccess(newSymNode(it), v, v.info) + else: + vnode = v.newSymNode + var vs = newNodeI(nkVarSection, iter.info) + addVar(vs, vnode) + result.add(vs) + result.add(newCall(getSysSym"internalNew", vnode)) + + let upField = lookupInRecord(v.typ.lastSon.n, getIdent(upName)) + if upField != nil: + let u = setupEnvVar(owner, d, c) + if u.typ == upField.typ: + result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info), + u, iter.info)) else: - assert cc.kind == nkStmtList and insertPoint.kind == nkStmtList - for x in cc: insertPoint.add(x) - if env.createdVar == nil: env.createdVar = result + localError(iter.info, "internal error: cannot create up reference for iter") + result.add makeClosure(iter.sym, vnode, iter.info) + +proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + let access = setupEnvVar(owner, d, c) + let obj = access.typ.sons[0] + let field = getFieldFromObj(obj, n.sym) + if field != nil: + result = rawIndirectAccess(access, field, n.info) else: - result = env.createdVar - env.createdVarComesFromIter = true + localError(n.info, "internal error: not part of closure object type") + result = n -proc interestingIterVar(s: PSym): bool {.inline.} = - result = s.kind in {skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags +proc getStateField(owner: PSym): PSym = + getHiddenParam(owner).typ.sons[0].n.sons[0].sym -proc transformOuterProc(o: POuterContext, n: PNode, it: TIter): PNode +proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode -proc transformYield(c: POuterContext, n: PNode, it: TIter): PNode = - assert it.state != nil - assert it.state.typ != nil - assert it.state.typ.n != nil - inc it.state.typ.n.sons[1].intVal - let stateNo = it.state.typ.n.sons[1].intVal +proc transformYield(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + let state = getStateField(owner) + assert state != nil + assert state.typ != nil + assert state.typ.n != nil + inc state.typ.n.sons[1].intVal + let stateNo = state.typ.n.sons[1].intVal var stateAsgnStmt = newNodeI(nkAsgn, n.info) - stateAsgnStmt.add(rawIndirectAccess(newSymNode(it.closureParam), - it.state, n.info)) + stateAsgnStmt.add(rawIndirectAccess(newSymNode(getEnvParam(owner)), + state, n.info)) stateAsgnStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt))) var retStmt = newNodeI(nkReturnStmt, n.info) if n.sons[0].kind != nkEmpty: var a = newNodeI(nkAsgn, n.sons[0].info) - var retVal = transformOuterProc(c, n.sons[0], it) - addSon(a, newSymNode(it.resultSym)) - addSon(a, if retVal.isNil: n.sons[0] else: retVal) + var retVal = liftCapturedVars(n.sons[0], owner, d, c) + addSon(a, newSymNode(getIterResult(owner))) + addSon(a, retVal) retStmt.add(a) else: retStmt.add(emptyNode) @@ -712,262 +613,155 @@ proc transformYield(c: POuterContext, n: PNode, it: TIter): PNode = result.add(retStmt) result.add(stateLabelStmt) -proc transformReturn(c: POuterContext, n: PNode, it: TIter): PNode = +proc transformReturn(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + let state = getStateField(owner) result = newNodeI(nkStmtList, n.info) var stateAsgnStmt = newNodeI(nkAsgn, n.info) - stateAsgnStmt.add(rawIndirectAccess(newSymNode(it.closureParam), it.state, - n.info)) + stateAsgnStmt.add(rawIndirectAccess(newSymNode(getEnvParam(owner)), + state, n.info)) stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt))) result.add(stateAsgnStmt) result.add(n) -proc outerProcSons(o: POuterContext, n: PNode, it: TIter) = - for i in countup(0, sonsLen(n) - 1): - let x = transformOuterProc(o, n.sons[i], it) - if x != nil: n.sons[i] = x - -proc liftIterSym(n: PNode; owner: PSym): PNode = - # transforms (iter) to (let env = newClosure[iter](); (iter, env)) - let iter = n.sym - assert iter.kind == skClosureIterator - - result = newNodeIT(nkStmtListExpr, n.info, n.typ) - - let hp = getHiddenParam(iter) - let env = newSym(skLet, iter.name, owner, n.info) - env.typ = hp.typ - env.flags = hp.flags - var v = newNodeI(nkVarSection, n.info) - addVar(v, newSymNode(env)) - result.add(v) - # add 'new' statement: - let envAsNode = env.newSymNode - result.add newCall(getSysSym"internalNew", envAsNode) - result.add makeClosure(iter, envAsNode, n.info) - -when false: - proc transformRemainingLocals(n: PNode; it: TIter): PNode = - assert it.fn.kind == skClosureIterator - result = n - case n.kind - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard - of nkSym: - let local = n.sym - if interestingIterVar(local) and it.fn == local.owner: - addUniqueField(it.obj, local) - result = indirectAccess(newSymNode(it.closureParam), local, n.info) - else: - result = newNodeI(n.kind, n.info, n.len) - for i in 0.. <n.safeLen: - result.sons[i] = transformRemainingLocals(n.sons[i], it) - -template envActive(env): expr = - (env.capturedVars.len > 0 or env.upField != nil) - -# We have to split up environment creation in 2 steps: -# 1. Generate it and store it in env.replacementNode -# 2. Insert replacementNode into its forseen slot. -# This split is necessary so that assignments belonging to closure -# creation like 'env.param = param' are not transformed -# into 'env.param = env.param'. -proc createEnvironments(o: POuterContext) = - var env = o.head - while env != nil: - if envActive(env): - var scope = env.attachedNode - assert scope.kind == nkStmtList - if scope.sons[0].kind == nkEmpty: - # prepare for closure construction: - env.replacementNode = generateClosureCreation(o, env) - env = env.next - -proc finishEnvironments(o: POuterContext) = - var env = o.head - while env != nil: - if env.replacementNode != nil: - var scope = env.attachedNode - assert scope.kind == nkStmtList - if scope.sons[0].kind == nkEmpty: - # change the empty node to contain the closure construction: - scope.sons[0] = env.replacementNode - when false: - if env.fn.kind == skClosureIterator: - scope.sons[0] = transformRemainingLocals(env.replacementNode, - initIter(env.fn)) - else: - scope.sons[0] = env.replacementNode - env = env.next - -proc transformOuterProcBody(o: POuterContext, n: PNode; it: TIter): PNode = - if nfLL in n.flags: - result = nil - elif it.isIterator: +proc wrapIterBody(n: PNode; owner: PSym): PNode = + if not owner.isIterator: return n + when false: # unfortunately control flow is still convoluted and we can end up # multiple times here for the very same iterator. We shield against this # with some rather primitive check for now: if n.kind == nkStmtList and n.len > 0: - if n.sons[0].kind == nkGotoState: return nil + if n.sons[0].kind == nkGotoState: return n if n.len > 1 and n[1].kind == nkStmtList and n[1].len > 0 and n[1][0].kind == nkGotoState: - return nil - result = newNodeI(nkStmtList, it.fn.info) - var gs = newNodeI(nkGotoState, it.fn.info) - assert it.closureParam != nil - assert it.state != nil - gs.add(rawIndirectAccess(newSymNode(it.closureParam), it.state, it.fn.info)) - result.add(gs) - var state0 = newNodeI(nkState, it.fn.info) - state0.add(newIntNode(nkIntLit, 0)) - result.add(state0) - - let newBody = transformOuterProc(o, n, it) - if newBody != nil: - result.add(newBody) - else: - result.add(n) - - var stateAsgnStmt = newNodeI(nkAsgn, it.fn.info) - stateAsgnStmt.add(rawIndirectAccess(newSymNode(it.closureParam), - it.state, it.fn.info)) - stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt))) - result.add(stateAsgnStmt) - result.flags.incl nfLL - else: - result = transformOuterProc(o, n, it) - if result != nil: result.flags.incl nfLL + return n + let info = n.info + result = newNodeI(nkStmtList, info) + var gs = newNodeI(nkGotoState, info) + gs.add(rawIndirectAccess(newSymNode(owner.getHiddenParam), getStateField(owner), info)) + result.add(gs) + var state0 = newNodeI(nkState, info) + state0.add(newIntNode(nkIntLit, 0)) + result.add(state0) + + result.add(n) + + var stateAsgnStmt = newNodeI(nkAsgn, info) + stateAsgnStmt.add(rawIndirectAccess(newSymNode(owner.getHiddenParam), + getStateField(owner), info)) + stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt))) + result.add(stateAsgnStmt) -proc transformOuterProc(o: POuterContext, n: PNode; it: TIter): PNode = - if n == nil or nfLL in n.flags: return nil +proc symToClosure(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + let s = n.sym + if s == owner: + # recursive calls go through (lambda, hiddenParam): + let available = getHiddenParam(owner) + result = makeClosure(s, available.newSymNode, n.info) + elif s.isIterator: + result = closureCreationForIter(n, d, c) + elif s.skipGenericOwner == owner: + # direct dependency, so use the outer's env variable: + result = makeClosure(s, setupEnvVar(owner, d, c), n.info) + else: + let available = getHiddenParam(owner) + let wanted = getHiddenParam(s).typ + # ugh: call through some other inner proc; + var access = newSymNode(available) + while true: + if access.typ == wanted: + return makeClosure(s, access, n.info) + let obj = access.typ.sons[0] + let upField = lookupInRecord(obj.n, getIdent(upName)) + if upField == nil: + localError(n.info, "internal error: no environment found") + return n + access = rawIndirectAccess(access, upField, n.info) + +proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + result = n case n.kind - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard of nkSym: - var local = n.sym - - if isInnerProc(local, o.fn) and o.processed.contains(local.id): - o.processed.excl(local.id) - let body = local.getBody - let newBody = transformOuterProcBody(o, body, initIter(local)) - if newBody != nil: - local.ast.sons[bodyPos] = newBody - - if it.isIterator and interestingIterVar(local) and - it.fn == local.owner: - # every local goes through the closure: - #if not containsOrIncl(o.capturedVars, local.id): - # addField(it.obj, local) - if contains(o.capturedVars, local.id): - # change 'local' to 'closure.local', unless it's a 'byCopy' variable: - # if sfByCopy notin local.flags: - result = idNodeTableGet(o.localsToAccess, local) - assert result != nil, "cannot find: " & local.name.s - return result - else: - addUniqueField(it.obj, local) - return indirectAccess(newSymNode(it.closureParam), local, n.info) - - if local.kind == skClosureIterator: - # bug #3354; allow for - #iterator iter(): int {.closure.}= - # s.add(iter) - # yield 1 - - #if local == o.fn or local == it.fn: - # message(n.info, errRecursiveDependencyX, local.name.s) - - # consider: [i1, i2, i1] Since we merged the iterator's closure - # with the captured owning variables, we need to generate the - # closure generation code again: - # XXX why doesn't this work? - var closure = PEnv(idTableGet(o.lambdasToEnv, local)) - if closure.isNil: - return liftIterSym(n, o.fn) - else: - let createdVar = generateIterClosureCreation(o, closure, - closure.attachedNode) - let lpt = getHiddenParam(local).typ - if lpt != createdVar.typ: - assert lpt.kind == tyRef and createdVar.typ.kind == tyRef - # fix bug 'tshallowcopy_closures' but report if this gets any weirder: - if createdVar.typ.sons[0].len == 1 and lpt.sons[0].len >= 1: - createdVar.typ = lpt - if createdVar.kind == nkSym: createdVar.sym.typ = lpt - closure.obj = lpt.sons[0] - else: - internalError(n.info, "environment computation failed") - return makeClosure(local, createdVar, n.info) - - var closure = PEnv(idTableGet(o.lambdasToEnv, local)) - if closure != nil: - # we need to replace the lambda with '(lambda, env)': - let a = closure.createdVar - if a != nil: - return makeClosure(local, a, n.info) + let s = n.sym + if isInnerProc(s): + if not c.processed.containsOrIncl(s.id): + #if s.name.s == "temp": + # echo renderTree(s.getBody, {renderIds}) + let body = wrapIterBody(liftCapturedVars(s.getBody, s, d, c), s) + if c.envvars.getOrDefault(s.id).isNil: + s.ast.sons[bodyPos] = body + else: + s.ast.sons[bodyPos] = newTree(nkStmtList, rawClosureCreation(s, d, c), body) + if s.typ.callConv == ccClosure: + result = symToClosure(n, owner, d, c) + elif s.id in d.capturedVars: + if s.owner != owner: + result = accessViaEnvParam(n, owner) + elif owner.isIterator and interestingIterVar(s): + result = accessViaEnvParam(n, owner) else: - # can happen for dummy closures: - var scope = closure.attachedNode - assert scope.kind == nkStmtList - if scope.sons[0].kind == nkEmpty: - # change the empty node to contain the closure construction: - scope.sons[0] = generateClosureCreation(o, closure) - let x = closure.createdVar - assert x != nil - return makeClosure(local, x, n.info) - - if not contains(o.capturedVars, local.id): return - # change 'local' to 'closure.local', unless it's a 'byCopy' variable: - # if sfByCopy notin local.flags: - result = idNodeTableGet(o.localsToAccess, local) - assert result != nil, "cannot find: " & local.name.s - # else it is captured by copy and this means that 'outer' should continue - # to access the local as a local. - of nkLambdaKinds, nkIteratorDef: - if n.typ != nil: - result = transformOuterProc(o, n.sons[namePos], it) - of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef: - # don't recurse here: + result = accessViaEnvVar(n, owner, d, c) + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, + nkTemplateDef, nkTypeSection: + discard + of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef: discard of nkClosure: - if n.sons[0].kind == nkSym: - var local = n.sons[0].sym - if isInnerProc(local, o.fn) and o.processed.contains(local.id): - o.processed.excl(local.id) - let body = local.getBody - let newBody = transformOuterProcBody(o, body, initIter(local)) - if newBody != nil: - local.ast.sons[bodyPos] = newBody - when false: - if n.sons[1].kind == nkSym: - var local = n.sons[1].sym - if it.isIterator and interestingIterVar(local) and - it.fn == local.owner: - # every local goes through the closure: - addUniqueField(it.obj, local) - n.sons[1] = indirectAccess(newSymNode(it.closureParam), local, n.info) - of nkHiddenStdConv, nkHiddenSubConv, nkConv: - let x = transformOuterProc(o, n.sons[1], it) - if x != nil: n.sons[1] = x - result = transformOuterConv(n) - of nkYieldStmt: - if it.isIterator: result = transformYield(o, n, it) - else: outerProcSons(o, n, it) - of nkReturnStmt: - if it.isIterator: result = transformReturn(o, n, it) - else: outerProcSons(o, n, it) + if n[1].kind == nkNilLit: + n.sons[0] = liftCapturedVars(n[0], owner, d, c) + #if n.sons[0].kind == nkClosure: result = n.sons[0] + of nkLambdaKinds, nkIteratorDef: + if n.typ != nil and n[namePos].kind == nkSym: + let m = newSymNode(n[namePos].sym) + m.typ = n.typ + result = liftCapturedVars(m, owner, d, c) else: - outerProcSons(o, n, it) + if owner.isIterator: + if n.kind == nkYieldStmt: + return transformYield(n, owner, d, c) + elif n.kind == nkReturnStmt: + return transformReturn(n, owner, d, c) + elif nfLL in n.flags: + # special case 'when nimVm' due to bug #3636: + n.sons[1] = liftCapturedVars(n[1], owner, d, c) + return + for i in 0..<n.len: + n.sons[i] = liftCapturedVars(n[i], owner, d, c) + +# ------------------ old stuff ------------------------------------------- + +proc semCaptureSym*(s, owner: PSym) = + if interestingVar(s) and s.kind != skResult: + if owner.typ != nil and not isGenericRoutine(owner): + # XXX: is this really safe? + # if we capture a var from another generic routine, + # it won't be consider captured. + var o = owner.skipGenericOwner + while o.kind != skModule and o != nil: + if s.owner == o: + owner.typ.callConv = ccClosure + #echo "computing .closure for ", owner.name.s, " ", owner.info, " because of ", s.name.s + o = o.skipGenericOwner + # since the analysis is not entirely correct, we don't set 'tfCapturesEnv' + # here proc liftIterToProc*(fn: PSym; body: PNode; ptrType: PType): PNode = - var o = newOuterContext(fn) - let ex = closureCreationPoint(body) - let env = newEnv(o, nil, ex, fn) - addParamsToEnv(fn, env) - searchForInnerProcs(o, body, env) - createEnvironments(o) - let it = initIter(fn, ptrType) - result = transformOuterProcBody(o, body, it) - finishEnvironments(o) - -proc liftLambdas*(fn: PSym, body: PNode): PNode = + var d = initDetectionPass(fn) + var c = initLiftingPass(fn) + # pretend 'fn' is a closure iterator for the analysis: + let oldKind = fn.kind + let oldCC = fn.typ.callConv + fn.kind = skIterator + fn.typ.callConv = ccClosure + d.ownerToType[fn.id] = ptrType + detectCapturedVars(body, fn, d) + result = wrapIterBody(liftCapturedVars(body, fn, d, c), fn) + fn.kind = oldKind + fn.typ.callConv = oldCC + +proc liftLambdas*(fn: PSym, body: PNode; tooEarly: var bool): PNode = # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs # the transformation even when compiling to JS ... @@ -978,40 +772,35 @@ proc liftLambdas*(fn: PSym, body: PNode): PNode = fn.skipGenericOwner.kind != skModule: # ignore forward declaration: result = body + tooEarly = true else: - #if fn.name.s == "sort": - # echo rendertree(fn.ast, {renderIds}) - var o = newOuterContext(fn) - let ex = closureCreationPoint(body) - let env = newEnv(o, nil, ex, fn) - addParamsToEnv(fn, env) - searchForInnerProcs(o, body, env) - createEnvironments(o) - if fn.kind == skClosureIterator: - result = transformOuterProcBody(o, body, initIter(fn)) + var d = initDetectionPass(fn) + detectCapturedVars(body, fn, d) + if not d.somethingToDo and fn.isIterator: + addClosureParam(d, fn) + d.somethingToDo = true + if d.somethingToDo: + var c = initLiftingPass(fn) + var newBody = liftCapturedVars(body, fn, d, c) + if c.envvars.getOrDefault(fn.id) != nil: + newBody = newTree(nkStmtList, rawClosureCreation(fn, d, c), newBody) + result = wrapIterBody(newBody, fn) else: - discard transformOuterProcBody(o, body, initIter(fn)) - result = ex - finishEnvironments(o) - #if fn.name.s == "parseLong": - # echo rendertree(result, {renderIds}) + result = body + #if fn.name.s == "get2": + # echo "had something to do ", d.somethingToDo + # echo renderTree(result, {renderIds}) proc liftLambdasForTopLevel*(module: PSym, body: PNode): PNode = if body.kind == nkEmpty or gCmd == cmdCompileToJS: result = body else: - var o = newOuterContext(module) - let ex = closureCreationPoint(body) - let env = newEnv(o, nil, ex, module) - searchForInnerProcs(o, body, env) - createEnvironments(o) - discard transformOuterProc(o, body, initIter(module)) - finishEnvironments(o) - result = ex + # XXX implement it properly + result = body # ------------------- iterator transformation -------------------------------- -proc liftForLoop*(body: PNode): PNode = +proc liftForLoop*(body: PNode; owner: PSym): PNode = # problem ahead: the iterator could be invoked indirectly, but then # we don't know what environment to create here: # @@ -1049,17 +838,27 @@ proc liftForLoop*(body: PNode): PNode = # static binding? var env: PSym - if call[0].kind == nkSym and call[0].sym.kind == skClosureIterator: + let op = call[0] + if op.kind == nkSym and op.sym.isIterator: # createClosure() - let iter = call[0].sym - assert iter.kind == skClosureIterator - env = copySym(getHiddenParam(iter)) + let iter = op.sym + + let hp = getHiddenParam(iter) + env = newSym(skLet, iter.name, owner, body.info) + env.typ = hp.typ + env.flags = hp.flags var v = newNodeI(nkVarSection, body.info) addVar(v, newSymNode(env)) result.add(v) # add 'new' statement: result.add(newCall(getSysSym"internalNew", env.newSymNode)) + elif op.kind == nkStmtListExpr: + let closure = op.lastSon + if closure.kind == nkClosure: + call.sons[0] = closure + for i in 0 .. op.len-2: + result.add op[i] var loopBody = newNodeI(nkStmtList, body.info, 3) var whileLoop = newNodeI(nkWhileStmt, body.info, 2) @@ -1072,8 +871,8 @@ proc liftForLoop*(body: PNode): PNode = var v2 = newNodeI(nkLetSection, body.info) var vpart = newNodeI(if L == 3: nkIdentDefs else: nkVarTuple, body.info) for i in 0 .. L-3: - assert body[i].kind == nkSym - body[i].sym.kind = skLet + if body[i].kind == nkSym: + body[i].sym.kind = skLet addSon(vpart, body[i]) addSon(vpart, ast.emptyNode) # no explicit type diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 9a69ede3e..fc43f8d6a 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -769,24 +769,88 @@ proc getOperator(L: var TLexer, tok: var TToken) = if buf[pos] in {CR, LF, nimlexbase.EndOfFile}: tok.strongSpaceB = -1 +proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; + isDoc: bool) = + var pos = start + var buf = L.buf + var toStrip = 0 + # detect the amount of indentation: + if isDoc: + toStrip = getColNumber(L, pos) + while buf[pos] == ' ': inc pos + if buf[pos] in {CR, LF}: + pos = handleCRLF(L, pos) + buf = L.buf + toStrip = 0 + while buf[pos] == ' ': + inc pos + inc toStrip + var nesting = 0 + while true: + case buf[pos] + of '#': + if isDoc: + if buf[pos+1] == '#' and buf[pos+2] == '[': + inc nesting + tok.literal.add '#' + elif buf[pos+1] == '[': + inc nesting + inc pos + of ']': + if isDoc: + if buf[pos+1] == '#' and buf[pos+2] == '#': + if nesting == 0: + inc(pos, 3) + break + dec nesting + tok.literal.add ']' + elif buf[pos+1] == '#': + if nesting == 0: + inc(pos, 2) + break + dec nesting + inc pos + of '\t': + lexMessagePos(L, errTabulatorsAreNotAllowed, pos) + inc(pos) + if isDoc: tok.literal.add '\t' + of CR, LF: + pos = handleCRLF(L, pos) + buf = L.buf + # strip leading whitespace: + if isDoc: + tok.literal.add "\n" + inc tok.iNumber + var c = toStrip + while buf[pos] == ' ' and c > 0: + inc pos + dec c + of nimlexbase.EndOfFile: + lexMessagePos(L, errGenerated, pos, "end of multiline comment expected") + break + else: + if isDoc: tok.literal.add buf[pos] + inc(pos) + L.bufpos = pos + proc scanComment(L: var TLexer, tok: var TToken) = var pos = L.bufpos var buf = L.buf + tok.tokType = tkComment + # iNumber contains the number of '\n' in the token + tok.iNumber = 0 when not defined(nimfix): assert buf[pos+1] == '#' if buf[pos+2] == '[': - if buf[pos+3] == ']': - # ##[] is the (rather complex) "cursor token" for idetools - tok.tokType = tkComment - tok.literal = "[]" - inc(L.bufpos, 4) - return - else: - lexMessagePos(L, warnDeprecated, pos, "use '## [' instead; '##['") + skipMultiLineComment(L, tok, pos+3, true) + return + inc(pos, 2) + + var toStrip = 0 + while buf[pos] == ' ': + inc pos + inc toStrip - tok.tokType = tkComment - # iNumber contains the number of '\n' in the token - tok.iNumber = 0 when defined(nimfix): var col = getColNumber(L, pos) while true: @@ -820,6 +884,12 @@ proc scanComment(L: var TLexer, tok: var TToken) = if doContinue(): tok.literal.add "\n" when defined(nimfix): col = indent + else: + inc(pos, 2) + var c = toStrip + while buf[pos] == ' ' and c > 0: + inc pos + dec c inc tok.iNumber else: if buf[pos] > ' ': @@ -843,9 +913,16 @@ proc skip(L: var TLexer, tok: var TToken) = pos = handleCRLF(L, pos) buf = L.buf var indent = 0 - while buf[pos] == ' ': - inc(pos) - inc(indent) + while true: + if buf[pos] == ' ': + inc(pos) + inc(indent) + elif buf[pos] == '#' and buf[pos+1] == '[': + skipMultiLineComment(L, tok, pos+2, false) + pos = L.bufpos + buf = L.buf + else: + break tok.strongSpaceA = 0 when defined(nimfix): template doBreak(): expr = buf[pos] > ' ' @@ -863,8 +940,11 @@ proc skip(L: var TLexer, tok: var TToken) = # do not skip documentation comment: if buf[pos+1] == '#': break if buf[pos+1] == '[': - lexMessagePos(L, warnDeprecated, pos, "use '# [' instead; '#['") - while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) + skipMultiLineComment(L, tok, pos+2, false) + pos = L.bufpos + buf = L.buf + else: + while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) else: break # EndOfFile also leaves the loop L.bufpos = pos diff --git a/compiler/lookups.nim b/compiler/lookups.nim index e88589c3e..a337bf0f3 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -133,7 +133,7 @@ type proc getSymRepr*(s: PSym): string = case s.kind - of skProc, skMethod, skConverter, skIterators: result = getProcHeader(s) + of skProc, skMethod, skConverter, skIterator: result = getProcHeader(s) else: result = s.name.s proc ensureNoMissingOrUnusedSymbols(scope: PScope) = diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 20800b809..7a5c7f44b 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -165,9 +165,10 @@ proc indirectAccess*(a: PNode, b: string, info: TLineInfo): PNode = deref.typ = a.typ.skipTypes(abstractInst).sons[0] var t = deref.typ.skipTypes(abstractInst) var field: PSym + let bb = getIdent(b) while true: assert t.kind == tyObject - field = getSymFromList(t.n, getIdent(b)) + field = getSymFromList(t.n, bb) if field != nil: break t = t.sons[0] if t == nil: break @@ -585,7 +586,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType; objType.addField(field) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0]) fn = indirectAccess(castExpr, field, n.info) - elif fn.kind == nkSym and fn.sym.kind in {skClosureIterator, skIterator}: + elif fn.kind == nkSym and fn.sym.kind == skIterator: localError(n.info, "iterator in spawn environment is not allowed") elif fn.typ.callConv == ccClosure: localError(n.info, "closure in spawn environment is not allowed") diff --git a/compiler/nimsets.nim b/compiler/nimsets.nim index 055bae909..f15ad6368 100644 --- a/compiler/nimsets.nim +++ b/compiler/nimsets.nim @@ -106,13 +106,17 @@ proc toTreeSet(s: TBitSet, settype: PType, info: TLineInfo): PNode = inc(b) if (b >= len(s) * ElemSize) or not bitSetIn(s, b): break dec(b) + let aa = newIntTypeNode(nkIntLit, a + first, elemType) + aa.info = info if a == b: - addSon(result, newIntTypeNode(nkIntLit, a + first, elemType)) + addSon(result, aa) else: n = newNodeI(nkRange, info) n.typ = elemType - addSon(n, newIntTypeNode(nkIntLit, a + first, elemType)) - addSon(n, newIntTypeNode(nkIntLit, b + first, elemType)) + addSon(n, aa) + let bb = newIntTypeNode(nkIntLit, b + first, elemType) + bb.info = info + addSon(n, bb) addSon(result, n) e = b inc(e) diff --git a/compiler/parser.nim b/compiler/parser.nim index 11dd6788a..c4681a5cd 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -112,12 +112,7 @@ proc rawSkipComment(p: var TParser, node: PNode) = if p.tok.tokType == tkComment: if node != nil: if node.comment == nil: node.comment = "" - if p.tok.literal == "[]": - node.flags.incl nfIsCursor - #echo "parser: " - #debug node - else: - add(node.comment, p.tok.literal) + add(node.comment, p.tok.literal) else: parMessage(p, errInternal, "skipComment") getTok(p) @@ -250,12 +245,14 @@ proc isUnary(p: TParser): bool = if p.tok.tokType in {tkOpr, tkDotDot} and p.tok.strongSpaceB == 0 and p.tok.strongSpaceA > 0: - # XXX change this after 0.10.4 is out - if p.strongSpaces: result = true - else: - parMessage(p, warnDeprecated, - "will be parsed as unary operator; inconsistent spacing") + # versions prior to 0.13.0 used to do this: + when false: + if p.strongSpaces: + result = true + else: + parMessage(p, warnDeprecated, + "will be parsed as unary operator; inconsistent spacing") proc checkBinary(p: TParser) {.inline.} = ## Check if the current parser token is a binary operator. diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 7cd8e25ee..8e4aa1831 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -167,33 +167,24 @@ proc makeNimString(s: string): string = proc putComment(g: var TSrcGen, s: string) = if s.isNil: return var i = 0 - var comIndent = 1 var isCode = (len(s) >= 2) and (s[1] != ' ') var ind = g.lineLen - var com = "" + var com = "## " while true: case s[i] of '\0': break of '\x0D': put(g, tkComment, com) - com = "" + com = "## " inc(i) if s[i] == '\x0A': inc(i) optNL(g, ind) of '\x0A': put(g, tkComment, com) - com = "" + com = "## " inc(i) optNL(g, ind) - of '#': - add(com, s[i]) - inc(i) - comIndent = 0 - while s[i] == ' ': - add(com, s[i]) - inc(i) - inc(comIndent) of ' ', '\x09': add(com, s[i]) inc(i) @@ -206,7 +197,7 @@ proc putComment(g: var TSrcGen, s: string) = if not isCode and (g.lineLen + (j - i) > MaxLineLen): put(g, tkComment, com) optNL(g, ind) - com = '#' & spaces(comIndent) + com = "## " while s[i] > ' ': add(com, s[i]) inc(i) @@ -283,7 +274,7 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool = result = false if n.comment != nil: result = (renderNoComments notin g.flags) or - (renderDocComments in g.flags) and startsWith(n.comment, "##") + (renderDocComments in g.flags) proc gcom(g: var TSrcGen, n: PNode) = assert(n != nil) @@ -1330,6 +1321,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = initContext c putWithSpace g, tkSymbol, if n.kind == nkState: "state" else: "goto" gsons(g, n, c) + of nkBreakState: + put(g, tkTuple, "breakstate") of nkTypeClassTy: gTypeClassTy(g, n) else: diff --git a/compiler/sem.nim b/compiler/sem.nim index f6078830b..cddd763ce 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -186,6 +186,8 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym = result.owner = getCurrOwner() else: result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info) + #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule: + # incl(result.flags, sfGlobal) proc semIdentVis(c: PContext, kind: TSymKind, n: PNode, allowed: TSymFlags): PSym @@ -202,7 +204,7 @@ proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind) = "' in this context: '" & typeToString(typ) & "'") proc paramsTypeCheck(c: PContext, typ: PType) {.inline.} = - typeAllowedCheck(typ.n.info, typ, skConst) + typeAllowedCheck(typ.n.info, typ, skProc) proc expectMacroOrTemplateCall(c: PContext, n: PNode): PSym proc semDirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode diff --git a/compiler/semcall.nim b/compiler/semcall.nim index d8838e347..8445b24d9 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -75,7 +75,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, errors.add(err) if z.state == csMatch: # little hack so that iterators are preferred over everything else: - if sym.kind in skIterators: inc(z.exactMatches, 200) + if sym.kind == skIterator: inc(z.exactMatches, 200) case best.state of csEmpty, csNoMatch: best = z of csMatch: @@ -395,7 +395,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode = for i in countup(0, len(a)-1): var candidate = a.sons[i].sym if candidate.kind in {skProc, skMethod, skConverter, - skIterator, skClosureIterator}: + skIterator}: # it suffices that the candidate has the proper number of generic # type parameters: if safeLen(candidate.ast.sons[genericParamsPos]) == n.len-1: diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 9b2f2e2ce..656bfc449 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -315,7 +315,7 @@ proc makeRangeType*(c: PContext; first, last: BiggestInt; addSonSkipIntLit(result, intType) # basetype of range proc markIndirect*(c: PContext, s: PSym) {.inline.} = - if s.kind in {skProc, skConverter, skMethod, skIterator, skClosureIterator}: + if s.kind in {skProc, skConverter, skMethod, skIterator}: incl(s.flags, sfAddrTaken) # XXX add to 'c' for global analysis diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 95a90463c..87d7764a2 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -753,11 +753,11 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, flags: TExprFlags): PNode = if flags*{efInTypeof, efWantIterator} != {}: # consider: 'for x in pReturningArray()' --> we don't want the restriction - # to 'skIterators' anymore; skIterators are preferred in sigmatch already + # to 'skIterator' anymore; skIterator is preferred in sigmatch already # for typeof support. # for ``type(countup(1,3))``, see ``tests/ttoseq``. result = semOverloadedCall(c, n, nOrig, - {skProc, skMethod, skConverter, skMacro, skTemplate}+skIterators) + {skProc, skMethod, skConverter, skMacro, skTemplate, skIterator}) else: result = semOverloadedCall(c, n, nOrig, {skProc, skMethod, skConverter, skMacro, skTemplate}) @@ -770,7 +770,7 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, case callee.kind of skMacro, skTemplate: discard else: - if callee.kind in skIterators and callee.id == c.p.owner.id: + if callee.kind == skIterator and callee.id == c.p.owner.id: localError(n.info, errRecursiveDependencyX, callee.name.s) # error correction, prevents endless for loop elimination in transf. # See bug #2051: @@ -1201,7 +1201,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = let s = if n.sons[0].kind == nkSym: n.sons[0].sym elif n[0].kind in nkSymChoices: n.sons[0][0].sym else: nil - if s != nil and s.kind in {skProc, skMethod, skConverter}+skIterators: + if s != nil and s.kind in {skProc, skMethod, skConverter, skIterator}: # type parameters: partial generic specialization n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s) result = explicitGenericInstantiation(c, n, s) @@ -1349,8 +1349,8 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = proc semReturn(c: PContext, n: PNode): PNode = result = n checkSonsLen(n, 1) - if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro, - skClosureIterator}: + if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro} or ( + c.p.owner.kind == skIterator and c.p.owner.typ.callConv == ccClosure): if n.sons[0].kind != nkEmpty: # transform ``return expr`` to ``result = expr; return`` if c.p.resultSym != nil: @@ -1426,7 +1426,7 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) = proc semYield(c: PContext, n: PNode): PNode = result = n checkSonsLen(n, 1) - if c.p.owner == nil or c.p.owner.kind notin skIterators: + if c.p.owner == nil or c.p.owner.kind != skIterator: localError(n.info, errYieldNotAllowedHere) elif c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline: localError(n.info, errYieldNotAllowedInTryStmt) @@ -1817,6 +1817,7 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = whenNimvm = lookUp(c, exprNode).magic == mNimvm elif exprNode.kind == nkSym: whenNimvm = exprNode.sym.magic == mNimvm + if whenNimvm: n.flags.incl nfLL for i in countup(0, sonsLen(n) - 1): var it = n.sons[i] @@ -2124,7 +2125,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = var s = lookUp(c, n) if c.inTypeClass == 0: semCaptureSym(s, c.p.owner) result = semSym(c, n, s, flags) - if s.kind in {skProc, skMethod, skConverter}+skIterators: + if s.kind in {skProc, skMethod, skConverter, skIterator}: #performProcvarCheck(c, n, s) result = symChoice(c, n, s, scClosed) if result.kind == nkSym: @@ -2212,7 +2213,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = localError(n.info, errUseQualifier, s.name.s) elif s.magic == mNone: result = semDirectOp(c, n, flags) else: result = semMagic(c, n, s, flags) - of skProc, skMethod, skConverter, skIterators: + of skProc, skMethod, skConverter, skIterator: if s.magic == mNone: result = semDirectOp(c, n, flags) else: result = semMagic(c, n, s, flags) else: diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index 620453277..6651de78e 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -58,7 +58,7 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym, of skUnknown: # Introduced in this pass! Leave it as an identifier. result = n - of skProc, skMethod, skIterators, skConverter, skModule: + of skProc, skMethod, skIterator, skConverter, skModule: result = symChoice(c, n, s, scOpen) of skTemplate: if macroToExpand(s): @@ -226,7 +226,7 @@ proc semGenericStmt(c: PContext, n: PNode, of skUnknown, skParam: # Leave it as an identifier. discard - of skProc, skMethod, skIterators, skConverter, skModule: + of skProc, skMethod, skIterator, skConverter, skModule: result.sons[0] = symChoice(c, fn, s, scOption) # do not check of 's.magic==mRoof' here because it might be some # other '^' but after overload resolution the proper one: diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index deef38ae3..f98ff0266 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -207,7 +207,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, result = n.sons[1] else: result = newNodeIT(nkCall, n.info, getSysType(tyInt)) - result.add newSymNode(createMagic("-", mSubI), n.info) + result.add newSymNode(getSysMagic("-", mSubI), n.info) result.add lenExprB result.add n.sons[1] of mPlugin: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index e80f1cfda..fdf147a2e 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -84,7 +84,7 @@ proc performProcvarCheck(c: PContext, n: PNode, s: PSym) = proc semProcvarCheck(c: PContext, n: PNode) = let n = n.skipConv if n.kind == nkSym and n.sym.kind in {skProc, skMethod, skConverter, - skIterator, skClosureIterator}: + skIterator}: performProcvarCheck(c, n, n.sym) proc semProc(c: PContext, n: PNode): PNode @@ -326,6 +326,8 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym = incl(result.flags, sfGlobal) else: result = semIdentWithPragma(c, kind, n, {}) + if result.owner.kind == skModule: + incl(result.flags, sfGlobal) suggestSym(n.info, result) styleCheckDef(result) @@ -598,7 +600,7 @@ proc semFor(c: PContext, n: PNode): PNode = # first class iterator: result = semForVars(c, n) elif not isCallExpr or call.sons[0].kind != nkSym or - call.sons[0].sym.kind notin skIterators: + call.sons[0].sym.kind != skIterator: if length == 3: n.sons[length-2] = implicitIterator(c, "items", n.sons[length-2]) elif length == 4: @@ -958,15 +960,17 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode = var n = n let original = n.sons[namePos].sym - let s = copySym(original, false) - incl(s.flags, sfFromGeneric) + let s = original #copySym(original, false) + #incl(s.flags, sfFromGeneric) + #s.owner = original n = replaceTypesInBody(c, pt, n, original) result = n s.ast = result n.sons[namePos].sym = s n.sons[genericParamsPos] = emptyNode - let params = n.typ.n + # for LL we need to avoid wrong aliasing + let params = copyTree n.typ.n n.sons[paramsPos] = params s.typ = n.typ for i in 1..<params.len: @@ -974,6 +978,7 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode = tyFromExpr, tyFieldAccessor}+tyTypeClasses: localError(params[i].info, "cannot infer type of parameter: " & params[i].sym.name.s) + #params[i].sym.owner = s openScope(c) pushOwner(s) addParams(c, params, skProc) @@ -1006,7 +1011,8 @@ proc activate(c: PContext, n: PNode) = discard proc maybeAddResult(c: PContext, s: PSym, n: PNode) = - if s.typ.sons[0] != nil and s.kind != skIterator: + if s.typ.sons[0] != nil and not + (s.kind == skIterator and s.typ.callConv != ccClosure): addResult(c, s.typ.sons[0], n.info, s.kind) addResultNode(c, n) @@ -1143,13 +1149,15 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if tfTriggersCompileTime in s.typ.flags: incl(s.flags, sfCompileTime) if n.sons[patternPos].kind != nkEmpty: n.sons[patternPos] = semPattern(c, n.sons[patternPos]) - if s.kind in skIterators: + if s.kind == skIterator: s.typ.flags.incl(tfIterator) var proto = searchForProc(c, oldScope, s) if proto == nil: - if s.kind == skClosureIterator: s.typ.callConv = ccClosure - else: s.typ.callConv = lastOptionEntry(c).defaultCC + if s.kind == skIterator and s.typ.callConv == ccClosure: + discard + else: + s.typ.callConv = lastOptionEntry(c).defaultCC # add it here, so that recursive procs are possible: if sfGenSym in s.flags: discard elif kind in OverloadableSyms: @@ -1209,7 +1217,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, n.sons[bodyPos] = transformBody(c.module, semBody, s) popProcCon(c) else: - if s.typ.sons[0] != nil and kind notin skIterators: + if s.typ.sons[0] != nil and kind != skIterator: addDecl(c, newSym(skUnknown, getIdent"result", nil, n.info)) openScope(c) n.sons[bodyPos] = semGenericStmt(c, n.sons[bodyPos]) @@ -1230,9 +1238,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if n.sons[patternPos].kind != nkEmpty: c.patterns.add(s) if isAnon: result.typ = s.typ - if isTopLevel(c) and s.kind != skClosureIterator and + if isTopLevel(c) and s.kind != skIterator and s.typ.callConv == ccClosure: - message(s.info, warnDeprecated, "top level '.closure' calling convention") + localError(s.info, "'.closure' calling convention for top level routines is invalid") proc determineType(c: PContext, s: PSym) = if s.typ != nil: return @@ -1240,15 +1248,12 @@ proc determineType(c: PContext, s: PSym) = discard semProcAux(c, s.ast, s.kind, {}, stepDetermineType) proc semIterator(c: PContext, n: PNode): PNode = - let kind = if hasPragma(n[pragmasPos], wClosure) or - n[namePos].kind == nkEmpty: skClosureIterator - else: skIterator # gensym'ed iterator? if n[namePos].kind == nkSym: # gensym'ed iterators might need to become closure iterators: n[namePos].sym.owner = getCurrOwner() - n[namePos].sym.kind = kind - result = semProcAux(c, n, kind, iteratorPragmas) + n[namePos].sym.kind = skIterator + result = semProcAux(c, n, skIterator, iteratorPragmas) var s = result.sons[namePos].sym var t = s.typ if t.sons[0] == nil and s.typ.callConv != ccClosure: diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 2dda8276d..a4498a3ae 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -228,10 +228,7 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym): PNode = of skParam: result = n of skType: - if (s.typ != nil) and (s.typ.kind != tyGenericParam): - result = newSymNodeTypeDesc(s, n.info) - else: - result = n + result = newSymNodeTypeDesc(s, n.info) else: result = newSymNode(s, n.info) @@ -456,9 +453,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode = of nkMethodDef: result = semRoutineInTemplBody(c, n, skMethod) of nkIteratorDef: - let kind = if hasPragma(n[pragmasPos], wClosure): skClosureIterator - else: skIterator - result = semRoutineInTemplBody(c, n, kind) + result = semRoutineInTemplBody(c, n, skIterator) of nkTemplateDef: result = semRoutineInTemplBody(c, n, skTemplate) of nkMacroDef: diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 65edb756f..ac425ba15 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -960,10 +960,6 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, var r: PType if n.sons[0].kind != nkEmpty: r = semTypeNode(c, n.sons[0], nil) - elif kind == skIterator: - # XXX This is special magic we should likely get rid of - r = newTypeS(tyExpr, c) - message(n.info, warnDeprecated, "implicit return type for 'iterator'") if r != nil: # turn explicit 'void' return type into 'nil' because the rest of the @@ -1296,7 +1292,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = child.flags.incl tfIterator result.addSonSkipIntLit(child) else: - result = semProcTypeWithScope(c, n, prev, skClosureIterator) + result = semProcTypeWithScope(c, n, prev, skIterator) result.flags.incl(tfIterator) if n.lastSon.kind == nkPragma and hasPragma(n.lastSon, wInline): result.callConv = ccInline diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index aee42e021..354cf965e 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -167,12 +167,12 @@ proc sumGeneric(t: PType): int = t = t.lastSon if t.kind == tyEmpty: break inc result - of tyGenericInvocation, tyTuple: + of tyGenericInvocation, tyTuple, tyProc: result += ord(t.kind == tyGenericInvocation) for i in 0 .. <t.len: result += t.sons[i].sumGeneric break of tyGenericParam, tyExpr, tyStatic, tyStmt: break - of tyBool, tyChar, tyEnum, tyObject, tyProc, tyPointer, + of tyBool, tyChar, tyEnum, tyObject, tyPointer, tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128, tyUInt..tyUInt64: return isvar @@ -1442,7 +1442,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType, z.calleeSym = m.calleeSym var best = -1 for i in countup(0, sonsLen(arg) - 1): - if arg.sons[i].sym.kind in {skProc, skMethod, skConverter}+skIterators: + if arg.sons[i].sym.kind in {skProc, skMethod, skConverter, skIterator}: copyCandidate(z, m) z.callee = arg.sons[i].typ z.calleeSym = arg.sons[i].sym diff --git a/compiler/transf.nim b/compiler/transf.nim index 3a5ff982e..3e074841e 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -45,7 +45,7 @@ type inlining: int # > 0 if we are in inlining context (copy vars) nestedProcs: int # > 0 if we are in a nested proc contSyms, breakSyms: seq[PSym] # to transform 'continue' and 'break' - deferDetected: bool + deferDetected, tooEarly: bool PTransf = ref TTransfContext proc newTransNode(a: PNode): PTransNode {.inline.} = @@ -93,10 +93,15 @@ proc getCurrOwner(c: PTransf): PSym = if c.transCon != nil: result = c.transCon.owner else: result = c.module -proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PSym = - result = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info) - result.typ = skipTypes(typ, {tyGenericInst}) - incl(result.flags, sfFromGeneric) +proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode = + let r = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info) + r.typ = skipTypes(typ, {tyGenericInst}) + incl(r.flags, sfFromGeneric) + let owner = getCurrOwner(c) + if owner.isIterator and not c.tooEarly: + result = freshVarForClosureIter(r, owner) + else: + result = newSymNode(r) proc transform(c: PTransf, n: PNode): PTransNode @@ -111,13 +116,22 @@ proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode = result[1] = ri proc transformSymAux(c: PTransf, n: PNode): PNode = - #if n.sym.kind == skClosureIterator: - # return liftIterSym(n) + let s = n.sym + if s.typ != nil and s.typ.callConv == ccClosure: + if s.kind == skIterator: + if c.tooEarly: return n + else: return liftIterSym(n, getCurrOwner(c)) + elif s.kind in {skProc, skConverter, skMethod} and not c.tooEarly: + # top level .closure procs are still somewhat supported for 'Nake': + return makeClosure(s, nil, n.info) + #elif n.sym.kind in {skVar, skLet} and n.sym.typ.callConv == ccClosure: + # echo n.info, " come heer for ", c.tooEarly + # if not c.tooEarly: var b: PNode var tc = c.transCon - if sfBorrow in n.sym.flags and n.sym.kind in routineKinds: + if sfBorrow in s.flags and s.kind in routineKinds: # simply exchange the symbol: - b = n.sym.getBody + b = s.getBody if b.kind != nkSym: internalError(n.info, "wrong AST for borrowed symbol") b = newSymNode(b.sym) b.info = n.info @@ -132,6 +146,16 @@ proc transformSymAux(c: PTransf, n: PNode): PNode = proc transformSym(c: PTransf, n: PNode): PTransNode = result = PTransNode(transformSymAux(c, n)) +proc freshVar(c: PTransf; v: PSym): PNode = + let owner = getCurrOwner(c) + if owner.isIterator and not c.tooEarly: + result = freshVarForClosureIter(v, owner) + else: + var newVar = copySym(v) + incl(newVar.flags, sfFromGeneric) + newVar.owner = owner + result = newSymNode(newVar) + proc transformVarSection(c: PTransf, v: PNode): PTransNode = result = newTransNode(v) for i in countup(0, sonsLen(v)-1): @@ -141,35 +165,30 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode = elif it.kind == nkIdentDefs: if it.sons[0].kind == nkSym: internalAssert(it.len == 3) - var newVar = copySym(it.sons[0].sym) - incl(newVar.flags, sfFromGeneric) - # fixes a strange bug for rodgen: - #include(it.sons[0].sym.flags, sfFromGeneric); - newVar.owner = getCurrOwner(c) - idNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar)) + let x = freshVar(c, it.sons[0].sym) + idNodeTablePut(c.transCon.mapping, it.sons[0].sym, x) var defs = newTransNode(nkIdentDefs, it.info, 3) if importantComments(): # keep documentation information: PNode(defs).comment = it.comment - defs[0] = newSymNode(newVar).PTransNode + defs[0] = x.PTransNode defs[1] = it.sons[1].PTransNode defs[2] = transform(c, it.sons[2]) - newVar.ast = defs[2].PNode + if x.kind == nkSym: x.sym.ast = defs[2].PNode result[i] = defs else: - # has been transformed into 'param.x' for closure iterators, so keep it: - result[i] = PTransNode(it) + # has been transformed into 'param.x' for closure iterators, so just + # transform it: + result[i] = transform(c, it) else: if it.kind != nkVarTuple: internalError(it.info, "transformVarSection: not nkVarTuple") var L = sonsLen(it) var defs = newTransNode(it.kind, it.info, L) for j in countup(0, L-3): - var newVar = copySym(it.sons[j].sym) - incl(newVar.flags, sfFromGeneric) - newVar.owner = getCurrOwner(c) - idNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar)) - defs[j] = newSymNode(newVar).PTransNode + let x = freshVar(c, it.sons[j].sym) + idNodeTablePut(c.transCon.mapping, it.sons[j].sym, x) + defs[j] = x.PTransNode assert(it.sons[L-2].kind == nkEmpty) defs[L-2] = ast.emptyNode.PTransNode defs[L-1] = transform(c, it.sons[L-1]) @@ -294,10 +313,18 @@ proc introduceNewLocalVars(c: PTransf, n: PNode): PTransNode = result = PTransNode(n) of nkVarSection, nkLetSection: result = transformVarSection(c, n) + of nkClosure: + # it can happen that for-loop-inlining produced a fresh + # set of variables, including some computed environment + # (bug #2604). We need to patch this environment here too: + let a = n[1] + if a.kind == nkSym: + n.sons[1] = transformSymAux(c, a) + return PTransNode(n) else: result = newTransNode(n) for i in countup(0, sonsLen(n)-1): - result[i] = introduceNewLocalVars(c, n.sons[i]) + result[i] = introduceNewLocalVars(c, n.sons[i]) proc transformYield(c: PTransf, n: PNode): PTransNode = result = newTransNode(nkStmtList, n.info, 0) @@ -348,6 +375,20 @@ proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode = # addr ( deref ( x )) --> x result = PTransNode(n.sons[0].sons[0]) +proc generateThunk(prc: PNode, dest: PType): PNode = + ## Converts 'prc' into '(thunk, nil)' so that it's compatible with + ## a closure. + + # we cannot generate a proper thunk here for GC-safety reasons + # (see internal documentation): + if gCmd == cmdCompileToJS: return prc + result = newNodeIT(nkClosure, prc.info, dest) + var conv = newNodeIT(nkHiddenStdConv, prc.info, dest) + conv.add(emptyNode) + conv.add(prc) + result.add(conv) + result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil))) + proc transformConv(c: PTransf, n: PNode): PTransNode = # numeric types need range checks: var dest = skipTypes(n.typ, abstractVarRange) @@ -428,6 +469,10 @@ proc transformConv(c: PTransf, n: PNode): PTransNode = of tyGenericParam, tyOrdinal: result = transform(c, n.sons[1]) # happens sometimes for generated assignments, etc. + of tyProc: + result = transformSons(c, n) + if dest.callConv == ccClosure and source.callConv == ccDefault: + result = generateThunk(result[1].PNode, dest).PTransNode else: result = transformSons(c, n) @@ -479,9 +524,13 @@ proc transformFor(c: PTransf, n: PNode): PTransNode = return result c.breakSyms.add(labl) if call.kind notin nkCallKinds or call.sons[0].kind != nkSym or - call.sons[0].sym.kind != skIterator: + call.sons[0].typ.callConv == ccClosure: n.sons[length-1] = transformLoopBody(c, n.sons[length-1]).PNode - result[1] = lambdalifting.liftForLoop(n).PTransNode + if not c.tooEarly: + n.sons[length-2] = transform(c, n.sons[length-2]).PNode + result[1] = lambdalifting.liftForLoop(n, getCurrOwner(c)).PTransNode + else: + result[1] = newNode(nkEmpty).PTransNode discard c.breakSyms.pop return result @@ -517,9 +566,9 @@ proc transformFor(c: PTransf, n: PNode): PTransNode = of paFastAsgn: # generate a temporary and produce an assignment statement: var temp = newTemp(c, formal.typ, formal.info) - addVar(v, newSymNode(temp)) - add(stmtList, newAsgnStmt(c, newSymNode(temp), arg.PTransNode)) - idNodeTablePut(newC.mapping, formal, newSymNode(temp)) + addVar(v, temp) + add(stmtList, newAsgnStmt(c, temp, arg.PTransNode)) + idNodeTablePut(newC.mapping, formal, temp) of paVarAsgn: assert(skipTypes(formal.typ, abstractInst).kind == tyVar) idNodeTablePut(newC.mapping, formal, arg) @@ -700,18 +749,13 @@ proc transform(c: PTransf, n: PNode): PTransNode = result = PTransNode(n) of nkBracketExpr: result = transformArrayAccess(c, n) of procDefs: - when false: - if n.sons[genericParamsPos].kind == nkEmpty: - var s = n.sons[namePos].sym - n.sons[bodyPos] = PNode(transform(c, s.getBody)) - if s.ast.sons[bodyPos] != n.sons[bodyPos]: - # somehow this can happen ... :-/ - s.ast.sons[bodyPos] = n.sons[bodyPos] - #n.sons[bodyPos] = liftLambdas(s, n) - #if n.kind == nkMethodDef: methodDef(s, false) - #if n.kind == nkIteratorDef and n.typ != nil: - # return liftIterSym(n.sons[namePos]).PTransNode - result = PTransNode(n) + var s = n.sons[namePos].sym + if n.typ != nil and s.typ.callConv == ccClosure: + result = transformSym(c, n.sons[namePos]) + # use the same node as before if still a symbol: + if result.PNode.kind == nkSym: result = PTransNode(n) + else: + result = PTransNode(n) of nkMacroDef: # XXX no proper closure support yet: when false: @@ -748,7 +792,7 @@ proc transform(c: PTransf, n: PNode): PTransNode = result = newTransNode(nkCommentStmt, n.info, 0) tryStmt.addSon(deferPart) # disable the original 'defer' statement: - n.kind = nkCommentStmt + n.kind = nkEmpty of nkContinueStmt: result = PTransNode(newNodeI(nkBreakStmt, n.info)) var labl = c.contSyms[c.contSyms.high] @@ -794,7 +838,14 @@ proc transform(c: PTransf, n: PNode): PTransNode = # XXX comment handling really sucks: if importantComments(): PNode(result).comment = n.comment - of nkClosure: return PTransNode(n) + of nkClosure: + # it can happen that for-loop-inlining produced a fresh + # set of variables, including some computed environment + # (bug #2604). We need to patch this environment here too: + let a = n[1] + if a.kind == nkSym: + n.sons[1] = transformSymAux(c, a) + return PTransNode(n) else: result = transformSons(c, n) when false: @@ -866,11 +917,11 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode = result = n else: var c = openTransf(module, "") - result = processTransf(c, n, prc) + result = liftLambdas(prc, n, c.tooEarly) + #result = n + result = processTransf(c, result, prc) liftDefer(c, result) - result = liftLambdas(prc, result) - #if prc.kind == skClosureIterator: - # result = lambdalifting.liftIterator(prc, result) + #result = liftLambdas(prc, result) incl(result.flags, nfTransf) when useEffectSystem: trackProc(prc, result) #if prc.name.s == "testbody": @@ -883,9 +934,11 @@ proc transformStmt*(module: PSym, n: PNode): PNode = var c = openTransf(module, "") result = processTransf(c, n, module) liftDefer(c, result) - result = liftLambdasForTopLevel(module, result) + #result = liftLambdasForTopLevel(module, result) incl(result.flags, nfTransf) when useEffectSystem: trackTopLevelStmt(module, result) + #if n.info ?? "temp.nim": + # echo renderTree(result, {renderIds}) proc transformExpr*(module: PSym, n: PNode): PNode = if nfTransf in n.flags: diff --git a/compiler/types.nim b/compiler/types.nim index 3846be8a0..71ab84022 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1061,7 +1061,8 @@ proc typeAllowedNode(marker: var IntSet, n: PNode, kind: TSymKind, else: for i in countup(0, sonsLen(n) - 1): let it = n.sons[i] - if it.kind == nkRecCase and kind == skConst: return n.typ + if it.kind == nkRecCase and kind in {skProc, skConst}: + return n.typ result = typeAllowedNode(marker, it, kind, flags) if result != nil: break @@ -1076,7 +1077,7 @@ proc matchType*(a: PType, pattern: openArray[tuple[k:TTypeKind, i:int]], proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, flags: TTypeAllowedFlags = {}): PType = - assert(kind in {skVar, skLet, skConst, skParam, skResult}) + assert(kind in {skVar, skLet, skConst, skProc, skParam, skResult}) # if we have already checked the type, return true, because we stop the # evaluation if something is wrong: result = nil @@ -1085,7 +1086,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, var t = skipTypes(typ, abstractInst-{tyTypeDesc}) case t.kind of tyVar: - if kind == skConst: return t + if kind in {skProc, skConst}: return t var t2 = skipTypes(t.sons[0], abstractInst-{tyTypeDesc}) case t2.kind of tyVar: @@ -1097,6 +1098,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, if kind notin {skParam, skResult}: result = t else: result = typeAllowedAux(marker, t2, kind, flags) of tyProc: + if kind == skConst and t.callConv == ccClosure: return t for i in countup(1, sonsLen(t) - 1): result = typeAllowedAux(marker, t.sons[i], skParam, flags) if result != nil: break @@ -1144,7 +1146,8 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = typeAllowedAux(marker, t.sons[i], kind, flags) if result != nil: break of tyObject, tyTuple: - if kind == skConst and t.kind == tyObject and t.sons[0] != nil: return t + if kind in {skProc, skConst} and + t.kind == tyObject and t.sons[0] != nil: return t let flags = flags+{taField} for i in countup(0, sonsLen(t) - 1): result = typeAllowedAux(marker, t.sons[i], kind, flags) diff --git a/compiler/vm.nim b/compiler/vm.nim index 0e63daf89..80c2c0fbf 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -359,7 +359,14 @@ proc opConv*(dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): bool = of tyFloat..tyFloat64: dest.intVal = int(src.floatVal) else: - dest.intVal = src.intVal and ((1 shl (desttyp.size*8))-1) + let srcDist = (sizeof(src.intVal) - srctyp.size) * 8 + let destDist = (sizeof(dest.intVal) - desttyp.size) * 8 + when system.cpuEndian == bigEndian: + dest.intVal = (src.intVal shr srcDist) shl srcDist + dest.intVal = (dest.intVal shr destDist) shl destDist + else: + dest.intVal = (src.intVal shl srcDist) shr srcDist + dest.intVal = (dest.intVal shl destDist) shr destDist of tyFloat..tyFloat64: if dest.kind != rkFloat: myreset(dest); dest.kind = rkFloat @@ -611,7 +618,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = addSon(regs[ra].node, r.copyTree) of opcExcl: decodeB(rkNode) - var b = newNodeIT(nkCurly, regs[rb].node.info, regs[rb].node.typ) + var b = newNodeIT(nkCurly, regs[ra].node.info, regs[ra].node.typ) addSon(b, regs[rb].regToNode) var r = diffSets(regs[ra].node, b) discardSons(regs[ra].node) diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 2cc4a107b..a4f02092d 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -70,7 +70,7 @@ proc atomicTypeX(name: string; t: PType; info: TLineInfo): PNode = proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode proc mapTypeToBracket(name: string; t: PType; info: TLineInfo): PNode = - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicTypeX(name, t, info) for i in 0 .. < t.len: if t.sons[i] == nil: @@ -92,19 +92,19 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyStmt: result = atomicType("stmt") of tyEmpty: result = atomicType"void" of tyArrayConstr, tyArray: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("array") result.add mapTypeToAst(t.sons[0], info) result.add mapTypeToAst(t.sons[1], info) of tyTypeDesc: if t.base != nil: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("typeDesc") result.add mapTypeToAst(t.base, info) else: result = atomicType"typeDesc" of tyGenericInvocation: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) for i in 0 .. < t.len: result.add mapTypeToAst(t.sons[i], info) of tyGenericInst, tyGenericBody, tyOrdinal, tyUserTypeClassInst: @@ -117,7 +117,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyGenericParam, tyForward: result = atomicType(t.sym.name.s) of tyObject: if allowRecursion: - result = newNodeIT(nkObjectTy, info, t) + result = newNodeIT(nkObjectTy, if t.n.isNil: info else: t.n.info, t) if t.sons[0] == nil: result.add ast.emptyNode else: @@ -126,7 +126,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = else: result = atomicType(t.sym.name.s) of tyEnum: - result = newNodeIT(nkEnumTy, info, t) + result = newNodeIT(nkEnumTy, if t.n.isNil: info else: t.n.info, t) result.add copyTree(t.n) of tyTuple: result = mapTypeToBracket("tuple", t, info) of tySet: result = mapTypeToBracket("set", t, info) @@ -137,7 +137,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyProc: result = mapTypeToBracket("proc", t, info) of tyOpenArray: result = mapTypeToBracket("openArray", t, info) of tyRange: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("range") result.add t.n.sons[0].copyTree result.add t.n.sons[1].copyTree @@ -174,7 +174,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyNot: result = mapTypeToBracket("not", t, info) of tyAnything: result = atomicType"anything" of tyStatic, tyFromExpr, tyFieldAccessor: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("static") if t.n != nil: result.add t.n.copyTree diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 2a16406e7..75c1378e5 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1209,7 +1209,7 @@ proc checkCanEval(c: PCtx; n: PNode) = not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl: cannotEval(n) elif s.kind in {skProc, skConverter, skMethod, - skIterator, skClosureIterator} and sfForward in s.flags: + skIterator} and sfForward in s.flags: cannotEval(n) proc isTemp(c: PCtx; dest: TDest): bool = @@ -1638,7 +1638,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = case s.kind of skVar, skForVar, skTemp, skLet, skParam, skResult: genRdVar(c, n, dest, flags) - of skProc, skConverter, skMacro, skTemplate, skMethod, skIterators: + of skProc, skConverter, skMacro, skTemplate, skMethod, skIterator: # 'skTemplate' is only allowed for 'getAst' support: if procIsCallback(c, s): discard elif sfImportc in s.flags: c.importcSym(n.info, s) |