diff options
113 files changed, 1621 insertions, 1130 deletions
diff --git a/compiler/ast.nim b/compiler/ast.nim index 0652bd3e7..97f48b253 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -13,7 +13,7 @@ import msgs, hashes, nversion, options, strutils, crc, ropes, idents, lists, intsets, idgen -type +type TCallingConvention* = enum ccDefault, # proc has no explicit calling convention ccStdCall, # procedure is stdcall @@ -299,7 +299,7 @@ const nkEffectList* = nkArgList # hacks ahead: an nkEffectList is a node with 4 children: exceptionEffects* = 0 # exceptions at position 0 - readEffects* = 1 # read effects at position 1 + usesEffects* = 1 # read effects at position 1 writeEffects* = 2 # write effects at position 2 tagEffects* = 3 # user defined tag ('gc', 'time' etc.) effectListLen* = 4 # list of effects list @@ -432,7 +432,7 @@ type tfAcyclic, # type is acyclic (for GC optimization) tfEnumHasHoles, # enum cannot be mapped into a range tfShallow, # type can be shallow copied on assignment - tfThread, # proc type is marked as ``thread`` + tfThread, # proc type is marked as ``thread``; alias for ``gcsafe`` tfFromGeneric, # type is an instantiation of a generic; this is needed # because for instantiations of objects, structural # type equality has to be used @@ -509,6 +509,7 @@ const tfIncompleteStruct* = tfVarargs tfUncheckedArray* = tfVarargs tfUnion* = tfNoSideEffect + tfGcSafe* = tfThread skError* = skUnknown # type flags that are essential for type equality: @@ -559,7 +560,7 @@ type mFloat, mFloat32, mFloat64, mFloat128, mBool, mChar, mString, mCstring, mPointer, mEmptySet, mIntSetBaseType, mNil, mExpr, mStmt, mTypeDesc, - mVoidType, mPNimrodNode, + mVoidType, mPNimrodNode, mShared, mGuarded, mLock, mSpawn, mIsMainModule, mCompileDate, mCompileTime, mNimrodVersion, mNimrodMajor, mNimrodMinor, mNimrodPatch, mCpuEndian, mHostOS, mHostCPU, mAppType, mNaN, mInf, mNegInf, @@ -978,6 +979,8 @@ template `{}=`*(n: PNode, i: int, s: PNode): stmt = var emptyNode* = newNode(nkEmpty) # There is a single empty node that is shared! Do not overwrite it! +var anyGlobal* = newSym(skVar, getIdent("*"), nil, unknownLineInfo()) + proc isMetaType*(t: PType): bool = return t.kind in tyMetaTypes or (t.kind == tyStatic and t.n == nil) or @@ -1308,10 +1311,13 @@ proc skipTypes*(t: PType, kinds: TTypeKinds): PType = result = t while result.kind in kinds: result = lastSon(result) +proc isGCedMem*(t: PType): bool {.inline.} = + result = t.kind in {tyString, tyRef, tySequence} or + t.kind == tyProc and t.callConv == ccClosure + proc propagateToOwner*(owner, elem: PType) = const HaveTheirOwnEmpty = {tySequence, tySet} - owner.flags = owner.flags + (elem.flags * {tfHasShared, tfHasMeta, - tfHasGCedMem}) + owner.flags = owner.flags + (elem.flags * {tfHasShared, tfHasMeta}) if tfNotNil in elem.flags: if owner.kind in {tyGenericInst, tyGenericBody, tyGenericInvokation}: owner.flags.incl tfNotNil @@ -1328,9 +1334,9 @@ proc propagateToOwner*(owner, elem: PType) = if elem.isMetaType: owner.flags.incl tfHasMeta - if elem.kind in {tyString, tyRef, tySequence} or - elem.kind == tyProc and elem.callConv == ccClosure: - owner.flags.incl tfHasGCedMem + if owner.kind != tyProc: + if elem.isGCedMem or tfHasGCedMem in elem.flags: + owner.flags.incl tfHasGCedMem proc rawAddSon*(father, son: PType) = if isNil(father.sons): father.sons = @[] diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 36dd7f562..dbf13f764 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -448,9 +448,9 @@ proc debug(n: PSym) = writeln(stdout, "skUnknown") else: #writeln(stdout, ropeToStr(symToYaml(n, 0, 1))) - writeln(stdout, ropeToStr(ropef("$1_$2: $3, $4", [ + writeln(stdout, ropeToStr(ropef("$1_$2: $3, $4, $5", [ toRope(n.name.s), toRope(n.id), flagsToStr(n.flags), - flagsToStr(n.loc.flags)]))) + flagsToStr(n.loc.flags), lineInfoToStr(n.info)]))) proc debug(n: PType) = writeln(stdout, ropeToStr(debugType(n))) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 7c4cc2b80..49350fa9c 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1635,6 +1635,9 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = localError(e.info, errCannotGenerateCodeForX, e.sons[0].sym.name.s) of mSlurp..mQuoteAst: localError(e.info, errXMustBeCompileTime, e.sons[0].sym.name.s) + of mSpawn: + let n = lowerings.wrapProcForSpawn(p.module.module, e.sons[1]) + expr(p, n, d) else: internalError(e.info, "genMagicExpr: " & $op) proc genConstExpr(p: BProc, n: PNode): PRope diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 683aed069..f64ebacfb 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -14,7 +14,7 @@ import options, intsets, nversion, nimsets, msgs, crc, bitsets, idents, lists, types, ccgutils, os, times, ropes, math, passes, rodread, wordrecg, treetab, cgmeth, - rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases + rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings when options.hasTinyCBackend: import tccgen diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index ddf2075b4..4117fc461 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2013 Andreas Rumpf +# (c) Copyright 2014 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -48,6 +48,7 @@ proc initDefines*() = defineSymbol("nimbabel") defineSymbol("nimcomputedgoto") defineSymbol("nimunion") + defineSymbol("nimnewshared") # add platform specific symbols: case targetCPU diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 440468ac4..c5b9d0f00 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -141,12 +141,6 @@ type closureParam, state, resultSym: PSym # only if isIter obj: PType # only if isIter -proc createObj(owner: PSym, info: TLineInfo): PType = - result = newType(tyObject, owner) - rawAddSon(result, nil) - incl result.flags, tfFinal - result.n = newNodeI(nkRecList, info) - proc getStateType(iter: PSym): PType = var n = newNodeI(nkRange, iter.info) addSon(n, newIntNode(nkIntLit, -1)) @@ -189,15 +183,6 @@ proc getEnvParam(routine: PSym): PSym = let hidden = lastSon(params) if hidden.kind == nkSym and hidden.sym.name.s == paramName: result = hidden.sym - -proc addField(obj: PType; s: PSym) = - # because of 'gensym' support, we have to mangle the name with its ID. - # This is hacky but the clean solution is much more complex than it looks. - var field = newSym(skField, getIdent(s.name.s & $s.id), s.owner, s.info) - let t = skipIntLit(s.typ) - field.typ = t - field.position = sonsLen(obj.n) - addSon(obj.n, newSymNode(field)) proc initIterContext(c: POuterContext, iter: PSym) = c.fn = iter @@ -273,22 +258,6 @@ proc addDep(e, d: PEnv, owner: PSym): PSym = rawAddSon(result.typ, d.obj) addField(e.obj, result) e.deps.add((d, result)) - -proc indirectAccess(a: PNode, b: PSym, info: TLineInfo): PNode = - # returns a[].b as a node - var deref = newNodeI(nkHiddenDeref, info) - deref.typ = a.typ.sons[0] - assert deref.typ.kind == tyObject - let field = getSymFromList(deref.typ.n, getIdent(b.name.s & $b.id)) - assert field != nil, b.name.s - addSon(deref, a) - result = newNodeI(nkDotExpr, info) - addSon(result, deref) - addSon(result, newSymNode(field)) - result.typ = field.typ - -proc indirectAccess(a, b: PSym, info: TLineInfo): PNode = - result = indirectAccess(newSymNode(a), b, info) proc newCall(a, b: PSym): PNode = result = newNodeI(nkCall, a.info) diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 217e33675..2bfd8d1eb 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -43,7 +43,7 @@ type tkLambda, tkLet, tkMacro, tkMethod, tkMixin, tkMod, tkNil, tkNot, tkNotin, tkObject, tkOf, tkOr, tkOut, - tkProc, tkPtr, tkRaise, tkRef, tkReturn, tkShared, tkShl, tkShr, tkStatic, + tkProc, tkPtr, tkRaise, tkRef, tkReturn, tkShl, tkShr, tkStatic, tkTemplate, tkTry, tkTuple, tkType, tkUsing, tkVar, tkWhen, tkWhile, tkWith, tkWithout, tkXor, @@ -79,7 +79,7 @@ const "macro", "method", "mixin", "mod", "nil", "not", "notin", "object", "of", "or", "out", "proc", "ptr", "raise", "ref", "return", - "shared", "shl", "shr", "static", + "shl", "shr", "static", "template", "try", "tuple", "type", "using", "var", "when", "while", "with", "without", "xor", diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 2cf641d93..1b9e5fe0f 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -12,7 +12,7 @@ const genPrefix* = ":tmp" # prefix for generated names -import ast, types, idents, magicsys +import ast, astalgo, types, idents, magicsys, msgs, options proc newTupleAccess*(tup: PNode, i: int): PNode = result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes( @@ -34,6 +34,11 @@ proc newAsgnStmt(le, ri: PNode): PNode = result.sons[0] = le result.sons[1] = ri +proc newFastAsgnStmt(le, ri: PNode): PNode = + result = newNodeI(nkFastAsgn, le.info, 2) + result.sons[0] = le + result.sons[1] = ri + proc lowerTupleUnpacking*(n: PNode; owner: PSym): PNode = assert n.kind == nkVarTuple let value = n.lastSon @@ -50,3 +55,174 @@ proc lowerTupleUnpacking*(n: PNode; owner: PSym): PNode = result.add newAsgnStmt(newSymNode(temp), value) for i in 0 .. n.len-3: result.add newAsgnStmt(n.sons[i], newTupleAccess(value, i)) + +proc createObj*(owner: PSym, info: TLineInfo): PType = + result = newType(tyObject, owner) + rawAddSon(result, nil) + incl result.flags, tfFinal + result.n = newNodeI(nkRecList, info) + +proc addField*(obj: PType; s: PSym) = + # because of 'gensym' support, we have to mangle the name with its ID. + # This is hacky but the clean solution is much more complex than it looks. + var field = newSym(skField, getIdent(s.name.s & $s.id), s.owner, s.info) + let t = skipIntLit(s.typ) + field.typ = t + field.position = sonsLen(obj.n) + addSon(obj.n, newSymNode(field)) + +proc newDotExpr(obj, b: PSym): PNode = + result = newNodeI(nkDotExpr, obj.info) + let field = getSymFromList(obj.typ.n, getIdent(b.name.s & $b.id)) + assert field != nil, b.name.s + addSon(result, newSymNode(obj)) + addSon(result, newSymNode(field)) + result.typ = field.typ + +proc indirectAccess*(a: PNode, b: PSym, info: TLineInfo): PNode = + # returns a[].b as a node + var deref = newNodeI(nkHiddenDeref, info) + deref.typ = a.typ.sons[0] + assert deref.typ.kind == tyObject + let field = getSymFromList(deref.typ.n, getIdent(b.name.s & $b.id)) + assert field != nil, b.name.s + addSon(deref, a) + result = newNodeI(nkDotExpr, info) + addSon(result, deref) + addSon(result, newSymNode(field)) + result.typ = field.typ + +proc indirectAccess*(a, b: PSym, info: TLineInfo): PNode = + result = indirectAccess(newSymNode(a), b, info) + +proc genAddrOf*(n: PNode): PNode = + result = newNodeI(nkAddr, n.info, 1) + result.sons[0] = n + result.typ = newType(tyPtr, n.typ.owner) + result.typ.rawAddSon(n.typ) + +proc callCodegenProc*(name: string, arg1: PNode; + arg2, arg3: PNode = nil): PNode = + result = newNodeI(nkCall, arg1.info) + let sym = magicsys.getCompilerProc(name) + if sym == nil: + localError(arg1.info, errSystemNeeds, name) + else: + result.add newSymNode(sym) + result.add arg1 + if arg2 != nil: result.add arg2 + if arg3 != nil: result.add arg3 + +proc createWrapperProc(f: PNode; threadParam, argsParam: PSym; + varSection, call: PNode): PSym = + var body = newNodeI(nkStmtList, f.info) + body.add varSection + body.add callCodeGenProc("nimArgsPassingDone", newSymNode(threadParam)) + body.add call + + var params = newNodeI(nkFormalParams, f.info) + params.add emptyNode + params.add threadParam.newSymNode + params.add argsParam.newSymNode + + var t = newType(tyProc, threadParam.owner) + t.rawAddSon nil + t.rawAddSon threadParam.typ + t.rawAddSon argsParam.typ + t.n = newNodeI(nkFormalParams, f.info) + t.n.add newNodeI(nkEffectList, f.info) + t.n.add threadParam.newSymNode + t.n.add argsParam.newSymNode + + let name = (if f.kind == nkSym: f.sym.name.s else: genPrefix) & "Wrapper" + result = newSym(skProc, getIdent(name), argsParam.owner, f.info) + result.ast = newProcNode(nkProcDef, f.info, body, params, newSymNode(result)) + result.typ = t + +proc createCastExpr(argsParam: PSym; objType: PType): PNode = + result = newNodeI(nkCast, argsParam.info) + result.add emptyNode + result.add newSymNode(argsParam) + result.typ = newType(tyPtr, objType.owner) + result.typ.rawAddSon(objType) + +proc wrapProcForSpawn*(owner: PSym; n: PNode): PNode = + result = newNodeI(nkStmtList, n.info) + if n.kind notin nkCallKinds or not n.typ.isEmptyType: + localError(n.info, "'spawn' takes a call expression of type void") + return + if optThreadAnalysis in gGlobalOptions: + if {tfThread, tfNoSideEffect} * n[0].typ.flags == {}: + localError(n.info, "'spawn' takes a GC safe call expression") + var + threadParam = newSym(skParam, getIdent"thread", owner, n.info) + argsParam = newSym(skParam, getIdent"args", owner, n.info) + block: + let ptrType = getSysType(tyPointer) + threadParam.typ = ptrType + argsParam.typ = ptrType + argsParam.position = 1 + var objType = createObj(owner, n.info) + incl(objType.flags, tfFinal) + let castExpr = createCastExpr(argsParam, objType) + + var scratchObj = newSym(skVar, getIdent"scratch", owner, n.info) + block: + scratchObj.typ = objType + incl(scratchObj.flags, sfFromGeneric) + var varSectionB = newNodeI(nkVarSection, n.info) + varSectionB.addVar(scratchObj.newSymNode) + result.add varSectionB + + var call = newNodeI(nkCall, n.info) + var fn = n.sons[0] + # templates and macros are in fact valid here due to the nature of + # the transformation: + if not (fn.kind == nkSym and fn.sym.kind in {skProc, skTemplate, skMacro, + skMethod, skConverter}): + # for indirect calls we pass the function pointer in the scratchObj + var argType = n[0].typ.skipTypes(abstractInst) + var field = newSym(skField, getIdent"fn", owner, n.info) + field.typ = argType + objType.addField(field) + result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0]) + fn = indirectAccess(castExpr, field, n.info) + elif fn.kind == nkSym and fn.sym.kind in {skClosureIterator, skIterator}: + localError(n.info, "iterator in spawn environment is not allowed") + elif fn.typ.callConv == ccClosure: + localError(n.info, "closure in spawn environment is not allowed") + + call.add(fn) + var varSection = newNodeI(nkVarSection, n.info) + let formals = n[0].typ.n + let tmpName = getIdent(genPrefix) + for i in 1 .. <n.len: + # we pick n's type here, which hopefully is 'tyArray' and not + # 'tyOpenArray': + var argType = n[i].typ.skipTypes(abstractInst) + if i < formals.len and formals[i].typ.kind == tyVar: + localError(n[i].info, "'spawn'ed function cannot have a 'var' parameter") + elif containsTyRef(argType): + localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure") + + let fieldname = if i < formals.len: formals[i].sym.name else: tmpName + var field = newSym(skField, fieldname, owner, n.info) + field.typ = argType + objType.addField(field) + result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[i]) + + var temp = newSym(skTemp, tmpName, owner, n.info) + temp.typ = argType + incl(temp.flags, sfFromGeneric) + + var vpart = newNodeI(nkIdentDefs, n.info, 3) + vpart.sons[0] = newSymNode(temp) + vpart.sons[1] = ast.emptyNode + vpart.sons[2] = indirectAccess(castExpr, field, n.info) + varSection.add vpart + + call.add(newSymNode(temp)) + + let wrapper = createWrapperProc(fn, threadParam, argsParam, varSection, call) + result.add callCodeGenProc("nimSpawn", wrapper.newSymNode, + genAddrOf(scratchObj.newSymNode)) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index a63fbca7f..8374c92a7 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -118,7 +118,7 @@ type warnNilStatement, warnAnalysisLoophole, warnDifferentHeaps, warnWriteToForeignHeap, warnImplicitClosure, warnEachIdentIsTuple, warnShadowIdent, - warnProveInit, warnProveField, warnProveIndex, + warnProveInit, warnProveField, warnProveIndex, warnGcUnsafe, warnGcUnsafe2, warnUninit, warnGcMem, warnUser, hintSuccess, hintSuccessX, hintLineTooLong, hintXDeclaredButNotUsed, hintConvToBaseNotNeeded, @@ -170,7 +170,7 @@ const errInvalidNumberOfYieldExpr: "invalid number of \'yield\' expressions", errCannotReturnExpr: "current routine cannot return an expression", errAttemptToRedefine: "redefinition of \'$1\'", - errStmtInvalidAfterReturn: "statement not allowed after \'return\', \'break\' or \'raise\'", + errStmtInvalidAfterReturn: "statement not allowed after \'return\', \'break\', \'raise\' or \'continue'", errStmtExpected: "statement expected", errInvalidLabel: "\'$1\' is no label", errInvalidCmdLineOption: "invalid command line option: \'$1\'", @@ -386,6 +386,8 @@ const warnProveInit: "Cannot prove that '$1' is initialized. This will become a compile time error in the future. [ProveInit]", warnProveField: "cannot prove that field '$1' is accessible [ProveField]", warnProveIndex: "cannot prove index '$1' is valid [ProveIndex]", + warnGcUnsafe: "not GC-safe: '$1' [GcUnsafe]", + warnGcUnsafe2: "cannot prove '$1' is GC-safe. This will become a compile time error in the future.", warnUninit: "'$1' might not have been initialized [Uninit]", warnGcMem: "'$1' uses GC'ed memory [GcMem]", warnUser: "$1 [User]", @@ -407,7 +409,7 @@ const hintUser: "$1 [User]"] const - WarningsToStr*: array[0..24, string] = ["CannotOpenFile", "OctalEscape", + WarningsToStr*: array[0..26, string] = ["CannotOpenFile", "OctalEscape", "XIsNeverRead", "XmightNotBeenInit", "Deprecated", "ConfigDeprecated", "SmallLshouldNotBeUsed", "UnknownMagic", @@ -415,7 +417,8 @@ const "CommentXIgnored", "NilStmt", "AnalysisLoophole", "DifferentHeaps", "WriteToForeignHeap", "ImplicitClosure", "EachIdentIsTuple", "ShadowIdent", - "ProveInit", "ProveField", "ProveIndex", "Uninit", "GcMem", "User"] + "ProveInit", "ProveField", "ProveIndex", "GcUnsafe", "GcUnsafe2", "Uninit", + "GcMem", "User"] HintsToStr*: array[0..15, string] = ["Success", "SuccessX", "LineTooLong", "XDeclaredButNotUsed", "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded", @@ -557,7 +560,7 @@ proc sourceLine*(i: TLineInfo): PRope var gNotes*: TNoteKinds = {low(TNoteKind)..high(TNoteKind)} - {warnShadowIdent, warnUninit, - warnProveField, warnProveIndex} + warnProveField, warnProveIndex, warnGcUnsafe} gErrorCounter*: int = 0 # counts the number of errors gHintCounter*: int = 0 gWarnCounter*: int = 0 diff --git a/compiler/options.nim b/compiler/options.nim index fa8b77ead..36d343d1b 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -8,7 +8,7 @@ # import - os, lists, strutils, strtabs + os, lists, strutils, strtabs, osproc, sets const hasTinyCBackend* = defined(tinyc) @@ -16,6 +16,7 @@ const hasFFI* = defined(useFFI) newScopeForIf* = true useCaas* = not defined(noCaas) + noTimeMachine = defined(avoidTimeMachine) and defined(macosx) type # please make sure we have under 32 options # (improves code efficiency a lot!) @@ -94,7 +95,7 @@ var optBoundsCheck, optOverflowCheck, optAssert, optWarns, optHints, optStackTrace, optLineTrace, optPatterns, optNilCheck} - gGlobalOptions*: TGlobalOptions = {optThreadAnalysis} + gGlobalOptions*: TGlobalOptions = {} gExitcode*: int8 gCmd*: TCommands = cmdNone # the command gSelectedGC* = gcRefc # the selected GC @@ -263,6 +264,28 @@ proc toGeneratedFile*(path, ext: string): string = result = joinPath([getGeneratedPath(), changeFileExt(tail, ext)]) #echo "toGeneratedFile(", path, ", ", ext, ") = ", result +when noTimeMachine: + var alreadyExcludedDirs = initSet[string]() + proc excludeDirFromTimeMachine(dir: string) {.raises: [].} = + ## Calls a macosx command on the directory to exclude it from backups. + ## + ## The macosx tmutil command is invoked to mark the specified path as an + ## item to be excluded from time machine backups. If a path already exists + ## with files before excluding it, newer files won't be added to the + ## directory, but previous files won't be removed from the backup until the + ## user deletes that directory. + ## + ## The whole proc is optional and will ignore all kinds of errors. The only + ## way to be sure that it works is to call ``tmutil isexcluded path``. + if alreadyExcludedDirs.contains(dir): return + alreadyExcludedDirs.incl(dir) + try: + var p = startProcess("/usr/bin/tmutil", args = ["addexclusion", dir]) + discard p.waitForExit + p.close + except E_Base, EOS: + discard + proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string = var (head, tail) = splitPath(f) #if len(head) > 0: head = removeTrailingDirSep(shortenDir(head & dirSep)) @@ -270,6 +293,8 @@ proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string = if createSubDir: try: createDir(subdir) + when noTimeMachine: + excludeDirFromTimeMachine(subdir) except EOS: writeln(stdout, "cannot create directory: " & subdir) quit(1) diff --git a/compiler/parser.nim b/compiler/parser.nim index c68c80b46..5c7b86240 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -967,7 +967,7 @@ proc isExprStart(p: TParser): bool = of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkProc, tkIterator, tkBind, tkAddr, tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr, - tkTuple, tkObject, tkType, tkWhen, tkCase, tkShared: + tkTuple, tkObject, tkType, tkWhen, tkCase: result = true else: result = false @@ -1040,7 +1040,6 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode = of tkVar: result = parseTypeDescKAux(p, nkVarTy, mode) of tkRef: result = parseTypeDescKAux(p, nkRefTy, mode) of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, mode) - of tkShared: result = parseTypeDescKAux(p, nkSharedTy, mode) of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, mode) of tkType: result = parseTypeDescKAux(p, nkTypeOfExpr, mode) of tkTuple: result = parseTuple(p, mode == pmTypeDef) diff --git a/compiler/passes.nim b/compiler/passes.nim index 3dc31e7ac..66a1a4954 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -13,7 +13,7 @@ import strutils, lists, options, ast, astalgo, llstream, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math, magicsys, nversion, - nimsets, syntaxes, times, rodread, semthreads, idgen + nimsets, syntaxes, times, rodread, idgen type TPassContext* = object of TObject # the pass's context @@ -74,7 +74,8 @@ proc astNeeded*(s: PSym): bool = ({sfCompilerProc, sfCompileTime} * s.flags == {}) and (s.typ.callConv != ccInline) and (s.ast.sons[genericParamsPos].kind == nkEmpty): - result = semthreads.needsGlobalAnalysis() + result = false + # XXX this doesn't really make sense with excessive CTFE else: result = true diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 14d155539..db9fe7cbe 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2014 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -24,7 +24,7 @@ const wCompilerproc, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge, wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC, wAsmNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl, - wGensym, wInject, wRaises, wTags, wOperator, wDelegator} + wGensym, wInject, wRaises, wTags, wUses, wOperator, wDelegator, wGcSafe} converterPragmas* = procPragmas methodPragmas* = procPragmas templatePragmas* = {wImmediate, wDeprecated, wError, wGensym, wInject, wDirty, @@ -35,7 +35,7 @@ const iteratorPragmas* = {FirstCallConv..LastCallConv, wNosideeffect, wSideeffect, wImportc, wExportc, wNodecl, wMagic, wDeprecated, wBorrow, wExtern, wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wRaises, - wTags, wOperator} + wTags, wUses, wOperator, wGcSafe} exprPragmas* = {wLine} stmtPragmas* = {wChecks, wObjChecks, wFieldChecks, wRangechecks, wBoundchecks, wOverflowchecks, wNilchecks, wAssertions, wWarnings, wHints, @@ -48,12 +48,12 @@ const lambdaPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl, wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader, wDeprecated, wExtern, wThread, wImportCpp, wImportObjC, wAsmNoStackFrame, - wRaises, wTags} + wRaises, wUses, wTags, wGcSafe} typePragmas* = {wImportc, wExportc, wDeprecated, wMagic, wAcyclic, wNodecl, wPure, wHeader, wCompilerproc, wFinal, wSize, wExtern, wShallow, wImportCpp, wImportObjC, wError, wIncompleteStruct, wByCopy, wByRef, wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked, - wBorrow} + wBorrow, wGcSafe} fieldPragmas* = {wImportc, wExportc, wDeprecated, wExtern, wImportCpp, wImportObjC, wError} varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl, @@ -64,7 +64,7 @@ const wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject} letPragmas* = varPragmas procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNosideeffect, - wThread, wRaises, wTags} + wThread, wRaises, wUses, wTags, wGcSafe} allRoutinePragmas* = procPragmas + iteratorPragmas + lambdaPragmas proc pragma*(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords) @@ -513,6 +513,27 @@ proc pragmaRaisesOrTags(c: PContext, n: PNode) = else: invalidPragma(n) +proc pragmaUses(c: PContext, n: PNode) = + proc processExc(c: PContext, x: PNode): PNode = + if x.kind in {nkAccQuoted, nkIdent, nkSym, + nkOpenSymChoice, nkClosedSymChoice}: + if considerAcc(x).s == "*": + return newSymNode(ast.anyGlobal) + result = c.semExpr(c, x) + if result.kind != nkSym or sfGlobal notin result.sym.flags: + localError(x.info, "'$1' is not a global variable" % result.renderTree) + result = newSymNode(ast.anyGlobal) + + if n.kind == nkExprColonExpr: + let it = n.sons[1] + if it.kind notin {nkCurly, nkBracket}: + n.sons[1] = processExc(c, it) + else: + for i in 0 .. <it.len: + it.sons[i] = processExc(c, it.sons[i]) + else: + invalidPragma(n) + proc typeBorrow(sym: PSym, n: PNode) = if n.kind == nkExprColonExpr: let it = n.sons[1] @@ -667,6 +688,12 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, incl(sym.flags, sfThread) incl(sym.flags, sfProcvar) if sym.typ != nil: incl(sym.typ.flags, tfThread) + of wGcSafe: + if optThreadAnalysis in gGlobalOptions: + noVal(it) + if sym.kind != skType: incl(sym.flags, sfThread) + if sym.typ != nil: incl(sym.typ.flags, tfGcSafe) + else: invalidPragma(it) of wPacked: noVal(it) if sym.typ == nil: invalidPragma(it) @@ -759,6 +786,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, if sym == nil: invalidPragma(it) of wLine: pragmaLine(c, it) of wRaises, wTags: pragmaRaisesOrTags(c, it) + of wUses: pragmaUses(c, it) of wOperator: if sym == nil: invalidPragma(it) else: sym.position = expectIntLit(c, it) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index fa119eba9..6b62c48c5 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1082,12 +1082,6 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = gsub(g, n.sons[1]) else: put(g, tkIterator, "iterator") - of nkSharedTy: - if sonsLen(n) > 0: - putWithSpace(g, tkShared, "shared") - gsub(g, n.sons[0]) - else: - put(g, tkShared, "shared") of nkStaticTy: put(g, tkStatic, "static") put(g, tkBracketLe, "[") diff --git a/compiler/sem.nim b/compiler/sem.nim index e7bff0665..7d129caf4 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -14,7 +14,7 @@ import wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math, magicsys, parser, nversion, nimsets, semfold, importer, procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch, - semthreads, intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting, + intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting, evaltempl, patterns, parampatterns, sempass2, pretty, semmacrosanity # implementation @@ -415,12 +415,7 @@ proc myProcess(context: PPassContext, n: PNode): PNode = if getCurrentException() of ESuggestDone: result = nil else: result = ast.emptyNode #if gCmd == cmdIdeTools: findSuggest(c, n) - -proc checkThreads(c: PContext) = - if not needsGlobalAnalysis(): return - for i in 0 .. c.threadEntries.len-1: - semthreads.analyseThreadProc(c.threadEntries[i]) - + proc myClose(context: PPassContext, n: PNode): PNode = var c = PContext(context) closeScope(c) # close module's scope @@ -431,7 +426,6 @@ proc myClose(context: PPassContext, n: PNode): PNode = addCodeForGenerics(c, result) if c.module.ast != nil: result.add(c.module.ast) - checkThreads(c) popOwner() popProcCon(c) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 4cb5ad38c..987a70a41 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -57,7 +57,6 @@ type # can access private object fields instCounter*: int # to prevent endless instantiations - threadEntries*: TSymSeq # list of thread entries to check ambiguousSymbols*: TIntSet # ids of all ambiguous symbols (cannot # store this info in the syms themselves!) inTypeClass*: int # > 0 if we are in a user-defined type class @@ -170,7 +169,6 @@ proc newContext(module: PSym): PContext = append(result.optionStack, newOptionEntry()) result.module = module result.friendModule = module - result.threadEntries = @[] result.converters = @[] result.patterns = @[] result.includedFiles = initIntSet() diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 6c1721bdd..652e1dd4b 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -30,6 +30,8 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = if result.typ != nil: # XXX tyGenericInst here? if result.typ.kind == tyVar: result = newDeref(result) + elif efWantStmt in flags: + result.typ = newTypeS(tyEmpty, c) else: localError(n.info, errExprXHasNoType, renderTree(result, {renderNoComments})) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index caaab2291..79abfaf4d 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -404,7 +404,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode = mExit, mInc, ast.mDec, mEcho, mSwap, mAppendStrCh, mAppendStrStr, mAppendSeqElem, mSetLengthStr, mSetLengthSeq, mParseExprToAst, mParseStmtToAst, mExpandToAst, mTypeTrait, - mNLen..mNError, mEqRef, mSlurp, mStaticExec, mNGenSym: + mNLen..mNError, mEqRef, mSlurp, mStaticExec, mNGenSym, mSpawn: discard of mRand: result = newIntNodeT(math.random(a.getInt.int), n) diff --git a/compiler/seminst.nim b/compiler/seminst.nim index a5149a842..f7d5fa6f8 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -127,9 +127,6 @@ proc sideEffectsCheck(c: PContext, s: PSym) = if {sfNoSideEffect, sfSideEffect} * s.flags == {sfNoSideEffect, sfSideEffect}: localError(s.info, errXhasSideEffects, s.name.s) - elif sfThread in s.flags and semthreads.needsGlobalAnalysis() and - s.ast.sons[genericParamsPos].kind == nkEmpty: - c.threadEntries.add(s) proc instGenericContainer(c: PContext, info: TLineInfo, header: PType, allowMetaTypes = false): PType = diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index a00325277..6235fb76a 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -59,15 +59,19 @@ discard """ --> we need a stack of scopes for this analysis """ +const trackGlobals = false ## we don't need it for now + type TEffects = object exc: PNode # stack of exceptions tags: PNode # list of tags + uses: PNode # list of used global variables bottom: int owner: PSym init: seq[int] # list of initialized variables guards: TModel # nested guards locked: seq[PNode] # locked locations + gcUnsafe, isRecursive: bool PEffects = var TEffects proc isLocalVar(a: PEffects, s: PSym): bool = @@ -89,20 +93,30 @@ proc initVarViaNew(a: PEffects, n: PNode) = # are initialized: initVar(a, n) +when trackGlobals: + proc addUse(a: PEffects, e: PNode) = + var aa = a.uses + for i in 0 .. <aa.len: + if aa[i].sym.id == e.sym.id: return + a.uses.add(e) + proc useVar(a: PEffects, n: PNode) = let s = n.sym if isLocalVar(a, s): if s.id notin a.init: if {tfNeedsInit, tfNotNil} * s.typ.flags != {}: - when true: - message(n.info, warnProveInit, s.name.s) - else: - Message(n.info, errGenerated, - "'$1' might not have been initialized" % s.name.s) + message(n.info, warnProveInit, s.name.s) else: message(n.info, warnUninit, s.name.s) # prevent superfluous warnings about the same variable: a.init.add s.id + if {sfGlobal, sfThread} * s.flags == {sfGlobal} and s.kind == skVar: + when trackGlobals: + a.addUse(copyNode(n)) + if (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem) and + tfGcSafe notin s.typ.flags: + message(n.info, warnGcUnsafe, renderTree(n)) + a.gcUnsafe = true type TIntersection = seq[tuple[id, count: int]] # a simple count table @@ -132,6 +146,10 @@ proc createTag(n: PNode): PNode = result.typ = sysTypeFromName"TEffect" if not n.isNil: result.info = n.info +proc createAnyGlobal(n: PNode): PNode = + result = newSymNode(anyGlobal) + result.info = n.info + proc addEffect(a: PEffects, e: PNode, useLineInfo=true) = assert e.kind != nkRaiseStmt var aa = a.exc @@ -161,9 +179,17 @@ proc mergeTags(a: PEffects, b, comesFrom: PNode) = else: for effect in items(b): addTag(a, effect, useLineInfo=comesFrom != nil) +when trackGlobals: + proc mergeUses(a: PEffects, b, comesFrom: PNode) = + if b.isNil: + addUse(a, createAnyGlobal(comesFrom)) + else: + for effect in items(b): addUse(a, effect) + proc listEffects(a: PEffects) = for e in items(a.exc): message(e.info, hintUser, typeToString(e.typ)) for e in items(a.tags): message(e.info, hintUser, typeToString(e.typ)) + for e in items(a.uses): message(e.info, hintUser, e.sym.name.s) proc catches(tracked: PEffects, e: PType) = let e = skipTypes(e, skipPtrs) @@ -289,6 +315,13 @@ proc documentRaises*(n: PNode) = if n.sons[namePos].kind != nkSym: return documentEffect(n, n.sons[pragmasPos], wRaises, exceptionEffects) documentEffect(n, n.sons[pragmasPos], wTags, tagEffects) + documentEffect(n, n.sons[pragmasPos], wUses, usesEffects) + +template notGcSafe(t): expr = {tfGcSafe, tfNoSideEffect} * t.flags == {} + +proc importedFromC(n: PNode): bool = + # when imported from C, we assume GC-safety. + result = n.kind == nkSym and sfImportc in n.sym.flags proc propagateEffects(tracked: PEffects, n: PNode, s: PSym) = let pragma = s.ast.sons[pragmasPos] @@ -298,6 +331,14 @@ proc propagateEffects(tracked: PEffects, n: PNode, s: PSym) = let tagSpec = effectSpec(pragma, wTags) mergeTags(tracked, tagSpec, n) + if notGcSafe(s.typ) and sfImportc notin s.flags: + message(n.info, warnGcUnsafe, renderTree(n)) + tracked.gcUnsafe = true + + when trackGlobals: + let usesSpec = effectSpec(pragma, wUses) + mergeUses(tracked, usesSpec, n) + proc notNilCheck(tracked: PEffects, n: PNode, paramType: PType) = let n = n.skipConv if paramType != nil and tfNotNil in paramType.flags and @@ -330,9 +371,18 @@ proc trackOperand(tracked: PEffects, n: PNode, paramType: PType) = else: addEffect(tracked, createRaise(n)) addTag(tracked, createTag(n)) + when trackGlobals: addUse(tracked, createAnyGlobal(n)) + # assume GcUnsafe unless in its type: + if notGcSafe(op): + message(n.info, warnGcUnsafe, renderTree(n)) + tracked.gcUnsafe = true else: mergeEffects(tracked, effectList.sons[exceptionEffects], n) mergeTags(tracked, effectList.sons[tagEffects], n) + when trackGlobals: mergeUses(tracked, effectList.sons[usesEffects], n) + if notGcSafe(op): + message(n.info, warnGcUnsafe, renderTree(n)) + tracked.gcUnsafe = true notNilCheck(tracked, n, paramType) proc breaksBlock(n: PNode): bool = @@ -451,8 +501,13 @@ proc track(tracked: PEffects, n: PNode) = # XXX: in rare situations, templates and macros will reach here after # calling getAst(templateOrMacro()). Currently, templates and macros # are indistinguishable from normal procs (both have tyProc type) and - # we can detect them only by cheking for attached nkEffectList. + # we can detect them only by checking for attached nkEffectList. if op != nil and op.kind == tyProc and op.n.sons[0].kind == nkEffectList: + if a.kind == nkSym and a.sym == tracked.owner: + tracked.isRecursive = true + elif notGcSafe(op) and not importedFromC(a): + message(n.info, warnGcUnsafe, renderTree(n)) + tracked.gcUnsafe = true var effectList = op.n.sons[0] if a.kind == nkSym and a.sym.kind == skMethod: propagateEffects(tracked, n, a.sym) @@ -462,9 +517,12 @@ proc track(tracked: PEffects, n: PNode) = elif isIndirectCall(a, tracked.owner): addEffect(tracked, createRaise(n)) addTag(tracked, createTag(n)) + when trackGlobals: addUse(tracked, createAnyGlobal(n)) + # XXX handle 'gcsafe' properly for callbacks! else: mergeEffects(tracked, effectList.sons[exceptionEffects], n) mergeTags(tracked, effectList.sons[tagEffects], n) + when trackGlobals: mergeUses(tracked, effectList.sons[usesEffects], n) for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i)) if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, mNewSeq}: # may not look like an assignment, but it is: @@ -531,14 +589,21 @@ proc track(tracked: PEffects, n: PNode) = else: for i in 0 .. <safeLen(n): track(tracked, n.sons[i]) -proc checkRaisesSpec(spec, real: PNode, msg: string, hints: bool) = +proc subtypeRelation(spec, real: PNode): bool = + result = safeInheritanceDiff(real.excType, spec.typ) <= 0 + +proc symbolPredicate(spec, real: PNode): bool = + result = real.sym.id == spec.sym.id + +proc checkRaisesSpec(spec, real: PNode, msg: string, hints: bool; + effectPredicate: proc (a, b: PNode): bool {.nimcall.}) = # check that any real exception is listed in 'spec'; mark those as used; # report any unused exception var used = initIntSet() for r in items(real): block search: for s in 0 .. <spec.len: - if safeInheritanceDiff(r.excType, spec[s].typ) <= 0: + if effectPredicate(spec[s], r): used.incl(s) break search # XXX call graph analysis would be nice here! @@ -560,11 +625,18 @@ proc checkMethodEffects*(disp, branch: PSym) = let raisesSpec = effectSpec(p, wRaises) if not isNil(raisesSpec): checkRaisesSpec(raisesSpec, actual.sons[exceptionEffects], - "can raise an unlisted exception: ", hints=off) + "can raise an unlisted exception: ", hints=off, subtypeRelation) let tagsSpec = effectSpec(p, wTags) if not isNil(tagsSpec): checkRaisesSpec(tagsSpec, actual.sons[tagEffects], - "can have an unlisted effect: ", hints=off) + "can have an unlisted effect: ", hints=off, subtypeRelation) + let usesSpec = effectSpec(p, wUses) + if not isNil(usesSpec): + checkRaisesSpec(usesSpec, actual.sons[usesEffects], + "may use an unlisted global variable: ", hints=off, symbolPredicate) + if sfThread in disp.flags and notGcSafe(branch.typ): + localError(branch.info, "base method is GC-safe, but '$1' is not" % + branch.name.s) proc setEffectsForProcType*(t: PType, n: PNode) = var effects = t.n.sons[0] @@ -573,21 +645,26 @@ proc setEffectsForProcType*(t: PType, n: PNode) = let raisesSpec = effectSpec(n, wRaises) tagsSpec = effectSpec(n, wTags) - if not isNil(raisesSpec) or not isNil(tagsSpec): + usesSpec = effectSpec(n, wUses) + if not isNil(raisesSpec) or not isNil(tagsSpec) or not isNil(usesSpec): internalAssert effects.len == 0 newSeq(effects.sons, effectListLen) if not isNil(raisesSpec): effects.sons[exceptionEffects] = raisesSpec if not isNil(tagsSpec): effects.sons[tagEffects] = tagsSpec + if not isNil(usesSpec): + effects.sons[usesEffects] = usesSpec proc initEffects(effects: PNode; s: PSym; t: var TEffects) = newSeq(effects.sons, effectListLen) effects.sons[exceptionEffects] = newNodeI(nkArgList, s.info) effects.sons[tagEffects] = newNodeI(nkArgList, s.info) + effects.sons[usesEffects] = newNodeI(nkArgList, s.info) t.exc = effects.sons[exceptionEffects] t.tags = effects.sons[tagEffects] + t.uses = effects.sons[usesEffects] t.owner = s t.init = @[] t.guards = @[] @@ -602,7 +679,6 @@ proc trackProc*(s: PSym, body: PNode) = var t: TEffects initEffects(effects, s, t) track(t, body) - if not isEmptyType(s.typ.sons[0]) and tfNeedsInit in s.typ.sons[0].flags and s.kind in {skProc, skConverter, skMethod}: var res = s.ast.sons[resultPos].sym # get result symbol @@ -612,17 +688,29 @@ proc trackProc*(s: PSym, body: PNode) = let raisesSpec = effectSpec(p, wRaises) if not isNil(raisesSpec): checkRaisesSpec(raisesSpec, t.exc, "can raise an unlisted exception: ", - hints=on) + hints=on, subtypeRelation) # after the check, use the formal spec: effects.sons[exceptionEffects] = raisesSpec let tagsSpec = effectSpec(p, wTags) if not isNil(tagsSpec): checkRaisesSpec(tagsSpec, t.tags, "can have an unlisted effect: ", - hints=off) + hints=off, subtypeRelation) # after the check, use the formal spec: effects.sons[tagEffects] = tagsSpec - + + when trackGlobals: + let usesSpec = effectSpec(p, wUses) + if not isNil(usesSpec): + checkRaisesSpec(usesSpec, t.uses, + "uses an unlisted global variable: ", hints=on, symbolPredicate) + effects.sons[usesEffects] = usesSpec + if optThreadAnalysis in gGlobalOptions: + if sfThread in s.flags and t.gcUnsafe: + localError(s.info, warnGcUnsafe2, s.name.s) + #localError(s.info, "'$1' is not GC-safe" % s.name.s) + if not t.gcUnsafe: s.typ.flags.incl tfGcSafe + proc trackTopLevelStmt*(module: PSym; n: PNode) = if n.kind in {nkPragma, nkMacroDef, nkTemplateDef, nkProcDef, nkTypeSection, nkConverterDef, nkMethodDef, nkIteratorDef}: diff --git a/compiler/semthreads.nim b/compiler/semthreads.nim deleted file mode 100644 index d3426ca3e..000000000 --- a/compiler/semthreads.nim +++ /dev/null @@ -1,390 +0,0 @@ -# -# -# The Nimrod Compiler -# (c) Copyright 2013 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Semantic analysis that deals with threads: Possible race conditions should -## be reported some day. -## -## -## ======================== -## No heap sharing analysis -## ======================== -## -## The only crucial operation that can violate the heap invariants is the -## write access. The analysis needs to distinguish between 'unknown', 'mine', -## and 'theirs' memory and pointers. Assignments 'whatever <- unknown' are -## invalid, and so are 'theirs <- whatever' but not 'mine <- theirs'. Since -## strings and sequences are heap allocated they are affected too: -## -## .. code-block:: nimrod -## proc p() = -## global = "alloc this string" # ugh! -## -## Thus the analysis is concerned with any type that contains a GC'ed -## reference... -## If the type system would distinguish between 'ref' and '!ref' and threads -## could not have '!ref' as input parameters the analysis could simply need to -## reject any write access to a global variable which contains GC'ed data. -## Thanks to the write barrier of the GC, this is exactly what needs to be -## done! Every write access to a global that contains GC'ed data needs to -## be prevented! Unfortunately '!ref' is not implemented yet... -## -## The assignment target is essential for the algorithm: only -## write access to heap locations and global variables are critical and need -## to be checked. Access via 'var' parameters is no problem to analyse since -## we need the arguments' locations in the analysis. -## -## However, this is tricky: -## -## var x = globalVar # 'x' points to 'theirs' -## while true: -## globalVar = x # NOT OK: 'theirs <- theirs' invalid due to -## # write barrier! -## x = "new string" # ugh: 'x is toUnknown'! -## -## --> Solution: toUnknown is never allowed anywhere! -## -## -## Beware that the same proc might need to be -## analysed multiple times! Oh and watch out for recursion! Recursion is handled -## by a stack of symbols that we are processing, if we come back to the same -## symbol, we have to skip this check (assume no error in the recursive case). -## However this is wrong. We need to check for the particular combination -## of (procsym, threadOwner(arg1), threadOwner(arg2), ...)! - -import - ast, astalgo, strutils, hashes, options, msgs, idents, types, os, - renderer, tables, rodread - -type - TThreadOwner = enum - toUndefined, # not computed yet - toVoid, # no return type - toNil, # cycle in computation or nil: can be overwritten - toTheirs, # some other heap - toMine # mine heap - - TCall = object {.pure.} - callee: PSym # what if callee is an indirect call? - args: seq[TThreadOwner] - - PProcCtx = ref TProcCtx - TProcCtx = object {.pure.} - nxt: PProcCtx # can be stacked - mapping: tables.TTable[int, TThreadOwner] # int = symbol ID - owner: PSym # current owner - -var - computed = tables.initTable[TCall, TThreadOwner]() - -proc hash(c: TCall): THash = - result = hash(c.callee.id) - for a in items(c.args): result = result !& hash(ord(a)) - result = !$result - -proc `==`(a, b: TCall): bool = - if a.callee != b.callee: return - if a.args.len != b.args.len: return - for i in 0..a.args.len-1: - if a.args[i] != b.args[i]: return - result = true - -proc newProcCtx(owner: PSym): PProcCtx = - assert owner != nil - new(result) - result.mapping = tables.initTable[int, TThreadOwner]() - result.owner = owner - -proc analyse(c: PProcCtx, n: PNode): TThreadOwner - -proc analyseSym(c: PProcCtx, n: PNode): TThreadOwner = - var v = n.sym - result = c.mapping[v.id] - if result != toUndefined: return - case v.kind - of skVar, skForVar, skLet, skResult: - result = toNil - if sfGlobal in v.flags: - if sfThread in v.flags: - result = toMine - elif containsGarbageCollectedRef(v.typ): - result = toTheirs - of skTemp: result = toNil - of skConst: result = toMine - of skParam: - result = c.mapping[v.id] - if result == toUndefined: - internalError(n.info, "param not set: " & v.name.s) - else: - result = toNil - c.mapping[v.id] = result - -proc lvalueSym(n: PNode): PNode = - result = n - while result.kind in {nkDotExpr, nkCheckedFieldExpr, - nkBracketExpr, nkDerefExpr, nkHiddenDeref}: - result = result.sons[0] - -proc writeAccess(c: PProcCtx, n: PNode, owner: TThreadOwner) = - if owner notin {toNil, toMine, toTheirs}: - internalError(n.info, "writeAccess: " & $owner) - var a = lvalueSym(n) - if a.kind == nkSym: - var v = a.sym - var lastOwner = analyseSym(c, a) - case lastOwner - of toNil: - # fine, toNil can be overwritten - var newOwner: TThreadOwner - if sfGlobal in v.flags: - newOwner = owner - elif containsTyRef(v.typ): - # ``var local = gNode`` --> ok, but ``local`` is theirs! - newOwner = owner - else: - # ``var local = gString`` --> string copy: ``local`` is mine! - newOwner = toMine - # XXX BUG what if the tuple contains both ``tyRef`` and ``tyString``? - c.mapping[v.id] = newOwner - of toVoid, toUndefined: internalError(n.info, "writeAccess") - of toTheirs: message(n.info, warnWriteToForeignHeap) - of toMine: - if lastOwner != owner and owner != toNil: - message(n.info, warnDifferentHeaps) - else: - # we could not backtrack to a concrete symbol, but that's fine: - var lastOwner = analyse(c, n) - case lastOwner - of toNil: discard # fine, toNil can be overwritten - of toVoid, toUndefined: internalError(n.info, "writeAccess") - of toTheirs: message(n.info, warnWriteToForeignHeap) - of toMine: - if lastOwner != owner and owner != toNil: - message(n.info, warnDifferentHeaps) - -proc analyseAssign(c: PProcCtx, le, ri: PNode) = - var y = analyse(c, ri) # read access; ok - writeAccess(c, le, y) - -proc analyseAssign(c: PProcCtx, n: PNode) = - analyseAssign(c, n.sons[0], n.sons[1]) - -proc analyseCall(c: PProcCtx, n: PNode): TThreadOwner = - var prc = n[0].sym - var newCtx = newProcCtx(prc) - var call: TCall - call.callee = prc - newSeq(call.args, n.len-1) - for i in 1..n.len-1: - call.args[i-1] = analyse(c, n[i]) - if not computed.hasKey(call): - computed[call] = toUndefined # we are computing it - let prctyp = skipTypes(prc.typ, abstractInst).n - for i in 1.. prctyp.len-1: - var formal = prctyp.sons[i].sym - newCtx.mapping[formal.id] = call.args[i-1] - pushInfoContext(n.info) - result = analyse(newCtx, prc.getBody) - if prc.ast.sons[bodyPos].kind == nkEmpty and - {sfNoSideEffect, sfThread, sfImportc} * prc.flags == {}: - message(n.info, warnAnalysisLoophole, renderTree(n)) - if result == toUndefined: result = toNil - if prc.typ.sons[0] != nil: - if prc.ast.len > resultPos: - result = newCtx.mapping[prc.ast.sons[resultPos].sym.id] - # if the proc body does not set 'result', nor 'return's something - # explicitely, it returns a binary zero, so 'toNil' is correct: - if result == toUndefined: result = toNil - else: - result = toNil - else: - result = toVoid - computed[call] = result - popInfoContext() - else: - result = computed[call] - if result == toUndefined: - # ugh, cycle! We are already computing it but don't know the - # outcome yet... - if prc.typ.sons[0] == nil: result = toVoid - else: result = toNil - -proc analyseVarTuple(c: PProcCtx, n: PNode) = - if n.kind != nkVarTuple: internalError(n.info, "analyseVarTuple") - var L = n.len - for i in countup(0, L-3): analyseAssign(c, n.sons[i], n.sons[L-1]) - -proc analyseSingleVar(c: PProcCtx, a: PNode) = - if a.sons[2].kind != nkEmpty: analyseAssign(c, a.sons[0], a.sons[2]) - -proc analyseVarSection(c: PProcCtx, n: PNode): TThreadOwner = - for i in countup(0, sonsLen(n) - 1): - var a = n.sons[i] - if a.kind == nkCommentStmt: continue - if a.kind == nkIdentDefs: - #assert(a.sons[0].kind == nkSym); also valid for after - # closure transformation: - analyseSingleVar(c, a) - else: - analyseVarTuple(c, a) - result = toVoid - -proc analyseConstSection(c: PProcCtx, t: PNode): TThreadOwner = - for i in countup(0, sonsLen(t) - 1): - var it = t.sons[i] - if it.kind == nkCommentStmt: continue - if it.kind != nkConstDef: internalError(t.info, "analyseConstSection") - if sfFakeConst in it.sons[0].sym.flags: analyseSingleVar(c, it) - result = toVoid - -template aggregateOwner(result, ana: expr) = - var a = ana # eval once - if result != a: - if result == toNil: result = a - elif a != toNil: message(n.info, warnDifferentHeaps) - -proc analyseArgs(c: PProcCtx, n: PNode, start = 1) = - for i in start..n.len-1: discard analyse(c, n[i]) - -proc analyseOp(c: PProcCtx, n: PNode): TThreadOwner = - if n[0].kind != nkSym or n[0].sym.kind != skProc: - if {tfNoSideEffect, tfThread} * n[0].typ.flags == {}: - message(n.info, warnAnalysisLoophole, renderTree(n)) - result = toNil - else: - var prc = n[0].sym - case prc.magic - of mNone: - if sfSystemModule in prc.owner.flags: - # System module proc does no harm :-) - analyseArgs(c, n) - if prc.typ.sons[0] == nil: result = toVoid - else: result = toNil - else: - result = analyseCall(c, n) - of mNew, mNewFinalize, mNewSeq, mSetLengthStr, mSetLengthSeq, - mAppendSeqElem, mReset, mAppendStrCh, mAppendStrStr: - writeAccess(c, n[1], toMine) - result = toVoid - of mSwap: - var a = analyse(c, n[2]) - writeAccess(c, n[1], a) - writeAccess(c, n[2], a) - result = toVoid - of mIntToStr, mInt64ToStr, mFloatToStr, mBoolToStr, mCharToStr, - mCStrToStr, mStrToStr, mEnumToStr, - mConStrStr, mConArrArr, mConArrT, - mConTArr, mConTT, mSlice, - mRepr, mArrToSeq, mCopyStr, mCopyStrLast, - mNewString, mNewStringOfCap: - analyseArgs(c, n) - result = toMine - else: - # don't recurse, but check args: - analyseArgs(c, n) - if prc.typ.sons[0] == nil: result = toVoid - else: result = toNil - -proc analyse(c: PProcCtx, n: PNode): TThreadOwner = - case n.kind - of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, - nkCallStrLit, nkHiddenCallConv: - result = analyseOp(c, n) - of nkAsgn, nkFastAsgn: - analyseAssign(c, n) - result = toVoid - of nkSym: result = analyseSym(c, n) - of nkEmpty, nkNone: result = toVoid - of nkNilLit, nkCharLit..nkFloat64Lit: result = toNil - of nkStrLit..nkTripleStrLit: result = toMine - of nkDotExpr, nkBracketExpr, nkDerefExpr, nkHiddenDeref: - # field access: - # pointer deref or array access: - result = analyse(c, n.sons[0]) - of nkBind: result = analyse(c, n.sons[0]) - of nkPar, nkCurly, nkBracket, nkRange: - # container construction: - result = toNil # nothing until later - for i in 0..n.len-1: aggregateOwner(result, analyse(c, n[i])) - of nkObjConstr: - if n.typ != nil and containsGarbageCollectedRef(n.typ): - result = toMine - else: - result = toNil # nothing until later - for i in 1..n.len-1: aggregateOwner(result, analyse(c, n[i])) - of nkAddr, nkHiddenAddr: - var a = lvalueSym(n) - if a.kind == nkSym: - result = analyseSym(c, a) - assert result in {toNil, toMine, toTheirs} - if result == toNil: - # assume toMine here for consistency: - c.mapping[a.sym.id] = toMine - result = toMine - else: - # should never really happen: - result = analyse(c, n.sons[0]) - of nkIfExpr: - result = toNil - for i in countup(0, sonsLen(n) - 1): - var it = n.sons[i] - if it.len == 2: - discard analyse(c, it.sons[0]) - aggregateOwner(result, analyse(c, it.sons[1])) - else: - aggregateOwner(result, analyse(c, it.sons[0])) - of nkStmtListExpr, nkBlockExpr: - var n = if n.kind == nkBlockExpr: n.sons[1] else: n - var L = sonsLen(n) - for i in countup(0, L-2): discard analyse(c, n.sons[i]) - if L > 0: result = analyse(c, n.sons[L-1]) - else: result = toVoid - of nkHiddenStdConv, nkHiddenSubConv, nkConv, nkCast: - result = analyse(c, n.sons[1]) - of nkStringToCString, nkCStringToString, nkChckRangeF, nkChckRange64, - nkChckRange, nkCheckedFieldExpr, nkObjDownConv, - nkObjUpConv: - result = analyse(c, n.sons[0]) - of nkRaiseStmt: - var a = analyse(c, n.sons[0]) - if a != toMine: message(n.info, warnDifferentHeaps) - result = toVoid - of nkVarSection, nkLetSection: result = analyseVarSection(c, n) - of nkConstSection: result = analyseConstSection(c, n) - of nkTypeSection, nkCommentStmt: result = toVoid - of nkIfStmt, nkWhileStmt, nkTryStmt, nkCaseStmt, nkStmtList, nkBlockStmt, - nkElifBranch, nkElse, nkExceptBranch, nkOfBranch, nkFinally: - for i in 0 .. <n.len: discard analyse(c, n[i]) - result = toVoid - of nkBreakStmt, nkContinueStmt: result = toVoid - of nkReturnStmt, nkDiscardStmt: - if n.sons[0].kind != nkEmpty: result = analyse(c, n.sons[0]) - else: result = toVoid - of nkLambdaKinds, nkClosure: - result = toMine - of nkAsmStmt, nkPragma, nkIteratorDef, nkProcDef, nkMethodDef, - nkConverterDef, nkMacroDef, nkTemplateDef, - nkGotoState, nkState, nkBreakState, nkType, nkIdent: - result = toVoid - of nkExprColonExpr: - result = analyse(c, n.sons[1]) - else: internalError(n.info, "analysis not implemented for: " & $n.kind) - -proc analyseThreadProc*(prc: PSym) = - var c = newProcCtx(prc) - var formals = skipTypes(prc.typ, abstractInst).n - for i in 1 .. formals.len-1: - var formal = formals.sons[i].sym - # the input is copied and belongs to the thread: - c.mapping[formal.id] = toMine - discard analyse(c, prc.getBody) - -proc needsGlobalAnalysis*: bool = - result = gGlobalOptions * {optThreads, optThreadAnalysis} == - {optThreads, optThreadAnalysis} - diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 5e391cc93..384bdc8a3 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1277,6 +1277,15 @@ proc processMagicType(c: PContext, m: PSym) = setMagicType(m, tyOrdinal, 0) rawAddSon(m.typ, newTypeS(tyNone, c)) of mPNimrodNode: discard + of mShared: + setMagicType(m, tyObject, 0) + m.typ.n = newNodeI(nkRecList, m.info) + incl m.typ.flags, tfShared + of mGuarded: + setMagicType(m, tyObject, 0) + m.typ.n = newNodeI(nkRecList, m.info) + incl m.typ.flags, tfShared + rawAddSon(m.typ, sysTypeFromName"shared") else: localError(m.info, errTypeExpected) proc semGenericConstraints(c: PContext, x: PType): PType = diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 4a8a463f5..271a01266 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2014 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -14,11 +14,22 @@ import ast, astalgo, msgs, types, magicsys, semdata, renderer const tfInstClearedFlags = {tfHasMeta} +proc sharedPtrCheck(info: TLineInfo, t: PType) = + if t.kind == tyPtr and t.len > 1: + if t.sons[0].sym.magic in {mShared, mGuarded}: + incl(t.flags, tfShared) + if t.sons[0].sym.magic == mGuarded: incl(t.flags, tfGuarded) + if tfHasGCedMem in t.flags or t.isGCedMem: + localError(info, errGenerated, + "shared memory may not refer to GC'ed thread local memory") + proc checkPartialConstructedType(info: TLineInfo, t: PType) = if tfAcyclic in t.flags and skipTypes(t, abstractInst).kind != tyObject: localError(info, errInvalidPragmaX, "acyclic") elif t.kind == tyVar and t.sons[0].kind == tyVar: localError(info, errVarVarTypeNotAllowed) + else: + sharedPtrCheck(info, t) proc checkConstructedType*(info: TLineInfo, typ: PType) = var t = typ.skipTypes({tyDistinct}) @@ -29,7 +40,8 @@ proc checkConstructedType*(info: TLineInfo, typ: PType) = localError(info, errVarVarTypeNotAllowed) elif computeSize(t) == szIllegalRecursion: localError(info, errIllegalRecursionInTypeX, typeToString(t)) - + else: + sharedPtrCheck(info, t) when false: if t.kind == tyObject and t.sons[0] != nil: if t.sons[0].kind != tyObject or tfFinal in t.sons[0].flags: diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 9d1585c56..4b91a067e 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -148,6 +148,9 @@ proc sumGeneric(t: PType): int = result = ord(t.kind == tyGenericInvokation) for i in 0 .. <t.len: result += t.sons[i].sumGeneric break + of tyProc: + # proc matche proc better than 'stmt' to disambiguate 'spawn' + return 1 of tyGenericParam, tyExpr, tyStatic, tyStmt, tyTypeDesc: break else: return 0 @@ -402,7 +405,7 @@ proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation = if tfNoSideEffect in f.flags and tfNoSideEffect notin a.flags: return isNone elif tfThread in f.flags and a.flags * {tfThread, tfNoSideEffect} == {}: - # noSideEffect implies ``tfThread``! XXX really? + # noSideEffect implies ``tfThread``! return isNone elif f.flags * {tfIterator} != a.flags * {tfIterator}: return isNone @@ -851,7 +854,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = of tyAnd: considerPreviousT: for branch in f.sons: - if typeRel(c, branch, aOrig) == isNone: + if typeRel(c, branch, aOrig) < isSubtype: return isNone bindingRet isGeneric @@ -859,7 +862,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = of tyOr: considerPreviousT: for branch in f.sons: - if typeRel(c, branch, aOrig) != isNone: + if typeRel(c, branch, aOrig) >= isSubtype: bindingRet isGeneric return isNone @@ -1284,6 +1287,7 @@ proc prepareOperand(c: PContext; formal: PType; a: PNode): PNode = result = a elif a.typ.isNil: let flags = if formal.kind == tyIter: {efDetermineType, efWantIterator} + elif formal.kind == tyStmt: {efDetermineType, efWantStmt} else: {efDetermineType} result = c.semOperand(c, a, flags) else: diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 49611f649..fc6ba2f77 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -253,6 +253,7 @@ proc findUsages(node: PNode, s: PSym) = lastLineInfo = node.info proc findDefinition(node: PNode, s: PSym) = + if node.isNil or s.isNil: return if isTracked(node.info, s.name.s.len): suggestWriteln(symToStr(s, isLocal=false, sectionDef)) suggestQuit() diff --git a/compiler/types.nim b/compiler/types.nim index 1de0fc103..1f266d64f 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -342,9 +342,10 @@ proc canFormAcycleAux(marker: var TIntSet, typ: PType, startId: int): bool = result = t.id == startId # Inheritance can introduce cyclic types, however this is not relevant # as the type that is passed to 'new' is statically known! - #if t.kind == tyObject and tfFinal notin t.flags: - # # damn inheritance may introduce cycles: - # result = true + # er but we use it also for the write barrier ... + if t.kind == tyObject and tfFinal notin t.flags: + # damn inheritance may introduce cycles: + result = true of tyProc: result = typ.callConv == ccClosure else: discard @@ -537,7 +538,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = add(prag, "noSideEffect") if tfThread in t.flags: addSep(prag) - add(prag, "thread") + add(prag, "gcsafe") if len(prag) != 0: add(result, "{." & prag & ".}") of tyVarargs, tyIter: result = typeToStr[t.kind] % typeToString(t.sons[0]) diff --git a/compiler/vm.nim b/compiler/vm.nim index fb8749250..69a410f18 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -354,6 +354,11 @@ template handleJmpBack() {.dirty.} = globalError(c.debug[pc], errTooManyIterations) dec(c.loopIterations) +proc skipColon(n: PNode): PNode = + result = n + if n.kind == nkExprColonExpr: + result = n.sons[1] + proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = var pc = start var tos = tos @@ -454,7 +459,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = decodeBC(rkNode) let src = regs[rb].node if src.kind notin {nkEmpty..nkNilLit}: - let n = src.sons[rc] + let n = src.sons[rc].skipColon regs[ra].node = n else: stackTrace(c, tos, pc, errNilAccess) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 7c0c3d4f5..84577bb22 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -70,6 +70,9 @@ proc echoCode*(c: PCtx, start=0) {.deprecated.} = echo buf proc gABC(ctx: PCtx; n: PNode; opc: TOpcode; a, b, c: TRegister = 0) = + ## Takes the registers `b` and `c`, applies the operation `opc` to them, and + ## stores the result into register `a` + ## The node is needed for debug information assert opc.ord < 255 let ins = (opc.uint32 or (a.uint32 shl 8'u32) or (b.uint32 shl 16'u32) or @@ -78,6 +81,10 @@ proc gABC(ctx: PCtx; n: PNode; opc: TOpcode; a, b, c: TRegister = 0) = ctx.debug.add(n.info) proc gABI(c: PCtx; n: PNode; opc: TOpcode; a, b: TRegister; imm: BiggestInt) = + # Takes the `b` register and the immediate `imm`, appies the operation `opc`, + # and stores the output value into `a`. + # `imm` is signed and must be within [-127, 128] + assert(imm >= -127 and imm <= 128) let ins = (opc.uint32 or (a.uint32 shl 8'u32) or (b.uint32 shl 16'u32) or (imm+byteExcess).uint32 shl 24'u32).TInstr @@ -85,6 +92,9 @@ proc gABI(c: PCtx; n: PNode; opc: TOpcode; a, b: TRegister; imm: BiggestInt) = c.debug.add(n.info) proc gABx(c: PCtx; n: PNode; opc: TOpcode; a: TRegister = 0; bx: int) = + # Applies `opc` to `bx` and stores it into register `a` + # `bx` must be signed and in the range [-32767, 32768] + assert(bx >= -32767 and bx <= 32768) let ins = (opc.uint32 or a.uint32 shl 8'u32 or (bx+wordExcess).uint32 shl 16'u32).TInstr c.code.add(ins) @@ -316,15 +326,7 @@ proc genAndOr(c: PCtx; n: PNode; opc: TOpcode; dest: var TDest) = c.patch(L1) proc canonValue*(n: PNode): PNode = - if n.kind == nkExprColonExpr: - result = n.sons[1] - elif n.hasSubnodeWith(nkExprColonExpr): - result = n.copyNode - newSeq(result.sons, n.len) - for i in 0.. <n.len: - result.sons[i] = canonValue(n.sons[i]) - else: - result = n + result = n proc rawGenLiteral(c: PCtx; n: PNode): int = result = c.constants.len diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 9fdb3bac6..fbe005031 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2014 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -30,7 +30,7 @@ type wInclude, wInterface, wIs, wIsnot, wIterator, wLambda, wLet, wMacro, wMethod, wMixin, wMod, wNil, wNot, wNotin, wObject, wOf, wOr, wOut, wProc, wPtr, wRaise, wRef, wReturn, - wShared, wShl, wShr, wStatic, wTemplate, wTry, wTuple, wType, wUsing, wVar, + wShl, wShr, wStatic, wTemplate, wTry, wTuple, wType, wUsing, wVar, wWhen, wWhile, wWith, wWithout, wXor, wYield, wColon, wColonColon, wEquals, wDot, wDotDot, @@ -44,7 +44,8 @@ type wImportCompilerProc, wImportc, wExportc, wIncompleteStruct, wRequiresInit, wAlign, wNodecl, wPure, wSideeffect, wHeader, - wNosideeffect, wNoreturn, wMerge, wLib, wDynlib, wCompilerproc, wProcVar, + wNosideeffect, wGcSafe, wNoreturn, wMerge, wLib, wDynlib, + wCompilerproc, wProcVar, wFatal, wError, wWarning, wHint, wLine, wPush, wPop, wDefine, wUndef, wLinedir, wStacktrace, wLinetrace, wLink, wCompile, wLinksys, wDeprecated, wVarargs, wCallconv, wBreakpoint, wDebugger, @@ -63,7 +64,7 @@ type wAcyclic, wShallow, wUnroll, wLinearScanEnd, wComputedGoto, wInjectStmt, wWrite, wGensym, wInject, wDirty, wInheritable, wThreadVar, wEmit, wAsmNoStackFrame, - wImplicitStatic, wGlobal, wCodegenDecl, wUnchecked, + wImplicitStatic, wGlobal, wCodegenDecl, wUnchecked, wGuard, wUses, wAuto, wBool, wCatch, wChar, wClass, wConst_cast, wDefault, wDelete, wDouble, wDynamic_cast, @@ -110,7 +111,7 @@ const "macro", "method", "mixin", "mod", "nil", "not", "notin", "object", "of", "or", "out", "proc", "ptr", "raise", "ref", "return", - "shared", "shl", "shr", "static", + "shl", "shr", "static", "template", "try", "tuple", "type", "using", "var", "when", "while", "with", "without", "xor", "yield", @@ -125,7 +126,7 @@ const "importcpp", "importobjc", "importcompilerproc", "importc", "exportc", "incompletestruct", "requiresinit", "align", "nodecl", "pure", "sideeffect", - "header", "nosideeffect", "noreturn", "merge", "lib", "dynlib", + "header", "nosideeffect", "gcsafe", "noreturn", "merge", "lib", "dynlib", "compilerproc", "procvar", "fatal", "error", "warning", "hint", "line", "push", "pop", "define", "undef", "linedir", "stacktrace", "linetrace", "link", "compile", "linksys", "deprecated", "varargs", @@ -146,6 +147,7 @@ const "computedgoto", "injectstmt", "write", "gensym", "inject", "dirty", "inheritable", "threadvar", "emit", "asmnostackframe", "implicitstatic", "global", "codegendecl", "unchecked", + "guard", "uses", "auto", "bool", "catch", "char", "class", "const_cast", "default", "delete", "double", diff --git a/doc/advopt.txt b/doc/advopt.txt index 0ebc85370..f5ff90791 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -4,6 +4,7 @@ Advanced commands: //compileToOC, objc compile project to Objective C code //rst2html convert a reStructuredText file to HTML //rst2tex convert a reStructuredText file to TeX + //jsondoc extract the documentation to a json file //buildIndex build an index for the whole documentation //run run the project (with Tiny C backend; buggy!) //genDepend generate a DOT file containing the diff --git a/doc/keywords.txt b/doc/keywords.txt index 2d18d7969..60b100398 100644 --- a/doc/keywords.txt +++ b/doc/keywords.txt @@ -12,7 +12,7 @@ nil not notin object of or out proc ptr raise ref return -shared shl shr static +shl shr static template try tuple type using var diff --git a/doc/lib.txt b/doc/lib.txt index a209357f7..3ca519c9e 100644 --- a/doc/lib.txt +++ b/doc/lib.txt @@ -373,6 +373,9 @@ Miscellaneous * `logging <logging.html>`_ This module implements a simple logger. +* `future <future.html>`_ + This module implements new experimental features. Currently the syntax + sugar for anonymous procedures. Database support ---------------- diff --git a/doc/manual.txt b/doc/manual.txt index c0d4f2627..d6b9f296e 100644 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -5616,7 +5616,6 @@ Memory allocation requires no lock at all! This design easily scales to massive multicore processors that will become the norm in the future. - Thread pragma ------------- @@ -5626,12 +5625,32 @@ violations of the `no heap sharing restriction`:idx:\: This restriction implies that it is invalid to construct a data structure that consists of memory allocated from different (thread local) heaps. -Since the semantic checking of threads requires whole program analysis, -it is quite expensive and can be turned off with ``--threadanalysis:off`` to -improve compile times. +A thread proc is passed to ``createThread`` or ``spawn`` and invoked +indirectly; so the ``thread`` pragma implies ``procvar``. + + +GC safety +--------- + +We call a proc ``p`` `GC safe`:idx: when it doesn't access any global variable +that contains GC'ed memory (``string``, ``seq``, ``ref`` or a closure) either +directly or indirectly through a call to a GC unsafe proc. + +The `gcsafe`:idx: annotation can be used to mark a proc to be gcsafe +otherwise this property is inferred by the compiler. Note that ``noSideEfect`` +implies ``gcsafe``. The only way to create a thread is via ``spawn`` or +``createThead``. ``spawn`` is usually the preferable method. Either way +the invoked proc must not use ``var`` parameters nor must any of its parameters +contain a ``ref`` or ``closure`` type. This enforces +the *no heap sharing restriction*. + +Routines that are imported from C are always assumed to be ``gcsafe``. + +Future directions: -A thread proc is passed to ``createThread`` and invoked indirectly; so the -``thread`` pragma implies ``procvar``. +- For structured fork&join parallelism more efficient parameter passing can + be performed and much more can be proven safe. +- A shared GC'ed heap is planned. Threadvar pragma @@ -5648,78 +5667,6 @@ initialized within the ``var`` section. (Every thread local variable needs to be replicated at thread creation.) -Actor model ------------ - -**Caution**: This section is already outdated! XXX - -Nimrod supports the `actor model`:idx: of concurrency natively: - -.. code-block:: nimrod - type - TMsgKind = enum - mLine, mEof - TMsg = object - case k: TMsgKind - of mEof: nil - of mLine: data: string - - var - thr: TThread[TMsg] - printedLines = 0 - m: TMsg - - proc print() {.thread.} = - while true: - var x = recv[TMsg]() - if x.k == mEof: break - echo x.data - discard atomicInc(printedLines) - - createThread(thr, print) - - var input = open("readme.txt") - while not endOfFile(input): - m.data = input.readLine() - thr.send(m) - close(input) - m.k = mEof - thr.send(m) - joinThread(thr) - - echo printedLines - -In the actor model threads communicate only over sending messages (`send`:idx: -and `recv`:idx: built-ins), not by sharing memory. Every thread has -an `inbox`:idx: that keeps incoming messages until the thread requests a new -message via the ``recv`` operation. The inbox is an unlimited FIFO queue. - -In the above example the ``print`` thread also communicates with its -parent thread over the ``printedLines`` global variable. In general, it is -highly advisable to only read from globals, but not to write to them. In fact -a write to a global that contains GC'ed memory is always wrong, because it -violates the *no heap sharing restriction*: - -.. code-block:: nimrod - var - global: string - t: TThread[string] - - proc horrible() {.thread.} = - global = "string in thread local heap!" - - createThread(t, horrible) - joinThread(t) - -For the above code the compiler produces "Warning: write to foreign heap". This -warning might become an error message in future versions of the compiler. - -Creating a thread is an expensive operation, because a new stack and heap needs -to be created for the thread. It is therefore highly advisable that a thread -handles a large amount of work. Nimrod prefers *coarse grained* -over *fine grained* concurrency. - - Threads and exceptions ---------------------- @@ -5727,6 +5674,7 @@ The interaction between threads and exceptions is simple: A *handled* exception in one thread cannot affect any other thread. However, an *unhandled* exception in one thread terminates the whole *process*! + Taint mode ========== diff --git a/doc/tut1.txt b/doc/tut1.txt index 5a20629a2..46eda7ae3 100644 --- a/doc/tut1.txt +++ b/doc/tut1.txt @@ -132,7 +132,7 @@ a backslash: TMyObject {.final, pure, acyclic.} = object # comment continues: \ # we have lots of space here to comment 'TMyObject'. # This line belongs to the comment as it's properly aligned. - + Comments are tokens; they are only allowed at certain places in the input file as they belong to the syntax tree! This feature enables perfect source-to-source @@ -150,6 +150,18 @@ the syntax, watch their indentation: **Note**: To comment out a large piece of code, it is often better to use a ``when false:`` statement. +.. code-block:: nimrod + when false: + brokenCode() + +Another option is to use the `discard`_ statement together with +*long string literals* to create block comments: + +.. code-block:: nimrod + discard """ You can have any nimrod code text commented + out inside this with no indentation restrictions. + yes("May I ask a pointless question?") """ + Numbers ------- @@ -575,27 +587,46 @@ Some terminology: in the example ``question`` is called a (formal) *parameter*, Result variable --------------- -A procedure that returns a value has an implicit ``result`` variable that -represents the return value. A ``return`` statement with no expression is a -shorthand for ``return result``. So all three code snippets are equivalent: - -.. code-block:: nimrod - return 42 - -.. code-block:: nimrod - result = 42 - return - -.. code-block:: nimrod - result = 42 - return result - - +A procedure that returns a value has an implicit ``result`` variable declared +that represents the return value. A ``return`` statement with no expression is a +shorthand for ``return result``. The ``result`` value is always returned +automatically at the end a procedure if there is no ``return`` statement at +the exit. + +.. code-block:: nimrod + proc sumTillNegative(x: varargs[int]): int = + for i in x: + if i < 0: + return + result = result + i + + echo sumTillNegative() # echos 0 + echo sumTillNegative(3, 4, 5) # echos 12 + echo sumTillNegative(3, 4 , -1 , 6) # echos 7 + +The ``result`` variable is already implicitly declared at the start of the +function, so declaring it again with 'var result', for example, would shadow it +with a normal variable of the same name. The result variable is also already +initialised with the type's default value. Note that referential data types will +be ``nil`` at the start of the procedure, and thus may require manual +initialisation. + + Parameters ---------- -Parameters are constant in the procedure body. Their value cannot be changed -because this allows the compiler to implement parameter passing in the most -efficient way. If the procedure needs to modify the argument for the +Parameters are constant in the procedure body. By default, their value cannot be +changed because this allows the compiler to implement parameter passing in the +most efficient way. If a mutable variable is needed inside the procedure, it has +to be declared with ``var`` in the procedure body. Shadowing the parameter name +is possible, and actually an idiom: + +.. code-block:: nimrod + proc printSeq(s: seq, nprinted: int = -1) = + var nprinted = if nprinted == -1: s.len else: min(nprinted, s.len) + for i in 0 .. <nprinted: + echo s[i] + +If the procedure needs to modify the argument for the caller, a ``var`` parameter can be used: .. code-block:: nimrod @@ -634,6 +665,9 @@ been declared with the ``discardable`` pragma: p(3, 4) # now valid +The discard statement can also be used to create block comments as described +in the `Comments`_. + Named arguments --------------- diff --git a/koch.nim b/koch.nim index d7da56590..9d59344f2 100644 --- a/koch.nim +++ b/koch.nim @@ -53,6 +53,7 @@ Boot options: (not needed on Windows) -d:nativeStacktrace use native stack traces (only for Mac OS X or Linux) -d:noCaas build Nimrod without CAAS support + -d:avoidTimeMachine only for Mac OS X, excludes nimcache dir from backups """ proc exe(f: string): string = return addFileExt(f, ExeExt) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 79bf34b7c..09dc341df 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -7,6 +7,7 @@ # distribution, for details about the copyright. # +include "system/inclrtl" ## This module contains the interface to the compiler's abstract syntax ## tree (`AST`:idx:). Macros operate on this tree. @@ -109,19 +110,20 @@ const nnkCallKinds* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand, nnkCallStrLit} -proc `[]`*(n: PNimrodNode, i: int): PNimrodNode {.magic: "NChild".} +proc `[]`*(n: PNimrodNode, i: int): PNimrodNode {.magic: "NChild", noSideEffect.} ## get `n`'s `i`'th child. -proc `[]=`*(n: PNimrodNode, i: int, child: PNimrodNode) {.magic: "NSetChild".} +proc `[]=`*(n: PNimrodNode, i: int, child: PNimrodNode) {.magic: "NSetChild", + noSideEffect.} ## set `n`'s `i`'th child to `child`. -proc `!`*(s: string): TNimrodIdent {.magic: "StrToIdent".} +proc `!`*(s: string): TNimrodIdent {.magic: "StrToIdent", noSideEffect.} ## constructs an identifier from the string `s` -proc `$`*(i: TNimrodIdent): string {.magic: "IdentToStr".} +proc `$`*(i: TNimrodIdent): string {.magic: "IdentToStr", noSideEffect.} ## converts a Nimrod identifier to a string -proc `$`*(s: PNimrodSymbol): string {.magic: "IdentToStr".} +proc `$`*(s: PNimrodSymbol): string {.magic: "IdentToStr", noSideEffect.} ## converts a Nimrod symbol to a string proc `==`*(a, b: TNimrodIdent): bool {.magic: "EqIdent", noSideEffect.} @@ -130,35 +132,36 @@ proc `==`*(a, b: TNimrodIdent): bool {.magic: "EqIdent", noSideEffect.} proc `==`*(a, b: PNimrodNode): bool {.magic: "EqNimrodNode", noSideEffect.} ## compares two Nimrod nodes -proc len*(n: PNimrodNode): int {.magic: "NLen".} +proc len*(n: PNimrodNode): int {.magic: "NLen", noSideEffect.} ## returns the number of children of `n`. -proc add*(father, child: PNimrodNode): PNimrodNode {.magic: "NAdd", discardable.} +proc add*(father, child: PNimrodNode): PNimrodNode {.magic: "NAdd", discardable, + noSideEffect.} ## Adds the `child` to the `father` node. Returns the ## father node so that calls can be nested. proc add*(father: PNimrodNode, children: varargs[PNimrodNode]): PNimrodNode {. - magic: "NAddMultiple", discardable.} + magic: "NAddMultiple", discardable, noSideEffect.} ## Adds each child of `children` to the `father` node. ## Returns the `father` node so that calls can be nested. -proc del*(father: PNimrodNode, idx = 0, n = 1) {.magic: "NDel".} +proc del*(father: PNimrodNode, idx = 0, n = 1) {.magic: "NDel", noSideEffect.} ## deletes `n` children of `father` starting at index `idx`. -proc kind*(n: PNimrodNode): TNimrodNodeKind {.magic: "NKind".} +proc kind*(n: PNimrodNode): TNimrodNodeKind {.magic: "NKind", noSideEffect.} ## returns the `kind` of the node `n`. -proc intVal*(n: PNimrodNode): BiggestInt {.magic: "NIntVal".} -proc floatVal*(n: PNimrodNode): BiggestFloat {.magic: "NFloatVal".} -proc symbol*(n: PNimrodNode): PNimrodSymbol {.magic: "NSymbol".} -proc ident*(n: PNimrodNode): TNimrodIdent {.magic: "NIdent".} -proc typ*(n: PNimrodNode): typedesc {.magic: "NGetType".} -proc strVal*(n: PNimrodNode): string {.magic: "NStrVal".} - -proc `intVal=`*(n: PNimrodNode, val: BiggestInt) {.magic: "NSetIntVal".} -proc `floatVal=`*(n: PNimrodNode, val: BiggestFloat) {.magic: "NSetFloatVal".} -proc `symbol=`*(n: PNimrodNode, val: PNimrodSymbol) {.magic: "NSetSymbol".} -proc `ident=`*(n: PNimrodNode, val: TNimrodIdent) {.magic: "NSetIdent".} +proc intVal*(n: PNimrodNode): BiggestInt {.magic: "NIntVal", noSideEffect.} +proc floatVal*(n: PNimrodNode): BiggestFloat {.magic: "NFloatVal", noSideEffect.} +proc symbol*(n: PNimrodNode): PNimrodSymbol {.magic: "NSymbol", noSideEffect.} +proc ident*(n: PNimrodNode): TNimrodIdent {.magic: "NIdent", noSideEffect.} +proc typ*(n: PNimrodNode): typedesc {.magic: "NGetType", noSideEffect.} +proc strVal*(n: PNimrodNode): string {.magic: "NStrVal", noSideEffect.} + +proc `intVal=`*(n: PNimrodNode, val: BiggestInt) {.magic: "NSetIntVal", noSideEffect.} +proc `floatVal=`*(n: PNimrodNode, val: BiggestFloat) {.magic: "NSetFloatVal", noSideEffect.} +proc `symbol=`*(n: PNimrodNode, val: PNimrodSymbol) {.magic: "NSetSymbol", noSideEffect.} +proc `ident=`*(n: PNimrodNode, val: TNimrodIdent) {.magic: "NSetIdent", noSideEffect.} #proc `typ=`*(n: PNimrodNode, typ: typedesc) {.magic: "NSetType".} # this is not sound! Unfortunately forbidding 'typ=' is not enough, as you # can easily do: @@ -166,24 +169,24 @@ proc `ident=`*(n: PNimrodNode, val: TNimrodIdent) {.magic: "NSetIdent".} # let fake = semCheck(2.0) # bracket[0] = fake # constructs a mixed array with ints and floats! -proc `strVal=`*(n: PNimrodNode, val: string) {.magic: "NSetStrVal".} +proc `strVal=`*(n: PNimrodNode, val: string) {.magic: "NSetStrVal", noSideEffect.} proc newNimNode*(kind: TNimrodNodeKind, - n: PNimrodNode=nil): PNimrodNode {.magic: "NNewNimNode".} + n: PNimrodNode=nil): PNimrodNode {.magic: "NNewNimNode", noSideEffect.} -proc copyNimNode*(n: PNimrodNode): PNimrodNode {.magic: "NCopyNimNode".} -proc copyNimTree*(n: PNimrodNode): PNimrodNode {.magic: "NCopyNimTree".} +proc copyNimNode*(n: PNimrodNode): PNimrodNode {.magic: "NCopyNimNode", noSideEffect.} +proc copyNimTree*(n: PNimrodNode): PNimrodNode {.magic: "NCopyNimTree", noSideEffect.} -proc error*(msg: string) {.magic: "NError".} +proc error*(msg: string) {.magic: "NError", gcsafe.} ## writes an error message at compile time -proc warning*(msg: string) {.magic: "NWarning".} +proc warning*(msg: string) {.magic: "NWarning", gcsafe.} ## writes a warning message at compile time -proc hint*(msg: string) {.magic: "NHint".} +proc hint*(msg: string) {.magic: "NHint", gcsafe.} ## writes a hint message at compile time -proc newStrLitNode*(s: string): PNimrodNode {.compileTime.} = +proc newStrLitNode*(s: string): PNimrodNode {.compileTime, noSideEffect.} = ## creates a string literal node from `s` result = newNimNode(nnkStrLit) result.strVal = s @@ -219,7 +222,7 @@ type ## any other means in the language currently) proc bindSym*(ident: string, rule: TBindSymRule = brClosed): PNimrodNode {. - magic: "NBindSym".} + magic: "NBindSym", noSideEffect.} ## creates a node that binds `ident` to a symbol node. The bound symbol ## may be an overloaded symbol. ## If ``rule == brClosed`` either an ``nkClosedSymChoice`` tree is @@ -230,11 +233,11 @@ proc bindSym*(ident: string, rule: TBindSymRule = brClosed): PNimrodNode {. ## returned even if the symbol is not ambiguous. proc genSym*(kind: TNimrodSymKind = nskLet; ident = ""): PNimrodNode {. - magic: "NGenSym".} + magic: "NGenSym", noSideEffect.} ## generates a fresh symbol that is guaranteed to be unique. The symbol ## needs to occur in a declaration context. -proc callsite*(): PNimrodNode {.magic: "NCallSite".} +proc callsite*(): PNimrodNode {.magic: "NCallSite", gcsafe.} ## returns the AST if the invokation expression that invoked this macro. proc toStrLit*(n: PNimrodNode): PNimrodNode {.compileTime.} = @@ -242,19 +245,19 @@ proc toStrLit*(n: PNimrodNode): PNimrodNode {.compileTime.} = ## in a string literal node return newStrLitNode(repr(n)) -proc lineinfo*(n: PNimrodNode): string {.magic: "NLineInfo".} +proc lineinfo*(n: PNimrodNode): string {.magic: "NLineInfo", noSideEffect.} ## returns the position the node appears in the original source file ## in the form filename(line, col) -proc parseExpr*(s: string): PNimrodNode {.magic: "ParseExprToAst".} +proc parseExpr*(s: string): PNimrodNode {.magic: "ParseExprToAst", noSideEffect.} ## Compiles the passed string to its AST representation. ## Expects a single expression. -proc parseStmt*(s: string): PNimrodNode {.magic: "ParseStmtToAst".} +proc parseStmt*(s: string): PNimrodNode {.magic: "ParseStmtToAst", noSideEffect.} ## Compiles the passed string to its AST representation. ## Expects one or more statements. -proc getAst*(macroOrTemplate: expr): PNimrodNode {.magic: "ExpandToAst".} +proc getAst*(macroOrTemplate: expr): PNimrodNode {.magic: "ExpandToAst", noSideEffect.} ## Obtains the AST nodes returned from a macro or template invocation. ## Example: ## @@ -263,7 +266,7 @@ proc getAst*(macroOrTemplate: expr): PNimrodNode {.magic: "ExpandToAst".} ## macro FooMacro() = ## var ast = getAst(BarTemplate()) -proc quote*(bl: stmt, op = "``"): PNimrodNode {.magic: "QuoteAst".} +proc quote*(bl: stmt, op = "``"): PNimrodNode {.magic: "QuoteAst", noSideEffect.} ## Quasi-quoting operator. ## Accepts an expression or a block and returns the AST that represents it. ## Within the quoted AST, you are able to interpolate PNimrodNode expressions diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index c507f5e1c..80fbf3a51 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -52,7 +52,7 @@ const "finally", "for", "from", "generic", "if", "import", "in", "include", "interface", "is", "isnot", "iterator", "lambda", "let", "macro", "method", "mixin", "mod", "nil", "not", "notin", "object", "of", "or", "out", "proc", - "ptr", "raise", "ref", "return", "shared", "shl", "shr", "static", + "ptr", "raise", "ref", "return", "shl", "shr", "static", "template", "try", "tuple", "type", "using", "var", "when", "while", "with", "without", "xor", "yield"] @@ -61,6 +61,7 @@ proc getSourceLanguage*(name: string): TSourceLanguage = if cmpIgnoreStyle(name, sourceLanguageToStr[i]) == 0: return i result = langNone + proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: cstring) = g.buf = buf g.kind = low(TTokenClass) @@ -70,6 +71,7 @@ proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: cstring) = var pos = 0 # skip initial whitespace: while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) g.pos = pos + proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: string) = initGeneralTokenizer(g, cstring(buf)) @@ -554,7 +556,7 @@ when isMainModule: let input = string(readFile(filename)) keywords = input.split() break - doAssert (not keywords.isNil, "Couldn't read any keywords.txt file!") + doAssert(not keywords.isNil, "Couldn't read any keywords.txt file!") doAssert keywords.len == nimrodKeywords.len, "No matching lengths" for i in 0..keywords.len-1: #echo keywords[i], " == ", nimrodKeywords[i] diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 30cc9026b..23459ade6 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -311,8 +311,8 @@ proc newSharedState(options: TRstParseOptions, result.subs = @[] result.refs = @[] result.options = options - result.msgHandler = if isNil(msgHandler): defaultMsgHandler else: msgHandler - result.findFile = if isNil(findFile): defaultFindFile else: findFile + result.msgHandler = if not isNil(msgHandler): msgHandler else: defaultMsgHandler + result.findFile = if not isNil(findFile): findFile else: defaultFindFile proc rstMessage(p: TRstParser, msgKind: TMsgKind, arg: string) = p.s.msgHandler(p.filename, p.line + p.tok[p.idx].line, diff --git a/lib/posix/epoll.nim b/lib/posix/epoll.nim index 57a2f001f..ee04348e8 100644 --- a/lib/posix/epoll.nim +++ b/lib/posix/epoll.nim @@ -7,6 +7,8 @@ # distribution, for details about the copyright. # +{.deadCodeElim:on.} + from posix import TSocketHandle const diff --git a/lib/posix/inotify.nim b/lib/posix/inotify.nim index 28dcd652f..852eb12fa 100644 --- a/lib/posix/inotify.nim +++ b/lib/posix/inotify.nim @@ -7,6 +7,8 @@ # distribution, for details about the copyright. # +{.deadCodeElim:on.} + # Get the platform-dependent flags. # Structure describing an inotify event. type diff --git a/lib/posix/linux.nim b/lib/posix/linux.nim index 1ed1af3b6..be591e29a 100644 --- a/lib/posix/linux.nim +++ b/lib/posix/linux.nim @@ -1,3 +1,5 @@ +{.deadCodeElim:on.} + import posix const @@ -22,4 +24,5 @@ const # fn should be of type proc (a2: pointer): void {.cdecl.} proc clone*(fn: pointer; child_stack: pointer; flags: cint; - arg: pointer; ptid: ptr TPid; tls: pointer; ctid: ptr TPid): cint {.importc, header: "<sched.h>".} + arg: pointer; ptid: ptr TPid; tls: pointer; + ctid: ptr TPid): cint {.importc, header: "<sched.h>".} diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 131f23fdd..e206447cc 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -27,6 +27,8 @@ ## resulting C code will just ``#include <XYZ.h>`` and *not* define the ## symbols declared here. +{.deadCodeElim:on.} + from times import TTime const diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 5e638dc74..f50383038 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -7,6 +7,8 @@ # distribution, for details about the copyright. # +include "system/inclrtl" + import os, oids, tables, strutils, macros import rawsockets @@ -31,7 +33,7 @@ export TPort type PFutureBase* = ref object of PObject - cb: proc () {.closure.} + cb: proc () {.closure,gcsafe.} finished: bool PFuture*[T] = ref object of PFutureBase @@ -68,7 +70,7 @@ proc fail*[T](future: PFuture[T], error: ref EBase) = if future.cb != nil: future.cb() -proc `callback=`*(future: PFutureBase, cb: proc () {.closure.}) = +proc `callback=`*(future: PFutureBase, cb: proc () {.closure,gcsafe.}) = ## Sets the callback proc to be called when the future completes. ## ## If future has already completed then ``cb`` will be called immediately. @@ -80,7 +82,7 @@ proc `callback=`*(future: PFutureBase, cb: proc () {.closure.}) = future.cb() proc `callback=`*[T](future: PFuture[T], - cb: proc (future: PFuture[T]) {.closure.}) = + cb: proc (future: PFuture[T]) {.closure,gcsafe.}) = ## Sets the callback proc to be called when the future completes. ## ## If future has already completed then ``cb`` will be called immediately. @@ -122,7 +124,7 @@ when defined(windows) or defined(nimdoc): TCompletionData* = object sock: TAsyncFD cb: proc (sock: TAsyncFD, bytesTransferred: DWORD, - errcode: TOSErrorCode) {.closure.} + errcode: TOSErrorCode) {.closure,gcsafe.} PDispatcher* = ref object ioPort: THandle @@ -237,7 +239,7 @@ when defined(windows) or defined(nimdoc): let func = cast[proc (s: TSocketHandle, name: ptr TSockAddr, namelen: cint, lpSendBuffer: pointer, dwSendDataLength: dword, - lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool {.stdcall.}](connectExPtr) + lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool {.stdcall,gcsafe.}](connectExPtr) result = func(s, name, namelen, lpSendBuffer, dwSendDataLength, lpdwBytesSent, lpOverlapped) @@ -251,7 +253,7 @@ when defined(windows) or defined(nimdoc): cast[proc (listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer, dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD, - lpOverlapped: POverlapped): bool {.stdcall.}](acceptExPtr) + lpOverlapped: POverlapped): bool {.stdcall,gcsafe.}](acceptExPtr) result = func(listenSock, acceptSock, lpOutputBuffer, dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived, lpOverlapped) @@ -268,7 +270,7 @@ when defined(windows) or defined(nimdoc): dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength: DWORD, LocalSockaddr: ptr ptr TSockAddr, LocalSockaddrLength: lpint, RemoteSockaddr: ptr ptr TSockAddr, - RemoteSockaddrLength: lpint) {.stdcall.}](getAcceptExSockAddrsPtr) + RemoteSockaddrLength: lpint) {.stdcall,gcsafe.}](getAcceptExSockAddrsPtr) func(lpOutputBuffer, dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength, LocalSockaddr, LocalSockaddrLength, @@ -537,7 +539,7 @@ else: from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK type TAsyncFD* = distinct cint - TCallback = proc (sock: TAsyncFD): bool {.closure.} + TCallback = proc (sock: TAsyncFD): bool {.closure,gcsafe.} PData* = ref object of PObject sock: TAsyncFD @@ -757,7 +759,7 @@ proc accept*(socket: TAsyncFD): PFuture[TAsyncFD] = template createCb*(retFutureSym, iteratorNameSym: expr): stmt {.immediate.} = var nameIterVar = iteratorNameSym - proc cb {.closure.} = + proc cb {.closure,gcsafe.} = if not nameIterVar.finished: var next = nameIterVar() if next == nil: diff --git a/lib/pure/asyncio.nim b/lib/pure/asyncio.nim index ab09dc860..c68ca4350 100644 --- a/lib/pure/asyncio.nim +++ b/lib/pure/asyncio.nim @@ -6,6 +6,8 @@ # distribution, for details about the copyright. # +include "system/inclrtl" + import sockets, os ## This module implements an asynchronous event loop together with asynchronous sockets @@ -98,13 +100,13 @@ type fd*: TSocketHandle deleVal*: PObject - handleRead*: proc (h: PObject) {.nimcall.} - handleWrite*: proc (h: PObject) {.nimcall.} - handleError*: proc (h: PObject) {.nimcall.} - hasDataBuffered*: proc (h: PObject): bool {.nimcall.} + handleRead*: proc (h: PObject) {.nimcall, gcsafe.} + handleWrite*: proc (h: PObject) {.nimcall, gcsafe.} + handleError*: proc (h: PObject) {.nimcall, gcsafe.} + hasDataBuffered*: proc (h: PObject): bool {.nimcall, gcsafe.} open*: bool - task*: proc (h: PObject) {.nimcall.} + task*: proc (h: PObject) {.nimcall, gcsafe.} mode*: TFileMode PDelegate* = ref TDelegate @@ -118,13 +120,13 @@ type socket: TSocket info: TInfo - handleRead*: proc (s: PAsyncSocket) {.closure.} - handleWrite: proc (s: PAsyncSocket) {.closure.} - handleConnect*: proc (s: PAsyncSocket) {.closure.} + handleRead*: proc (s: PAsyncSocket) {.closure, gcsafe.} + handleWrite: proc (s: PAsyncSocket) {.closure, gcsafe.} + handleConnect*: proc (s: PAsyncSocket) {.closure, gcsafe.} - handleAccept*: proc (s: PAsyncSocket) {.closure.} + handleAccept*: proc (s: PAsyncSocket) {.closure, gcsafe.} - handleTask*: proc (s: PAsyncSocket) {.closure.} + handleTask*: proc (s: PAsyncSocket) {.closure, gcsafe.} lineBuffer: TaintedString ## Temporary storage for ``readLine`` sendBuffer: string ## Temporary storage for ``send`` @@ -213,7 +215,7 @@ proc asyncSockHandleRead(h: PObject) = else: PAsyncSocket(h).handleAccept(PAsyncSocket(h)) -proc close*(sock: PAsyncSocket) +proc close*(sock: PAsyncSocket) {.gcsafe.} proc asyncSockHandleWrite(h: PObject) = when defined(ssl): if PAsyncSocket(h).socket.isSSL and not @@ -254,7 +256,7 @@ proc asyncSockHandleWrite(h: PObject) = PAsyncSocket(h).deleg.mode = fmRead when defined(ssl): - proc asyncSockDoHandshake(h: PObject) = + proc asyncSockDoHandshake(h: PObject) {.gcsafe.} = if PAsyncSocket(h).socket.isSSL and not PAsyncSocket(h).socket.gotHandshake: if PAsyncSocket(h).sslNeedAccept: @@ -437,7 +439,7 @@ proc isSendDataBuffered*(s: PAsyncSocket): bool = return s.sendBuffer.len != 0 proc setHandleWrite*(s: PAsyncSocket, - handleWrite: proc (s: PAsyncSocket) {.closure.}) = + handleWrite: proc (s: PAsyncSocket) {.closure, gcsafe.}) = ## Setter for the ``handleWrite`` event. ## ## To remove this event you should use the ``delHandleWrite`` function. diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index 4e2b6f5f8..31fb24eef 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -377,7 +377,7 @@ proc setCookie*(name, value: string) = write(stdout, "Set-Cookie: ", name, "=", value, "\n") var - gcookies: PStringTable = nil + gcookies {.threadvar.}: PStringTable proc getCookie*(name: string): TaintedString = ## Gets a cookie. If no cookie of `name` exists, "" is returned. diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 412bebeee..cd28f9af0 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -196,7 +196,7 @@ proc `==`*[A, B](s, t: TTable[A, B]): bool = # to use the slow route here: for key, val in s: if not hasKey(t, key): return false - if mget(t, key) != val: return false + if t[key] != val: return false return true proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): TTable[C, B] = diff --git a/lib/pure/ftpclient.nim b/lib/pure/ftpclient.nim index 3bb55239b..53f6688b9 100644 --- a/lib/pure/ftpclient.nim +++ b/lib/pure/ftpclient.nim @@ -6,6 +6,8 @@ # distribution, for details about the copyright. # +include "system/inclrtl" + import sockets, strutils, parseutils, times, os, asyncio ## This module **partially** implements an FTP client as specified @@ -41,7 +43,7 @@ type dummyA, dummyB: pointer # workaround a Nimrod API issue asyncCSock: PAsyncSocket asyncDSock: PAsyncSocket - handleEvent*: proc (ftp: PAsyncFTPClient, ev: TFTPEvent) {.closure.} + handleEvent*: proc (ftp: PAsyncFTPClient, ev: TFTPEvent){.closure,gcsafe.} disp: PDispatcher asyncDSockID: PDelegate user, pass: string @@ -59,7 +61,7 @@ type JRetrText, JRetr, JStore TFTPJob = object - prc: proc (ftp: PFTPClient, async: bool): bool {.nimcall.} + prc: proc (ftp: PFTPClient, async: bool): bool {.nimcall, gcsafe.} case typ*: FTPJobType of JRetrText: lines: string @@ -148,7 +150,8 @@ proc assertReply(received: TaintedString, expected: varargs[string]) = [expected.join("' or '"), received.string]) proc createJob(ftp: PFTPClient, - prc: proc (ftp: PFTPClient, async: bool): bool {.nimcall.}, + prc: proc (ftp: PFTPClient, async: bool): bool {. + nimcall,gcsafe.}, cmd: FTPJobType) = if ftp.jobInProgress: raise newException(EFTP, "Unable to do two jobs at once.") @@ -558,7 +561,7 @@ proc csockHandleRead(s: PAsyncSocket, ftp: PAsyncFTPClient) = proc asyncFTPClient*(address: string, port = TPort(21), user, pass = "", - handleEvent: proc (ftp: PAsyncFTPClient, ev: TFTPEvent) {.closure.} = + handleEvent: proc (ftp: PAsyncFTPClient, ev: TFTPEvent) {.closure,gcsafe.} = (proc (ftp: PAsyncFTPClient, ev: TFTPEvent) = discard)): PAsyncFTPClient = ## Create a ``PAsyncFTPClient`` object. ## diff --git a/lib/pure/future.nim b/lib/pure/future.nim new file mode 100644 index 000000000..73c20e708 --- /dev/null +++ b/lib/pure/future.nim @@ -0,0 +1,118 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2014 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements experimental features which may soon be moved to +## the system module (or other more appropriate modules). + +import macros + +proc createProcType(p, b: PNimrodNode): PNimrodNode {.compileTime.} = + #echo treeRepr(p) + #echo treeRepr(b) + result = newNimNode(nnkProcTy) + var formalParams = newNimNode(nnkFormalParams) + + expectKind(b, nnkIdent) + formalParams.add b + + case p.kind + of nnkPar: + for i in 0 .. <p.len: + let ident = p[i] + var identDefs = newNimNode(nnkIdentDefs) + case ident.kind + of nnkExprColonExpr: + identDefs.add ident[0] + identDefs.add ident[1] + of nnkIdent: + identDefs.add newIdentNode("i" & $i) + identDefs.add(ident) + else: + error("Incorrect type list in proc type declaration.") + identDefs.add newEmptyNode() + formalParams.add identDefs + of nnkIdent: + var identDefs = newNimNode(nnkIdentDefs) + identDefs.add newIdentNode("i0") + identDefs.add(p) + identDefs.add newEmptyNode() + formalParams.add identDefs + else: + error("Incorrect type list in proc type declaration.") + + result.add formalParams + result.add newEmptyNode() + #echo(treeRepr(result)) + #echo(result.toStrLit()) + +macro `=>`*(p, b: expr): expr {.immediate.} = + ## Syntax sugar for anonymous procedures. + ## + ## .. code-block:: nimrod + ## + ## proc passTwoAndTwo(f: (int, int) -> int): int = + ## f(2, 2) + ## + ## passTwoAndTwo((x, y) => x + y) # 4 + + #echo treeRepr(p) + #echo(treeRepr(b)) + var params: seq[PNimrodNode] = @[newIdentNode("auto")] + + case p.kind + of nnkPar: + for c in children(p): + var identDefs = newNimNode(nnkIdentDefs) + case c.kind + of nnkExprColonExpr: + identDefs.add(c[0]) + identDefs.add(c[1]) + identDefs.add(newEmptyNode()) + of nnkIdent: + identDefs.add(c) + identDefs.add(newEmptyNode()) + identDefs.add(newEmptyNode()) + else: + error("Incorrect procedure parameter list.") + params.add(identDefs) + of nnkIdent: + var identDefs = newNimNode(nnkIdentDefs) + identDefs.add(p) + identDefs.add(newEmptyNode()) + identDefs.add(newEmptyNode()) + params.add(identDefs) + of nnkInfix: + if p[0].kind == nnkIdent and p[0].ident == !"->": + var procTy = createProcType(p[1], p[2]) + params[0] = procTy[0][0] + for i in 1 .. <procTy[0].len: + params.add(procTy[0][i]) + else: + error("Expected proc type (->) got (" & $p[0].ident & ").") + else: + error("Incorrect procedure parameter list.") + result = newProc(params = params, body = b, procType = nnkLambda) + #echo(result.treeRepr) + #echo(result.toStrLit()) + #return result # TODO: Bug? + +macro `->`*(p, b: expr): expr {.immediate.} = + ## Syntax sugar for procedure types. + ## + ## .. code-block:: nimrod + ## + ## proc pass2(f: (float, float) -> float): float = + ## f(2, 2) + ## + ## # is the same as: + ## + ## proc pass2(f: proc (x, y: float): float): float = + ## f(2, 2) + + createProcType(p, b) diff --git a/lib/pure/httpserver.nim b/lib/pure/httpserver.nim index 901fdc779..8de708c5d 100644 --- a/lib/pure/httpserver.nim +++ b/lib/pure/httpserver.nim @@ -477,7 +477,7 @@ proc nextAsync(s: PAsyncHTTPServer) = s.path = data.substr(i, last-1) proc asyncHTTPServer*(handleRequest: proc (server: PAsyncHTTPServer, client: TSocket, - path, query: string): bool {.closure.}, + path, query: string): bool {.closure, gcsafe.}, port = TPort(80), address = "", reuseAddr = false): PAsyncHTTPServer = ## Creates an Asynchronous HTTP server at ``port``. diff --git a/lib/pure/irc.nim b/lib/pure/irc.nim index 31a673210..49d9a9a34 100644 --- a/lib/pure/irc.nim +++ b/lib/pure/irc.nim @@ -32,6 +32,8 @@ ## **Warning:** The API of this module is unstable, and therefore is subject ## to change. +include "system/inclrtl" + import sockets, strutils, parseutils, times, asyncio, os type @@ -41,7 +43,7 @@ type nick, user, realname, serverPass: string case isAsync: bool of true: - handleEvent: proc (irc: PAsyncIRC, ev: TIRCEvent) {.closure.} + handleEvent: proc (irc: PAsyncIRC, ev: TIRCEvent) {.closure, gcsafe.} asyncSock: PAsyncSocket myDispatcher: PDispatcher of false: @@ -445,7 +447,7 @@ proc asyncIRC*(address: string, port: TPort = 6667.TPort, realname = "NimrodBot", serverPass = "", joinChans: seq[string] = @[], msgLimit: bool = true, - ircEvent: proc (irc: PAsyncIRC, ev: TIRCEvent) {.closure.} + ircEvent: proc (irc: PAsyncIRC, ev: TIRCEvent) {.closure,gcsafe.} ): PAsyncIRC = ## Use this function if you want to use asyncio's dispatcher. ## diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 3997b059f..e4aecd272 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -12,6 +12,8 @@ ## Basic math routines for Nimrod. ## This module is available for the JavaScript target. +include "system/inclrtl" + {.push debugger:off .} # the user does not want to trace a part # of the standard library! @@ -127,25 +129,25 @@ proc variance*(x: openArray[float]): float {.noSideEffect.} = result = result + diff*diff result = result / toFloat(len(x)) -proc random*(max: int): int +proc random*(max: int): int {.gcsafe.} ## returns a random number in the range 0..max-1. The sequence of ## random number is always the same, unless `randomize` is called ## which initializes the random number generator with a "random" ## number, i.e. a tickcount. when not defined(windows): - proc random*(max: float): float + proc random*(max: float): float {.gcsafe.} ## returns a random number in the range 0..<max. The sequence of ## random number is always the same, unless `randomize` is called ## which initializes the random number generator with a "random" ## number, i.e. a tickcount. This is currently not supported for windows. -proc randomize*() +proc randomize*() {.gcsafe.} ## initializes the random number generator with a "random" ## number, i.e. a tickcount. Note: Does nothing for the JavaScript target, ## as JavaScript does not support this. -proc randomize*(seed: int) +proc randomize*(seed: int) {.gcsafe.} ## initializes the random number generator with a specific seed. ## Note: Does nothing for the JavaScript target, ## as JavaScript does not support this. @@ -227,8 +229,8 @@ else: result = int(floor(mathrandom() * float(max))) proc random(max: float): float = result = float(mathrandom() * float(max)) - proc randomize() = nil - proc randomize(seed: int) = nil + proc randomize() = discard + proc randomize(seed: int) = discard proc sqrt*(x: float): float {.importc: "Math.sqrt", nodecl.} proc ln*(x: float): float {.importc: "Math.log", nodecl.} diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 30a5ae949..00a33db75 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1567,7 +1567,7 @@ when defined(windows): # ourselves. This has the additional benefit that the program's behaviour # is always the same -- independent of the used C compiler. var - ownArgv: seq[string] + ownArgv {.threadvar.}: seq[string] proc paramCount*(): int {.rtl, extern: "nos$1", tags: [FReadIO].} = ## Returns the number of `command line arguments`:idx: given to the diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 5d6848565..6e250f9d5 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -137,6 +137,14 @@ proc startProcess*(command: string, ## to `startProcess`. See the documentation of ``TProcessOption`` for the ## meaning of these flags. You need to `close` the process when done. ## + ## Note that you can't pass any `args` if you use the option + ## ``poEvalCommand``, which invokes the system shell to run the specified + ## `command`. In this situation you have to concatenate manually the contents + ## of `args` to `command` carefully escaping/quoting any special characters, + ## since it will be passed *as is* to the system shell. Each system/shell may + ## feature different escaping rules, so try to avoid this kind of shell + ## invokation if possible as it leads to non portable software. + ## ## Return value: The newly created process object. Nil is never returned, ## but ``EOS`` is raised in case of an error. @@ -607,11 +615,13 @@ elif not defined(useNimRtl): optionPoStdErrToStdOut: bool when not defined(useFork): - proc startProcessAuxSpawn(data: TStartProcessData): TPid {.tags: [FExecIO, FReadEnv].} - proc startProcessAuxFork(data: TStartProcessData): TPid {.tags: [FExecIO, FReadEnv].} + proc startProcessAuxSpawn(data: TStartProcessData): TPid {. + tags: [FExecIO, FReadEnv], gcsafe.} + proc startProcessAuxFork(data: TStartProcessData): TPid {. + tags: [FExecIO, FReadEnv], gcsafe.} {.push stacktrace: off, profiler: off.} proc startProcessAfterFork(data: ptr TStartProcessData) {. - tags: [FExecIO, FReadEnv], cdecl.} + tags: [FExecIO, FReadEnv], cdecl, gcsafe.} {.pop.} proc startProcess(command: string, @@ -633,7 +643,7 @@ elif not defined(useNimRtl): if poEvalCommand in options: sysCommand = "/bin/sh" sysArgsRaw = @[sysCommand, "-c", command] - assert args.len == 0 + assert args.len == 0, "`args` has to be empty when using poEvalCommand." else: sysCommand = command sysArgsRaw = @[command] @@ -938,7 +948,7 @@ elif not defined(useNimRtl): proc execCmdEx*(command: string, options: set[TProcessOption] = { poStdErrToStdOut, poUsePath}): tuple[ output: TaintedString, - exitCode: int] {.tags: [FExecIO, FReadIO].} = + exitCode: int] {.tags: [FExecIO, FReadIO], gcsafe.} = ## a convenience proc that runs the `command`, grabs all its output and ## exit code and returns both. var p = startCmd(command, options) diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index f3249b107..727a8efd8 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -70,7 +70,7 @@ const SymChars: TCharSet = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.', '/', '\\'} -proc rawGetTok(c: var TCfgParser, tok: var TToken) +proc rawGetTok(c: var TCfgParser, tok: var TToken) {.gcsafe.} proc open*(c: var TCfgParser, input: PStream, filename: string, lineOffset = 0) {. diff --git a/lib/pure/redis.nim b/lib/pure/redis.nim index f4c45b99c..959f5c6ef 100644 --- a/lib/pure/redis.nim +++ b/lib/pure/redis.nim @@ -19,10 +19,21 @@ import sockets, os, strutils, parseutils const redisNil* = "\0\0" +type + PPipeline = ref object + enabled: bool + buffer: string + expected: int ## number of replies expected if pipelined + +type + TSendMode = enum + normal, pipelined, multiple + type TRedis* {.pure, final.} = object socket: TSocket connected: bool + pipeline: PPipeline TRedisStatus* = string TRedisInteger* = biggestInt @@ -32,25 +43,42 @@ type EInvalidReply* = object of ESynch ## Invalid reply from redis ERedis* = object of ESynch ## Error in redis +proc newPipeline(): PPipeline = + new(result) + result.buffer = "" + result.enabled = false + result.expected = 0 + proc open*(host = "localhost", port = 6379.TPort): TRedis = ## Opens a connection to the redis server. result.socket = socket(buffered = false) if result.socket == InvalidSocket: OSError(OSLastError()) result.socket.connect(host, port) + result.pipeline = newPipeline() proc raiseInvalidReply(expected, got: char) = raise newException(EInvalidReply, "Expected '$1' at the beginning of a status reply got '$2'" % [$expected, $got]) -proc raiseNoOK(status: string) = - if status != "QUEUED" and status != "OK": +proc raiseNoOK(status: string, pipelineEnabled:bool) = + if pipelineEnabled and not (status == "QUEUED" or status == "PIPELINED"): + raise newException(EInvalidReply, "Expected \"QUEUED\" or \"PIPELINED\" got \"$1\"" % status) + elif not pipelineEnabled and status != "OK": raise newException(EInvalidReply, "Expected \"OK\" got \"$1\"" % status) -proc parseStatus(r: TRedis): TRedisStatus = - var line = "" - r.socket.readLine(line) +template readSocket(r: TRedis, dummyVal:expr): stmt = + var line {.inject.} :TaintedString = "" + if r.pipeline.enabled: + return dummyVal + else: + readLine(r.socket, line) + +proc parseStatus(r: TRedis, line: string = ""): TRedisStatus = + if r.pipeline.enabled: + return "PIPELINED" + if line == "": raise newException(ERedis, "Server closed connection prematurely") @@ -60,13 +88,16 @@ proc parseStatus(r: TRedis): TRedisStatus = raiseInvalidReply('+', line[0]) return line.substr(1) # Strip '+' - -proc parseInteger(r: TRedis): TRedisInteger = - var line = "" - r.socket.readLine(line) - if line == "+QUEUED": # inside of multi - return -1 +proc readStatus(r:TRedis): TRedisStatus = + r.readSocket("PIPELINED") + return r.parseStatus(line) + +proc parseInteger(r: TRedis, line: string = ""): TRedisInteger = + if r.pipeline.enabled: return -1 + + #if line == "+QUEUED": # inside of multi + # return -1 if line == "": raise newException(ERedis, "Server closed connection prematurely") @@ -80,12 +111,18 @@ proc parseInteger(r: TRedis): TRedisInteger = if parseBiggestInt(line, result, 1) == 0: raise newException(EInvalidReply, "Unable to parse integer.") +proc readInteger(r: TRedis): TRedisInteger = + r.readSocket(-1) + return r.parseInteger(line) + proc recv(sock: TSocket, size: int): TaintedString = result = newString(size).TaintedString if sock.recv(cstring(result), size) != size: raise newException(EInvalidReply, "recv failed") -proc parseSingle(r: TRedis, line:string, allowMBNil = False): TRedisString = +proc parseSingleString(r: TRedis, line:string, allowMBNil = False): TRedisString = + if r.pipeline.enabled: return "" + # Error. if line[0] == '-': raise newException(ERedis, strip(line)) @@ -95,9 +132,6 @@ proc parseSingle(r: TRedis, line:string, allowMBNil = False): TRedisString = if line == "*-1": return RedisNil - if line == "+QUEUED" or line == "+OK" : # inside of a transaction (multi) - return nil - if line[0] != '$': raiseInvalidReply('$', line[0]) @@ -108,41 +142,84 @@ proc parseSingle(r: TRedis, line:string, allowMBNil = False): TRedisString = var s = r.socket.recv(numBytes+2) result = strip(s.string) -proc parseMultiLines(r: TRedis, countLine:string): TRedisList = +proc readSingleString(r: TRedis): TRedisString = + r.readSocket("") + return r.parseSingleString(line) + +proc readNext(r: TRedis): TRedisList + +proc parseArrayLines(r: TRedis, countLine:string): TRedisList = if countLine.string[0] != '*': raiseInvalidReply('*', countLine.string[0]) var numElems = parseInt(countLine.string.substr(1)) if numElems == -1: return nil result = @[] + for i in 1..numElems: - var line = "" - r.socket.readLine(line.TaintedString) - if line[0] == '*': # after exec() may contain more multi-bulk replies - var parsed = r.parseMultiLines(line) + var parsed = r.readNext() + if not isNil(parsed): for item in parsed: result.add(item) - else: - result.add(r.parseSingle(line)) - -proc parseBulk(r: TRedis, allowMBNil = False): TRedisString = - var line = "" - r.socket.readLine(line.TaintedString) - if line == "+QUEUED" or line == "+OK": # inside of a transaction (multi) - return nil - - return r.parseSingle(line, allowMBNil) +proc readArrayLines(r: TRedis): TRedisList = + r.readSocket(nil) + return r.parseArrayLines(line) + +proc parseBulkString(r: TRedis, allowMBNil = False, line:string = ""): TRedisString = + if r.pipeline.enabled: return "" + + return r.parseSingleString(line, allowMBNil) + +proc readBulkString(r: TRedis, allowMBNil = false): TRedisString = + r.readSocket("") + return r.parseBulkString(allowMBNil, line) + +proc readArray(r: TRedis): TRedisList = + r.readSocket(@[]) + return r.parseArrayLines(line) + +proc readNext(r: TRedis): TRedisList = + r.readSocket(@[]) + + var res = case line[0] + of '+', '-': @[r.parseStatus(line)] + of ':': @[$(r.parseInteger(line))] + of '$': @[r.parseBulkString(true,line)] + of '*': r.parseArrayLines(line) + else: + raise newException(EInvalidReply, "readNext failed on line: " & line) + nil + r.pipeline.expected -= 1 + return res + +proc flushPipeline*(r: TRedis, wasMulti = false): TRedisList = + ## Send buffered commands, clear buffer, return results + if r.pipeline.buffer.len > 0: + r.socket.send(r.pipeline.buffer) + r.pipeline.buffer = "" + + r.pipeline.enabled = false + result = @[] + + var tot = r.pipeline.expected -proc parseMultiBulk(r: TRedis): TRedisList = - var line = TaintedString"" - r.socket.readLine(line) + for i in 0..tot-1: + var ret = r.readNext() + for item in ret: + if not (item.contains("OK") or item.contains("QUEUED")): + result.add(item) - if line == "+QUEUED": # inside of a transaction (multi) - return nil - - return r.parseMultiLines(line) + r.pipeline.expected = 0 +proc startPipelining*(r: TRedis) = + ## Enable command pipelining (reduces network roundtrips). + ## Note that when enabled, you must call flushPipeline to actually send commands, except + ## for multi/exec() which enable and flush the pipeline automatically. + ## Commands return immediately with dummy values; actual results returned from + ## flushPipeline() or exec() + r.pipeline.expected = 0 + r.pipeline.enabled = true proc sendCommand(r: TRedis, cmd: string, args: varargs[string]) = var request = "*" & $(1 + args.len()) & "\c\L" @@ -151,7 +228,12 @@ proc sendCommand(r: TRedis, cmd: string, args: varargs[string]) = for i in items(args): request.add("$" & $i.len() & "\c\L") request.add(i & "\c\L") - r.socket.send(request) + + if r.pipeline.enabled: + r.pipeline.buffer.add(request) + r.pipeline.expected += 1 + else: + r.socket.send(request) proc sendCommand(r: TRedis, cmd: string, arg1: string, args: varargs[string]) = @@ -163,75 +245,80 @@ proc sendCommand(r: TRedis, cmd: string, arg1: string, for i in items(args): request.add("$" & $i.len() & "\c\L") request.add(i & "\c\L") - r.socket.send(request) + + if r.pipeline.enabled: + r.pipeline.expected += 1 + r.pipeline.buffer.add(request) + else: + r.socket.send(request) # Keys proc del*(r: TRedis, keys: varargs[string]): TRedisInteger = ## Delete a key or multiple keys r.sendCommand("DEL", keys) - return r.parseInteger() + return r.readInteger() proc exists*(r: TRedis, key: string): bool = ## Determine if a key exists r.sendCommand("EXISTS", key) - return r.parseInteger() == 1 + return r.readInteger() == 1 proc expire*(r: TRedis, key: string, seconds: int): bool = ## Set a key's time to live in seconds. Returns `false` if the key could ## not be found or the timeout could not be set. r.sendCommand("EXPIRE", key, $seconds) - return r.parseInteger() == 1 + return r.readInteger() == 1 proc expireAt*(r: TRedis, key: string, timestamp: int): bool = ## Set the expiration for a key as a UNIX timestamp. Returns `false` ## if the key could not be found or the timeout could not be set. r.sendCommand("EXPIREAT", key, $timestamp) - return r.parseInteger() == 1 + return r.readInteger() == 1 proc keys*(r: TRedis, pattern: string): TRedisList = ## Find all keys matching the given pattern r.sendCommand("KEYS", pattern) - return r.parseMultiBulk() + return r.readArray() proc move*(r: TRedis, key: string, db: int): bool = ## Move a key to another database. Returns `true` on a successful move. r.sendCommand("MOVE", key, $db) - return r.parseInteger() == 1 + return r.readInteger() == 1 proc persist*(r: TRedis, key: string): bool = ## Remove the expiration from a key. ## Returns `true` when the timeout was removed. r.sendCommand("PERSIST", key) - return r.parseInteger() == 1 + return r.readInteger() == 1 proc randomKey*(r: TRedis): TRedisString = ## Return a random key from the keyspace r.sendCommand("RANDOMKEY") - return r.parseBulk() + return r.readBulkString() proc rename*(r: TRedis, key, newkey: string): TRedisStatus = ## Rename a key. ## ## **WARNING:** Overwrites `newkey` if it exists! r.sendCommand("RENAME", key, newkey) - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc renameNX*(r: TRedis, key, newkey: string): bool = ## Same as ``rename`` but doesn't continue if `newkey` exists. ## Returns `true` if key was renamed. r.sendCommand("RENAMENX", key, newkey) - return r.parseInteger() == 1 + return r.readInteger() == 1 proc ttl*(r: TRedis, key: string): TRedisInteger = ## Get the time to live for a key r.sendCommand("TTL", key) - return r.parseInteger() + return r.readInteger() proc keyType*(r: TRedis, key: string): TRedisStatus = ## Determine the type stored at key r.sendCommand("TYPE", key) - return r.parseStatus() + return r.readStatus() # Strings @@ -239,125 +326,125 @@ proc keyType*(r: TRedis, key: string): TRedisStatus = proc append*(r: TRedis, key, value: string): TRedisInteger = ## Append a value to a key r.sendCommand("APPEND", key, value) - return r.parseInteger() + return r.readInteger() proc decr*(r: TRedis, key: string): TRedisInteger = ## Decrement the integer value of a key by one r.sendCommand("DECR", key) - return r.parseInteger() + return r.readInteger() proc decrBy*(r: TRedis, key: string, decrement: int): TRedisInteger = ## Decrement the integer value of a key by the given number r.sendCommand("DECRBY", key, $decrement) - return r.parseInteger() + return r.readInteger() proc get*(r: TRedis, key: string): TRedisString = ## Get the value of a key. Returns `redisNil` when `key` doesn't exist. r.sendCommand("GET", key) - return r.parseBulk() + return r.readBulkString() proc getBit*(r: TRedis, key: string, offset: int): TRedisInteger = ## Returns the bit value at offset in the string value stored at key r.sendCommand("GETBIT", key, $offset) - return r.parseInteger() + return r.readInteger() proc getRange*(r: TRedis, key: string, start, stop: int): TRedisString = ## Get a substring of the string stored at a key r.sendCommand("GETRANGE", key, $start, $stop) - return r.parseBulk() + return r.readBulkString() proc getSet*(r: TRedis, key: string, value: string): TRedisString = ## Set the string value of a key and return its old value. Returns `redisNil` ## when key doesn't exist. r.sendCommand("GETSET", key, value) - return r.parseBulk() + return r.readBulkString() proc incr*(r: TRedis, key: string): TRedisInteger = ## Increment the integer value of a key by one. r.sendCommand("INCR", key) - return r.parseInteger() + return r.readInteger() proc incrBy*(r: TRedis, key: string, increment: int): TRedisInteger = ## Increment the integer value of a key by the given number r.sendCommand("INCRBY", key, $increment) - return r.parseInteger() + return r.readInteger() proc setk*(r: TRedis, key, value: string) = ## Set the string value of a key. ## ## NOTE: This function had to be renamed due to a clash with the `set` type. r.sendCommand("SET", key, value) - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc setNX*(r: TRedis, key, value: string): bool = ## Set the value of a key, only if the key does not exist. Returns `true` ## if the key was set. r.sendCommand("SETNX", key, value) - return r.parseInteger() == 1 + return r.readInteger() == 1 proc setBit*(r: TRedis, key: string, offset: int, value: string): TRedisInteger = ## Sets or clears the bit at offset in the string value stored at key r.sendCommand("SETBIT", key, $offset, value) - return r.parseInteger() + return r.readInteger() proc setEx*(r: TRedis, key: string, seconds: int, value: string): TRedisStatus = ## Set the value and expiration of a key r.sendCommand("SETEX", key, $seconds, value) - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc setRange*(r: TRedis, key: string, offset: int, value: string): TRedisInteger = ## Overwrite part of a string at key starting at the specified offset r.sendCommand("SETRANGE", key, $offset, value) - return r.parseInteger() + return r.readInteger() proc strlen*(r: TRedis, key: string): TRedisInteger = ## Get the length of the value stored in a key. Returns 0 when key doesn't ## exist. r.sendCommand("STRLEN", key) - return r.parseInteger() + return r.readInteger() # Hashes proc hDel*(r: TRedis, key, field: string): bool = ## Delete a hash field at `key`. Returns `true` if the field was removed. r.sendCommand("HDEL", key, field) - return r.parseInteger() == 1 + return r.readInteger() == 1 proc hExists*(r: TRedis, key, field: string): bool = ## Determine if a hash field exists. r.sendCommand("HEXISTS", key, field) - return r.parseInteger() == 1 + return r.readInteger() == 1 proc hGet*(r: TRedis, key, field: string): TRedisString = ## Get the value of a hash field r.sendCommand("HGET", key, field) - return r.parseBulk() + return r.readBulkString() proc hGetAll*(r: TRedis, key: string): TRedisList = ## Get all the fields and values in a hash r.sendCommand("HGETALL", key) - return r.parseMultiBulk() + return r.readArray() proc hIncrBy*(r: TRedis, key, field: string, incr: int): TRedisInteger = ## Increment the integer value of a hash field by the given number r.sendCommand("HINCRBY", key, field, $incr) - return r.parseInteger() + return r.readInteger() proc hKeys*(r: TRedis, key: string): TRedisList = ## Get all the fields in a hash r.sendCommand("HKEYS", key) - return r.parseMultiBulk() + return r.readArray() proc hLen*(r: TRedis, key: string): TRedisInteger = ## Get the number of fields in a hash r.sendCommand("HLEN", key) - return r.parseInteger() + return r.readInteger() proc hMGet*(r: TRedis, key: string, fields: varargs[string]): TRedisList = ## Get the values of all the given hash fields r.sendCommand("HMGET", key, fields) - return r.parseMultiBulk() + return r.readArray() proc hMSet*(r: TRedis, key: string, fieldValues: openarray[tuple[field, value: string]]) = @@ -367,22 +454,22 @@ proc hMSet*(r: TRedis, key: string, args.add(field) args.add(value) r.sendCommand("HMSET", args) - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc hSet*(r: TRedis, key, field, value: string): TRedisInteger = ## Set the string value of a hash field r.sendCommand("HSET", key, field, value) - return r.parseInteger() + return r.readInteger() proc hSetNX*(r: TRedis, key, field, value: string): TRedisInteger = ## Set the value of a hash field, only if the field does **not** exist r.sendCommand("HSETNX", key, field, value) - return r.parseInteger() + return r.readInteger() proc hVals*(r: TRedis, key: string): TRedisList = ## Get all the values in a hash r.sendCommand("HVALS", key) - return r.parseMultiBulk() + return r.readArray() # Lists @@ -393,7 +480,7 @@ proc bLPop*(r: TRedis, keys: varargs[string], timeout: int): TRedisList = for i in items(keys): args.add(i) args.add($timeout) r.sendCommand("BLPOP", args) - return r.parseMultiBulk() + return r.readArray() proc bRPop*(r: TRedis, keys: varargs[string], timeout: int): TRedisList = ## Remove and get the *last* element in a list, or block until one @@ -402,7 +489,7 @@ proc bRPop*(r: TRedis, keys: varargs[string], timeout: int): TRedisList = for i in items(keys): args.add(i) args.add($timeout) r.sendCommand("BRPOP", args) - return r.parseMultiBulk() + return r.readArray() proc bRPopLPush*(r: TRedis, source, destination: string, timeout: int): TRedisString = @@ -411,29 +498,29 @@ proc bRPopLPush*(r: TRedis, source, destination: string, ## ## http://redis.io/commands/brpoplpush r.sendCommand("BRPOPLPUSH", source, destination, $timeout) - return r.parseBulk(true) # Multi-Bulk nil allowed. + return r.readBulkString(true) # Multi-Bulk nil allowed. proc lIndex*(r: TRedis, key: string, index: int): TRedisString = ## Get an element from a list by its index r.sendCommand("LINDEX", key, $index) - return r.parseBulk() + return r.readBulkString() proc lInsert*(r: TRedis, key: string, before: bool, pivot, value: string): TRedisInteger = ## Insert an element before or after another element in a list var pos = if before: "BEFORE" else: "AFTER" r.sendCommand("LINSERT", key, pos, pivot, value) - return r.parseInteger() + return r.readInteger() proc lLen*(r: TRedis, key: string): TRedisInteger = ## Get the length of a list r.sendCommand("LLEN", key) - return r.parseInteger() + return r.readInteger() proc lPop*(r: TRedis, key: string): TRedisString = ## Remove and get the first element in a list r.sendCommand("LPOP", key) - return r.parseBulk() + return r.readBulkString() proc lPush*(r: TRedis, key, value: string, create: bool = True): TRedisInteger = ## Prepend a value to a list. Returns the length of the list after the push. @@ -444,39 +531,39 @@ proc lPush*(r: TRedis, key, value: string, create: bool = True): TRedisInteger = r.sendCommand("LPUSH", key, value) else: r.sendCommand("LPUSHX", key, value) - return r.parseInteger() + return r.readInteger() proc lRange*(r: TRedis, key: string, start, stop: int): TRedisList = ## Get a range of elements from a list. Returns `nil` when `key` ## doesn't exist. r.sendCommand("LRANGE", key, $start, $stop) - return r.parseMultiBulk() + return r.readArray() proc lRem*(r: TRedis, key: string, value: string, count: int = 0): TRedisInteger = ## Remove elements from a list. Returns the number of elements that have been ## removed. r.sendCommand("LREM", key, $count, value) - return r.parseInteger() + return r.readInteger() proc lSet*(r: TRedis, key: string, index: int, value: string) = ## Set the value of an element in a list by its index r.sendCommand("LSET", key, $index, value) - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) -proc lTrim*(r: TRedis, key: string, start, stop: int) = +proc lTrim*(r: TRedis, key: string, start, stop: int) = ## Trim a list to the specified range r.sendCommand("LTRIM", key, $start, $stop) - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc rPop*(r: TRedis, key: string): TRedisString = ## Remove and get the last element in a list r.sendCommand("RPOP", key) - return r.parseBulk() + return r.readBulkString() proc rPopLPush*(r: TRedis, source, destination: string): TRedisString = ## Remove the last element in a list, append it to another list and return it r.sendCommand("RPOPLPUSH", source, destination) - return r.parseBulk() + return r.readBulkString() proc rPush*(r: TRedis, key, value: string, create: bool = True): TRedisInteger = ## Append a value to a list. Returns the length of the list after the push. @@ -487,106 +574,106 @@ proc rPush*(r: TRedis, key, value: string, create: bool = True): TRedisInteger = r.sendCommand("RPUSH", key, value) else: r.sendCommand("RPUSHX", key, value) - return r.parseInteger() + return r.readInteger() # Sets proc sadd*(r: TRedis, key: string, member: string): TRedisInteger = ## Add a member to a set r.sendCommand("SADD", key, member) - return r.parseInteger() + return r.readInteger() proc scard*(r: TRedis, key: string): TRedisInteger = ## Get the number of members in a set r.sendCommand("SCARD", key) - return r.parseInteger() + return r.readInteger() proc sdiff*(r: TRedis, keys: varargs[string]): TRedisList = ## Subtract multiple sets r.sendCommand("SDIFF", keys) - return r.parseMultiBulk() + return r.readArray() proc sdiffstore*(r: TRedis, destination: string, keys: varargs[string]): TRedisInteger = ## Subtract multiple sets and store the resulting set in a key r.sendCommand("SDIFFSTORE", destination, keys) - return r.parseInteger() + return r.readInteger() proc sinter*(r: TRedis, keys: varargs[string]): TRedisList = ## Intersect multiple sets r.sendCommand("SINTER", keys) - return r.parseMultiBulk() + return r.readArray() proc sinterstore*(r: TRedis, destination: string, keys: varargs[string]): TRedisInteger = ## Intersect multiple sets and store the resulting set in a key r.sendCommand("SINTERSTORE", destination, keys) - return r.parseInteger() + return r.readInteger() proc sismember*(r: TRedis, key: string, member: string): TRedisInteger = ## Determine if a given value is a member of a set r.sendCommand("SISMEMBER", key, member) - return r.parseInteger() + return r.readInteger() proc smembers*(r: TRedis, key: string): TRedisList = ## Get all the members in a set r.sendCommand("SMEMBERS", key) - return r.parseMultiBulk() + return r.readArray() proc smove*(r: TRedis, source: string, destination: string, member: string): TRedisInteger = ## Move a member from one set to another r.sendCommand("SMOVE", source, destination, member) - return r.parseInteger() + return r.readInteger() proc spop*(r: TRedis, key: string): TRedisString = ## Remove and return a random member from a set r.sendCommand("SPOP", key) - return r.parseBulk() + return r.readBulkString() proc srandmember*(r: TRedis, key: string): TRedisString = ## Get a random member from a set r.sendCommand("SRANDMEMBER", key) - return r.parseBulk() + return r.readBulkString() proc srem*(r: TRedis, key: string, member: string): TRedisInteger = ## Remove a member from a set r.sendCommand("SREM", key, member) - return r.parseInteger() + return r.readInteger() proc sunion*(r: TRedis, keys: varargs[string]): TRedisList = ## Add multiple sets r.sendCommand("SUNION", keys) - return r.parseMultiBulk() + return r.readArray() proc sunionstore*(r: TRedis, destination: string, key: varargs[string]): TRedisInteger = ## Add multiple sets and store the resulting set in a key r.sendCommand("SUNIONSTORE", destination, key) - return r.parseInteger() + return r.readInteger() # Sorted sets proc zadd*(r: TRedis, key: string, score: int, member: string): TRedisInteger = ## Add a member to a sorted set, or update its score if it already exists r.sendCommand("ZADD", key, $score, member) - return r.parseInteger() + return r.readInteger() proc zcard*(r: TRedis, key: string): TRedisInteger = ## Get the number of members in a sorted set r.sendCommand("ZCARD", key) - return r.parseInteger() + return r.readInteger() proc zcount*(r: TRedis, key: string, min: string, max: string): TRedisInteger = ## Count the members in a sorted set with scores within the given values r.sendCommand("ZCOUNT", key, min, max) - return r.parseInteger() + return r.readInteger() proc zincrby*(r: TRedis, key: string, increment: string, member: string): TRedisString = ## Increment the score of a member in a sorted set r.sendCommand("ZINCRBY", key, increment, member) - return r.parseBulk() + return r.readBulkString() proc zinterstore*(r: TRedis, destination: string, numkeys: string, keys: openarray[string], weights: openarray[string] = [], @@ -605,7 +692,7 @@ proc zinterstore*(r: TRedis, destination: string, numkeys: string, r.sendCommand("ZINTERSTORE", args) - return r.parseInteger() + return r.readInteger() proc zrange*(r: TRedis, key: string, start: string, stop: string, withScores: bool): TRedisList = @@ -614,7 +701,7 @@ proc zrange*(r: TRedis, key: string, start: string, stop: string, r.sendCommand("ZRANGE", key, start, stop) else: r.sendCommand("ZRANGE", "WITHSCORES", key, start, stop) - return r.parseMultiBulk() + return r.readArray() proc zrangebyscore*(r: TRedis, key: string, min: string, max: string, withScore: bool = false, limit: bool = False, @@ -629,29 +716,29 @@ proc zrangebyscore*(r: TRedis, key: string, min: string, max: string, args.add($limitCount) r.sendCommand("ZRANGEBYSCORE", args) - return r.parseMultiBulk() + return r.readArray() proc zrank*(r: TRedis, key: string, member: string): TRedisString = ## Determine the index of a member in a sorted set r.sendCommand("ZRANK", key, member) - return r.parseBulk() + return r.readBulkString() proc zrem*(r: TRedis, key: string, member: string): TRedisInteger = ## Remove a member from a sorted set r.sendCommand("ZREM", key, member) - return r.parseInteger() + return r.readInteger() proc zremrangebyrank*(r: TRedis, key: string, start: string, stop: string): TRedisInteger = ## Remove all members in a sorted set within the given indexes r.sendCommand("ZREMRANGEBYRANK", key, start, stop) - return r.parseInteger() + return r.readInteger() proc zremrangebyscore*(r: TRedis, key: string, min: string, max: string): TRedisInteger = ## Remove all members in a sorted set within the given scores r.sendCommand("ZREMRANGEBYSCORE", key, min, max) - return r.parseInteger() + return r.readInteger() proc zrevrange*(r: TRedis, key: string, start: string, stop: string, withScore: bool): TRedisList = @@ -660,7 +747,7 @@ proc zrevrange*(r: TRedis, key: string, start: string, stop: string, if withScore: r.sendCommand("ZREVRANGE", "WITHSCORE", key, start, stop) else: r.sendCommand("ZREVRANGE", key, start, stop) - return r.parseMultiBulk() + return r.readArray() proc zrevrangebyscore*(r: TRedis, key: string, min: string, max: string, withScore: bool = false, limit: bool = False, @@ -676,18 +763,18 @@ proc zrevrangebyscore*(r: TRedis, key: string, min: string, max: string, args.add($limitCount) r.sendCommand("ZREVRANGEBYSCORE", args) - return r.parseMultiBulk() + return r.readArray() proc zrevrank*(r: TRedis, key: string, member: string): TRedisString = ## Determine the index of a member in a sorted set, with ## scores ordered from high to low r.sendCommand("ZREVRANK", key, member) - return r.parseBulk() + return r.readBulkString() proc zscore*(r: TRedis, key: string, member: string): TRedisString = ## Get the score associated with the given member in a sorted set r.sendCommand("ZSCORE", key, member) - return r.parseBulk() + return r.readBulkString() proc zunionstore*(r: TRedis, destination: string, numkeys: string, keys: openarray[string], weights: openarray[string] = [], @@ -705,7 +792,7 @@ proc zunionstore*(r: TRedis, destination: string, numkeys: string, r.sendCommand("ZUNIONSTORE", args) - return r.parseInteger() + return r.readInteger() # Pub/Sub @@ -720,7 +807,7 @@ proc psubscribe*(r: TRedis, pattern: openarray[string]): ???? = proc publish*(r: TRedis, channel: string, message: string): TRedisInteger = ## Post a message to a channel r.socket.send("PUBLISH $# $#\c\L" % [channel, message]) - return r.parseInteger() + return r.readInteger() proc punsubscribe*(r: TRedis, [pattern: openarray[string], : string): ???? = ## Stop listening for messages posted to channels matching the given patterns @@ -744,92 +831,96 @@ proc unsubscribe*(r: TRedis, [channel: openarray[string], : string): ???? = proc discardMulti*(r: TRedis) = ## Discard all commands issued after MULTI r.sendCommand("DISCARD") - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc exec*(r: TRedis): TRedisList = ## Execute all commands issued after MULTI - r.sendCommand("EXEC") - - return r.parseMultiBulk() + r.sendCommand("EXEC") + r.pipeline.enabled = false + # Will reply with +OK for MULTI/EXEC and +QUEUED for every command + # between, then with the results + return r.flushPipeline(true) + proc multi*(r: TRedis) = ## Mark the start of a transaction block + r.startPipelining() r.sendCommand("MULTI") - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc unwatch*(r: TRedis) = ## Forget about all watched keys r.sendCommand("UNWATCH") - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc watch*(r: TRedis, key: varargs[string]) = ## Watch the given keys to determine execution of the MULTI/EXEC block r.sendCommand("WATCH", key) - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) # Connection proc auth*(r: TRedis, password: string) = ## Authenticate to the server r.sendCommand("AUTH", password) - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc echoServ*(r: TRedis, message: string): TRedisString = ## Echo the given string r.sendCommand("ECHO", message) - return r.parseBulk() + return r.readBulkString() proc ping*(r: TRedis): TRedisStatus = ## Ping the server r.sendCommand("PING") - return r.parseStatus() + return r.readStatus() proc quit*(r: TRedis) = ## Close the connection r.sendCommand("QUIT") - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc select*(r: TRedis, index: int): TRedisStatus = ## Change the selected database for the current connection r.sendCommand("SELECT", $index) - return r.parseStatus() + return r.readStatus() # Server proc bgrewriteaof*(r: TRedis) = ## Asynchronously rewrite the append-only file r.sendCommand("BGREWRITEAOF") - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc bgsave*(r: TRedis) = ## Asynchronously save the dataset to disk r.sendCommand("BGSAVE") - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc configGet*(r: TRedis, parameter: string): TRedisList = ## Get the value of a configuration parameter r.sendCommand("CONFIG", "GET", parameter) - return r.parseMultiBulk() + return r.readArray() proc configSet*(r: TRedis, parameter: string, value: string) = ## Set a configuration parameter to the given value r.sendCommand("CONFIG", "SET", parameter, value) - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc configResetStat*(r: TRedis) = ## Reset the stats returned by INFO r.sendCommand("CONFIG", "RESETSTAT") - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc dbsize*(r: TRedis): TRedisInteger = ## Return the number of keys in the selected database r.sendCommand("DBSIZE") - return r.parseInteger() + return r.readInteger() proc debugObject*(r: TRedis, key: string): TRedisStatus = ## Get debugging information about a key r.sendCommand("DEBUG", "OBJECT", key) - return r.parseStatus() + return r.readStatus() proc debugSegfault*(r: TRedis) = ## Make the server crash @@ -838,34 +929,34 @@ proc debugSegfault*(r: TRedis) = proc flushall*(r: TRedis): TRedisStatus = ## Remove all keys from all databases r.sendCommand("FLUSHALL") - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc flushdb*(r: TRedis): TRedisStatus = ## Remove all keys from the current database r.sendCommand("FLUSHDB") - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc info*(r: TRedis): TRedisString = ## Get information and statistics about the server r.sendCommand("INFO") - return r.parseBulk() + return r.readBulkString() proc lastsave*(r: TRedis): TRedisInteger = ## Get the UNIX time stamp of the last successful save to disk r.sendCommand("LASTSAVE") - return r.parseInteger() + return r.readInteger() discard """ proc monitor*(r: TRedis) = ## Listen for all requests received by the server in real time r.socket.send("MONITOR\c\L") - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) """ proc save*(r: TRedis) = ## Synchronously save the dataset to disk r.sendCommand("SAVE") - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) proc shutdown*(r: TRedis) = ## Synchronously save the dataset to disk and then shut down the server @@ -877,7 +968,7 @@ proc shutdown*(r: TRedis) = proc slaveof*(r: TRedis, host: string, port: string) = ## Make the server a slave of another instance, or promote it as master r.sendCommand("SLAVEOF", host, port) - raiseNoOK(r.parseStatus()) + raiseNoOK(r.readStatus(), r.pipeline.enabled) iterator hPairs*(r: TRedis, key: string): tuple[key, value: string] = ## Iterator for keys and values in a hash. @@ -890,33 +981,72 @@ iterator hPairs*(r: TRedis, key: string): tuple[key, value: string] = else: yield (k, i) k = "" - -when false: - # sorry, deactivated for the test suite - var r = open() - r.auth("pass") +proc someTests(r: TRedis, how: TSendMode):seq[string] = + var list:seq[string] = @[] + if how == pipelined: + r.startPipelining() + elif how == multiple: + r.multi() + r.setk("nim:test", "Testing something.") r.setk("nim:utf8", "こんにちは") r.setk("nim:esc", "\\ths ągt\\") - - echo r.get("nim:esc") - echo r.incr("nim:int") - echo r.incr("nim:int") - echo r.get("nim:int") - echo r.get("nim:utf8") - echo repr(r.get("blahasha")) - echo r.randomKey() - + r.setk("nim:int", "1") + list.add(r.get("nim:esc")) + list.add($(r.incr("nim:int"))) + list.add(r.get("nim:int")) + list.add(r.get("nim:utf8")) + list.add($(r.hSet("test1", "name", "A Test"))) + var res = r.hGetAll("test1") + for r in res: + list.add(r) + list.add(r.get("invalid_key")) + list.add($(r.lpush("mylist","itema"))) + list.add($(r.lpush("mylist","itemb"))) + r.ltrim("mylist",0,1) var p = r.lrange("mylist", 0, -1) + for i in items(p): - echo(" ", i) + if not isNil(i): + list.add(i) - echo(r.debugObject("test")) + list.add(r.debugObject("mylist")) r.configSet("timeout", "299") - for i in items(r.configGet("timeout")): echo ">> ", i + var g = r.configGet("timeout") + for i in items(g): + list.add(i) + + list.add(r.echoServ("BLAH")) + + case how + of normal: + return list + of pipelined: + return r.flushPipeline() + of multiple: + return r.exec() + +proc assertListsIdentical(listA, listB: seq[string]) = + assert(listA.len == listB.len) + var i = 0 + for item in listA: + assert(item == listB[i]) + i = i + 1 + +when isMainModule: + when false: + var r = open() + + # Test with no pipelining + var listNormal = r.someTests(normal) - echo r.echoServ("BLAH") + # Test with pipelining enabled + var listPipelined = r.someTests(pipelined) + assertListsIdentical(listNormal, listPipelined) + # Test with multi/exec() (automatic pipelining) + var listMulti = r.someTests(multiple) + assertListsIdentical(listNormal, listMulti) diff --git a/lib/pure/scgi.nim b/lib/pure/scgi.nim index a6a0faabc..45f837833 100644 --- a/lib/pure/scgi.nim +++ b/lib/pure/scgi.nim @@ -26,6 +26,8 @@ ## **Warning:** The API of this module is unstable, and therefore is subject ## to change. +include "system/inclrtl" + import sockets, strutils, os, strtabs, asyncio type @@ -82,7 +84,7 @@ type TAsyncScgiState = object handleRequest: proc (client: PAsyncSocket, - input: string, headers: PStringTable) {.closure.} + input: string, headers: PStringTable) {.closure,gcsafe.} asyncServer: PAsyncSocket disp: PDispatcher PAsyncScgiState* = ref TAsyncScgiState @@ -150,7 +152,7 @@ proc writeStatusOkTextContent*(c: TSocket, contentType = "text/html") = "Content-Type: $1\r\L\r\L" % contentType) proc run*(handleRequest: proc (client: TSocket, input: string, - headers: PStringTable): bool {.nimcall.}, + headers: PStringTable): bool {.nimcall,gcsafe.}, port = TPort(4000)) = ## encapsulates the SCGI object and main loop. var s: TScgiState @@ -246,7 +248,8 @@ proc handleAccept(sock: PAsyncSocket, s: PAsyncScgiState) = s.disp.register(client) proc open*(handleRequest: proc (client: PAsyncSocket, - input: string, headers: PStringTable) {.closure.}, + input: string, headers: PStringTable) {. + closure, gcsafe.}, port = TPort(4000), address = "127.0.0.1", reuseAddr = false): PAsyncScgiState = ## Creates an ``PAsyncScgiState`` object which serves as a SCGI server. diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index b6acc329f..8d96cbaaf 100644 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -24,6 +24,8 @@ ## Asynchronous sockets are supported, however a better alternative is to use ## the `asyncio <asyncio.html>`_ module. +include "system/inclrtl" + {.deadCodeElim: on.} when hostOS == "solaris": @@ -544,7 +546,7 @@ proc acceptAddr*(server: TSocket, client: var TSocket, address: var string) {. else: SSLError("Unknown error") -proc setBlocking*(s: TSocket, blocking: bool) {.tags: [].} +proc setBlocking*(s: TSocket, blocking: bool) {.tags: [], gcsafe.} ## Sets blocking mode on socket when defined(ssl): diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 3b6dc87a5..63622a26c 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -12,6 +12,8 @@ ## interface for Nimrod file objects (`TFile`) and strings. Other modules ## may provide other implementations for this standard stream interface. +include "system/inclrtl" + proc newEIO(msg: string): ref EIO = new(result) result.msg = msg @@ -23,15 +25,15 @@ type ## here shouldn't be used directly. They are ## accessible so that a stream implementation ## can override them. - closeImpl*: proc (s: PStream) {.nimcall, tags: [].} - atEndImpl*: proc (s: PStream): bool {.nimcall, tags: [].} - setPositionImpl*: proc (s: PStream, pos: int) {.nimcall, tags: [].} - getPositionImpl*: proc (s: PStream): int {.nimcall, tags: [].} + closeImpl*: proc (s: PStream) {.nimcall, tags: [], gcsafe.} + atEndImpl*: proc (s: PStream): bool {.nimcall, tags: [], gcsafe.} + setPositionImpl*: proc (s: PStream, pos: int) {.nimcall, tags: [], gcsafe.} + getPositionImpl*: proc (s: PStream): int {.nimcall, tags: [], gcsafe.} readDataImpl*: proc (s: PStream, buffer: pointer, - bufLen: int): int {.nimcall, tags: [FReadIO].} + bufLen: int): int {.nimcall, tags: [FReadIO], gcsafe.} writeDataImpl*: proc (s: PStream, buffer: pointer, bufLen: int) {.nimcall, - tags: [FWriteIO].} - flushImpl*: proc (s: PStream) {.nimcall, tags: [FWriteIO].} + tags: [FWriteIO], gcsafe.} + flushImpl*: proc (s: PStream) {.nimcall, tags: [FWriteIO], gcsafe.} proc flush*(s: PStream) = ## flushes the buffers that the stream `s` might use. diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 2fce235e2..fdff06b2a 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -135,38 +135,38 @@ type months*: int ## The number of months years*: int ## The number of years -proc getTime*(): TTime {.tags: [FTime].} +proc getTime*(): TTime {.tags: [FTime], gcsafe.} ## gets the current calendar time as a UNIX epoch value (number of seconds ## elapsed since 1970) with integer precission. Use epochTime for higher ## resolution. -proc getLocalTime*(t: TTime): TTimeInfo {.tags: [FTime], raises: [].} +proc getLocalTime*(t: TTime): TTimeInfo {.tags: [FTime], raises: [], gcsafe.} ## converts the calendar time `t` to broken-time representation, ## expressed relative to the user's specified time zone. -proc getGMTime*(t: TTime): TTimeInfo {.tags: [FTime], raises: [].} +proc getGMTime*(t: TTime): TTimeInfo {.tags: [FTime], raises: [], gcsafe.} ## converts the calendar time `t` to broken-down time representation, ## expressed in Coordinated Universal Time (UTC). -proc timeInfoToTime*(timeInfo: TTimeInfo): TTime {.tags: [].} +proc timeInfoToTime*(timeInfo: TTimeInfo): TTime {.tags: [], gcsafe.} ## converts a broken-down time structure to ## calendar time representation. The function ignores the specified ## contents of the structure members `weekday` and `yearday` and recomputes ## them from the other information in the broken-down time structure. -proc fromSeconds*(since1970: float): TTime {.tags: [], raises: [].} +proc fromSeconds*(since1970: float): TTime {.tags: [], raises: [], gcsafe.} ## Takes a float which contains the number of seconds since the unix epoch and ## returns a time object. -proc fromSeconds*(since1970: int64): TTime {.tags: [], raises: [].} = +proc fromSeconds*(since1970: int64): TTime {.tags: [], raises: [], gcsafe.} = ## Takes an int which contains the number of seconds since the unix epoch and ## returns a time object. fromSeconds(float(since1970)) -proc toSeconds*(time: TTime): float {.tags: [], raises: [].} +proc toSeconds*(time: TTime): float {.tags: [], raises: [], gcsafe.} ## Returns the time in seconds since the unix epoch. -proc `$` *(timeInfo: TTimeInfo): string {.tags: [], raises: [].} +proc `$` *(timeInfo: TTimeInfo): string {.tags: [], raises: [], gcsafe.} ## converts a `TTimeInfo` object to a string representation. -proc `$` *(time: TTime): string {.tags: [], raises: [].} +proc `$` *(time: TTime): string {.tags: [], raises: [], gcsafe.} ## converts a calendar time to a string representation. proc `-`*(a, b: TTime): int64 {. @@ -189,14 +189,15 @@ proc `==`*(a, b: TTime): bool {. result = a - b == 0 when not defined(JS): - proc getTzname*(): tuple[nonDST, DST: string] {.tags: [FTime], raises: [].} + proc getTzname*(): tuple[nonDST, DST: string] {.tags: [FTime], raises: [], + gcsafe.} ## returns the local timezone; ``nonDST`` is the name of the local non-DST ## timezone, ``DST`` is the name of the local DST timezone. -proc getTimezone*(): int {.tags: [FTime], raises: [].} +proc getTimezone*(): int {.tags: [FTime], raises: [], gcsafe.} ## returns the offset of the local (non-DST) timezone in seconds west of UTC. -proc getStartMilsecs*(): int {.deprecated, tags: [FTime].} +proc getStartMilsecs*(): int {.deprecated, tags: [FTime], gcsafe.} ## get the miliseconds from the start of the program. **Deprecated since ## version 0.8.10.** Use ``epochTime`` or ``cpuTime`` instead. diff --git a/lib/system.nim b/lib/system.nim index 3cb6b08d5..273820b82 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -77,7 +77,7 @@ type TNumber* = TInteger|TReal ## type class matching all number types - + proc defined*(x: expr): bool {.magic: "Defined", noSideEffect.} ## Special compile-time procedure that checks whether `x` is ## defined. `x` has to be an identifier or a qualified identifier. @@ -188,6 +188,13 @@ when not defined(niminheritable): when not defined(nimunion): {.pragma: unchecked.} +when defined(nimNewShared): + type + `shared`* {.magic: "Shared".} + guarded* {.magic: "Guarded".} + +include "system/inclrtl" + const NoFakeVars* = defined(NimrodVM) ## true if the backend doesn't support \ ## "fake variables" like 'var EBADF {.importc.}: cint'. @@ -784,7 +791,8 @@ proc `is` *[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.} ## assert(test[int](3) == 3) ## assert(test[string]("xyz") == 0) template `isnot` *(x, y: expr): expr {.immediate.} = not (x is y) - ## Negated version of `is`. Equivalent to `not(is(x,y))` + ## Negated version of `is`. Equivalent to ``not(x is y)``. + proc `of` *[T, S](x: T, y: S): bool {.magic: "Of", noSideEffect.} ## Checks if `x` has a type of `y` ## @@ -1021,8 +1029,6 @@ template sysAssert(cond: bool, msg: string) = echo "[SYSASSERT] ", msg quit 1 -include "system/inclrtl" - when not defined(JS) and not defined(nimrodVm) and hostOS != "standalone": include "system/cgprocs" @@ -1201,20 +1207,20 @@ proc substr*(s: string, first, last: int): string {. ## or `limit`:idx: a string's length. when not defined(nimrodVM): - proc zeroMem*(p: pointer, size: int) {.importc, noDecl.} + proc zeroMem*(p: pointer, size: int) {.importc, noDecl, gcsafe.} ## overwrites the contents of the memory at ``p`` with the value 0. ## Exactly ``size`` bytes will be overwritten. Like any procedure ## dealing with raw memory this is *unsafe*. proc copyMem*(dest, source: pointer, size: int) {. - importc: "memcpy", header: "<string.h>".} + importc: "memcpy", header: "<string.h>", gcsafe.} ## copies the contents from the memory at ``source`` to the memory ## at ``dest``. Exactly ``size`` bytes will be copied. The memory ## regions may not overlap. Like any procedure dealing with raw ## memory this is *unsafe*. proc moveMem*(dest, source: pointer, size: int) {. - importc: "memmove", header: "<string.h>".} + importc: "memmove", header: "<string.h>", gcsafe.} ## copies the contents from the memory at ``source`` to the memory ## at ``dest``. Exactly ``size`` bytes will be copied. The memory ## regions may overlap, ``moveMem`` handles this case appropriately @@ -1229,14 +1235,14 @@ when not defined(nimrodVM): ## *unsafe*. when hostOS != "standalone": - proc alloc*(size: int): pointer {.noconv, rtl, tags: [].} + proc alloc*(size: int): pointer {.noconv, rtl, tags: [], gcsafe.} ## allocates a new memory block with at least ``size`` bytes. The ## block has to be freed with ``realloc(block, 0)`` or ## ``dealloc(block)``. The block is not initialized, so reading ## from it before writing to it is undefined behaviour! ## The allocated memory belongs to its allocating thread! ## Use `allocShared` to allocate from a shared heap. - proc createU*(T: typedesc, size = 1.Positive): ptr T {.inline.} = + proc createU*(T: typedesc, size = 1.Positive): ptr T {.inline, gcsafe.} = ## allocates a new memory block with at least ``T.sizeof * size`` ## bytes. The block has to be freed with ``resize(block, 0)`` or ## ``free(block)``. The block is not initialized, so reading @@ -1244,14 +1250,14 @@ when not defined(nimrodVM): ## The allocated memory belongs to its allocating thread! ## Use `createSharedU` to allocate from a shared heap. cast[ptr T](alloc(T.sizeof * size)) - proc alloc0*(size: int): pointer {.noconv, rtl, tags: [].} + proc alloc0*(size: int): pointer {.noconv, rtl, tags: [], gcsafe.} ## allocates a new memory block with at least ``size`` bytes. The ## block has to be freed with ``realloc(block, 0)`` or ## ``dealloc(block)``. The block is initialized with all bytes ## containing zero, so it is somewhat safer than ``alloc``. ## The allocated memory belongs to its allocating thread! ## Use `allocShared0` to allocate from a shared heap. - proc create*(T: typedesc, size = 1.Positive): ptr T {.inline.} = + proc create*(T: typedesc, size = 1.Positive): ptr T {.inline, gcsafe.} = ## allocates a new memory block with at least ``T.sizeof * size`` ## bytes. The block has to be freed with ``resize(block, 0)`` or ## ``free(block)``. The block is initialized with all bytes @@ -1259,7 +1265,8 @@ when not defined(nimrodVM): ## The allocated memory belongs to its allocating thread! ## Use `createShared` to allocate from a shared heap. cast[ptr T](alloc0(T.sizeof * size)) - proc realloc*(p: pointer, newSize: int): pointer {.noconv, rtl, tags: [].} + proc realloc*(p: pointer, newSize: int): pointer {.noconv, rtl, tags: [], + gcsafe.} ## grows or shrinks a given memory block. If p is **nil** then a new ## memory block is returned. In either way the block has at least ## ``newSize`` bytes. If ``newSize == 0`` and p is not **nil** @@ -1267,7 +1274,7 @@ when not defined(nimrodVM): ## be freed with ``dealloc``. ## The allocated memory belongs to its allocating thread! ## Use `reallocShared` to reallocate from a shared heap. - proc resize*[T](p: ptr T, newSize: Natural): ptr T {.inline.} = + proc resize*[T](p: ptr T, newSize: Natural): ptr T {.inline, gcsafe.} = ## grows or shrinks a given memory block. If p is **nil** then a new ## memory block is returned. In either way the block has at least ## ``T.sizeof * newSize`` bytes. If ``newSize == 0`` and p is not @@ -1276,7 +1283,7 @@ when not defined(nimrodVM): ## its allocating thread! ## Use `resizeShared` to reallocate from a shared heap. cast[ptr T](realloc(p, T.sizeof * newSize)) - proc dealloc*(p: pointer) {.noconv, rtl, tags: [].} + proc dealloc*(p: pointer) {.noconv, rtl, tags: [], gcsafe.} ## frees the memory allocated with ``alloc``, ``alloc0`` or ## ``realloc``. This procedure is dangerous! If one forgets to ## free the memory a leak occurs; if one tries to access freed @@ -1284,22 +1291,23 @@ when not defined(nimrodVM): ## or other memory may be corrupted. ## The freed memory must belong to its allocating thread! ## Use `deallocShared` to deallocate from a shared heap. - proc free*[T](p: ptr T) {.inline.} = + proc free*[T](p: ptr T) {.inline, gcsafe.} = dealloc(p) - proc allocShared*(size: int): pointer {.noconv, rtl.} + proc allocShared*(size: int): pointer {.noconv, rtl, gcsafe.} ## allocates a new memory block on the shared heap with at ## least ``size`` bytes. The block has to be freed with ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. The block ## is not initialized, so reading from it before writing to it is ## undefined behaviour! - proc createSharedU*(T: typedesc, size = 1.Positive): ptr T {.inline.} = + proc createSharedU*(T: typedesc, size = 1.Positive): ptr T {.inline, + gcsafe.} = ## allocates a new memory block on the shared heap with at ## least ``T.sizeof * size`` bytes. The block has to be freed with ## ``resizeShared(block, 0)`` or ``freeShared(block)``. The block ## is not initialized, so reading from it before writing to it is ## undefined behaviour! cast[ptr T](allocShared(T.sizeof * size)) - proc allocShared0*(size: int): pointer {.noconv, rtl.} + proc allocShared0*(size: int): pointer {.noconv, rtl, gcsafe.} ## allocates a new memory block on the shared heap with at ## least ``size`` bytes. The block has to be freed with ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. @@ -1312,7 +1320,8 @@ when not defined(nimrodVM): ## The block is initialized with all bytes ## containing zero, so it is somewhat safer than ``createSharedU``. cast[ptr T](allocShared0(T.sizeof * size)) - proc reallocShared*(p: pointer, newSize: int): pointer {.noconv, rtl.} + proc reallocShared*(p: pointer, newSize: int): pointer {.noconv, rtl, + gcsafe.} ## grows or shrinks a given memory block on the heap. If p is **nil** ## then a new memory block is returned. In either way the block has at ## least ``newSize`` bytes. If ``newSize == 0`` and p is not **nil** @@ -1325,13 +1334,13 @@ when not defined(nimrodVM): ## not **nil** ``resizeShared`` calls ``freeShared(p)``. In other ## cases the block has to be freed with ``freeShared``. cast[ptr T](reallocShared(p, T.sizeof * newSize)) - proc deallocShared*(p: pointer) {.noconv, rtl.} + proc deallocShared*(p: pointer) {.noconv, rtl, gcsafe.} ## frees the memory allocated with ``allocShared``, ``allocShared0`` or ## ``reallocShared``. This procedure is dangerous! If one forgets to ## free the memory a leak occurs; if one tries to access freed ## memory (or just freeing it twice!) a core dump may happen ## or other memory may be corrupted. - proc freeShared*[T](p: ptr T) {.inline.} = + proc freeShared*[T](p: ptr T) {.inline, gcsafe.} = ## frees the memory allocated with ``createShared``, ``createSharedU`` or ## ``resizeShared``. This procedure is dangerous! If one forgets to ## free the memory a leak occurs; if one tries to access freed @@ -1892,7 +1901,7 @@ const nimrodStackTrace = compileOption("stacktrace") # of the code var - globalRaiseHook*: proc (e: ref E_Base): bool {.nimcall.} + globalRaiseHook*: proc (e: ref E_Base): bool {.nimcall, gcsafe.} ## with this hook you can influence exception handling on a global level. ## If not nil, every 'raise' statement ends up calling this hook. Ordinary ## application code should never set this hook! You better know what you @@ -1900,7 +1909,7 @@ var ## exception is caught and does not propagate further through the call ## stack. - localRaiseHook* {.threadvar.}: proc (e: ref E_Base): bool {.nimcall.} + localRaiseHook* {.threadvar.}: proc (e: ref E_Base): bool {.nimcall, gcsafe.} ## with this hook you can influence exception handling on a ## thread local level. ## If not nil, every 'raise' statement ends up calling this hook. Ordinary @@ -1908,7 +1917,7 @@ var ## do when setting this. If ``localRaiseHook`` returns false, the exception ## is caught and does not propagate further through the call stack. - outOfMemHook*: proc () {.nimcall, tags: [].} + outOfMemHook*: proc () {.nimcall, tags: [], gcsafe.} ## set this variable to provide a procedure that should be called ## in case of an `out of memory`:idx: event. The standard handler ## writes an error message and terminates the program. `outOfMemHook` can @@ -1959,7 +1968,7 @@ elif hostOS != "standalone": inc(i) {.pop.} -proc echo*[T](x: varargs[T, `$`]) {.magic: "Echo", tags: [FWriteIO].} +proc echo*[T](x: varargs[T, `$`]) {.magic: "Echo", tags: [FWriteIO], gcsafe.} ## special built-in that takes a variable number of arguments. Each argument ## is converted to a string via ``$``, so it works for user-defined ## types that have an overloaded ``$`` operator. @@ -2006,7 +2015,7 @@ when not defined(sysFatal): e.msg = message & arg raise e -proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo".} +proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo", gcsafe.} ## get type information for `x`. Ordinary code should not use this, but ## the `typeinfo` module instead. @@ -2113,14 +2122,15 @@ when not defined(JS): #and not defined(NimrodVM): ## `useStdoutAsStdmsg` compile-time switch. proc open*(f: var TFile, filename: string, - mode: TFileMode = fmRead, bufSize: int = -1): bool {.tags: [].} + mode: TFileMode = fmRead, bufSize: int = -1): bool {.tags: [], + gcsafe.} ## Opens a file named `filename` with given `mode`. ## ## Default mode is readonly. Returns true iff the file could be opened. ## This throws no exception if the file could not be opened. proc open*(f: var TFile, filehandle: TFileHandle, - mode: TFileMode = fmRead): bool {.tags: [].} + mode: TFileMode = fmRead): bool {.tags: [], gcsafe.} ## Creates a ``TFile`` from a `filehandle` with given `mode`. ## ## Default mode is readonly. Returns true iff the file could be opened. @@ -2135,7 +2145,7 @@ when not defined(JS): #and not defined(NimrodVM): sysFatal(EIO, "cannot open: ", filename) proc reopen*(f: TFile, filename: string, mode: TFileMode = fmRead): bool {. - tags: [].} + tags: [], gcsafe.} ## reopens the file `f` with given `filename` and `mode`. This ## is often used to redirect the `stdin`, `stdout` or `stderr` ## file variables. @@ -2145,7 +2155,7 @@ when not defined(JS): #and not defined(NimrodVM): proc close*(f: TFile) {.importc: "fclose", header: "<stdio.h>", tags: [].} ## Closes the file. - proc endOfFile*(f: TFile): bool {.tags: [].} + proc endOfFile*(f: TFile): bool {.tags: [], gcsafe.} ## Returns true iff `f` is at the end. proc readChar*(f: TFile): char {. @@ -2155,39 +2165,40 @@ when not defined(JS): #and not defined(NimrodVM): importc: "fflush", header: "<stdio.h>", tags: [FWriteIO].} ## Flushes `f`'s buffer. - proc readAll*(file: TFile): TaintedString {.tags: [FReadIO].} + proc readAll*(file: TFile): TaintedString {.tags: [FReadIO], gcsafe.} ## Reads all data from the stream `file`. ## ## Raises an IO exception in case of an error. It is an error if the ## current file position is not at the beginning of the file. - proc readFile*(filename: string): TaintedString {.tags: [FReadIO].} + proc readFile*(filename: string): TaintedString {.tags: [FReadIO], gcsafe.} ## Opens a file named `filename` for reading. Then calls `readAll` ## and closes the file afterwards. Returns the string. ## Raises an IO exception in case of an error. - proc writeFile*(filename, content: string) {.tags: [FWriteIO].} + proc writeFile*(filename, content: string) {.tags: [FWriteIO], gcsafe.} ## Opens a file named `filename` for writing. Then writes the ## `content` completely to the file and closes the file afterwards. ## Raises an IO exception in case of an error. - proc write*(f: TFile, r: float32) {.tags: [FWriteIO].} - proc write*(f: TFile, i: int) {.tags: [FWriteIO].} - proc write*(f: TFile, i: BiggestInt) {.tags: [FWriteIO].} - proc write*(f: TFile, r: BiggestFloat) {.tags: [FWriteIO].} - proc write*(f: TFile, s: string) {.tags: [FWriteIO].} - proc write*(f: TFile, b: bool) {.tags: [FWriteIO].} - proc write*(f: TFile, c: char) {.tags: [FWriteIO].} - proc write*(f: TFile, c: cstring) {.tags: [FWriteIO].} - proc write*(f: TFile, a: varargs[string, `$`]) {.tags: [FWriteIO].} + proc write*(f: TFile, r: float32) {.tags: [FWriteIO], gcsafe.} + proc write*(f: TFile, i: int) {.tags: [FWriteIO], gcsafe.} + proc write*(f: TFile, i: BiggestInt) {.tags: [FWriteIO], gcsafe.} + proc write*(f: TFile, r: BiggestFloat) {.tags: [FWriteIO], gcsafe.} + proc write*(f: TFile, s: string) {.tags: [FWriteIO], gcsafe.} + proc write*(f: TFile, b: bool) {.tags: [FWriteIO], gcsafe.} + proc write*(f: TFile, c: char) {.tags: [FWriteIO], gcsafe.} + proc write*(f: TFile, c: cstring) {.tags: [FWriteIO], gcsafe.} + proc write*(f: TFile, a: varargs[string, `$`]) {.tags: [FWriteIO], gcsafe.} ## Writes a value to the file `f`. May throw an IO exception. - proc readLine*(f: TFile): TaintedString {.tags: [FReadIO].} + proc readLine*(f: TFile): TaintedString {.tags: [FReadIO], gcsafe.} ## reads a line of text from the file `f`. May throw an IO exception. ## A line of text may be delimited by ``CR``, ``LF`` or ## ``CRLF``. The newline character(s) are not part of the returned string. - proc readLine*(f: TFile, line: var TaintedString): bool {.tags: [FReadIO].} + proc readLine*(f: TFile, line: var TaintedString): bool {.tags: [FReadIO], + gcsafe.} ## reads a line of text from the file `f` into `line`. `line` must not be ## ``nil``! May throw an IO exception. ## A line of text may be delimited by ``CR``, ``LF`` or @@ -2195,53 +2206,59 @@ when not defined(JS): #and not defined(NimrodVM): ## Returns ``false`` if the end of the file has been reached, ``true`` ## otherwise. If ``false`` is returned `line` contains no new data. - proc writeln*[Ty](f: TFile, x: varargs[Ty, `$`]) {.inline, tags: [FWriteIO].} - ## writes the values `x` to `f` and then writes "\n". - ## May throw an IO exception. + when not defined(booting): + proc writeln*[Ty](f: TFile, x: varargs[Ty, `$`]) {.inline, + tags: [FWriteIO], gcsafe.} + ## writes the values `x` to `f` and then writes "\n". + ## May throw an IO exception. + else: + proc writeln*[Ty](f: TFile, x: varargs[Ty, `$`]) {.inline, + tags: [FWriteIO].} - proc getFileSize*(f: TFile): int64 {.tags: [FReadIO].} + proc getFileSize*(f: TFile): int64 {.tags: [FReadIO], gcsafe.} ## retrieves the file size (in bytes) of `f`. proc readBytes*(f: TFile, a: var openArray[int8], start, len: int): int {. - tags: [FReadIO].} + tags: [FReadIO], gcsafe.} ## reads `len` bytes into the buffer `a` starting at ``a[start]``. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. proc readChars*(f: TFile, a: var openArray[char], start, len: int): int {. - tags: [FReadIO].} + tags: [FReadIO], gcsafe.} ## reads `len` bytes into the buffer `a` starting at ``a[start]``. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. - proc readBuffer*(f: TFile, buffer: pointer, len: int): int {.tags: [FReadIO].} + proc readBuffer*(f: TFile, buffer: pointer, len: int): int {. + tags: [FReadIO], gcsafe.} ## reads `len` bytes into the buffer pointed to by `buffer`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. proc writeBytes*(f: TFile, a: openArray[int8], start, len: int): int {. - tags: [FWriteIO].} + tags: [FWriteIO], gcsafe.} ## writes the bytes of ``a[start..start+len-1]`` to the file `f`. Returns ## the number of actual written bytes, which may be less than `len` in case ## of an error. proc writeChars*(f: TFile, a: openArray[char], start, len: int): int {. - tags: [FWriteIO].} + tags: [FWriteIO], gcsafe.} ## writes the bytes of ``a[start..start+len-1]`` to the file `f`. Returns ## the number of actual written bytes, which may be less than `len` in case ## of an error. proc writeBuffer*(f: TFile, buffer: pointer, len: int): int {. - tags: [FWriteIO].} + tags: [FWriteIO], gcsafe.} ## writes the bytes of buffer pointed to by the parameter `buffer` to the ## file `f`. Returns the number of actual written bytes, which may be less ## than `len` in case of an error. - proc setFilePos*(f: TFile, pos: int64) + proc setFilePos*(f: TFile, pos: int64) {.gcsafe.} ## sets the position of the file pointer that is used for read/write ## operations. The file's first byte has the index zero. - proc getFilePos*(f: TFile): int64 + proc getFilePos*(f: TFile): int64 {.gcsafe.} ## retrieves the current position of the file pointer that is used to ## read from the file `f`. The file's first byte has the index zero. @@ -2284,10 +2301,12 @@ when not defined(JS): #and not defined(NimrodVM): dealloc(a) when not defined(NimrodVM): - proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable.} + proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, + discardable, gcsafe.} ## atomic increment of `memLoc`. Returns the value after the operation. - proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, discardable.} + proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, + discardable, gcsafe.} ## atomic decrement of `memLoc`. Returns the value after the operation. include "system/atomics" @@ -2917,3 +2936,6 @@ when not defined(booting): template isStatic*(x): expr = compiles(static(x)) # checks whether `x` is a value known at compile-time + +when hasThreadSupport: + when hostOS != "standalone": include "system/sysspawn" diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 954485eb4..eaef6cd95 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -722,10 +722,13 @@ proc alloc0(allocator: var TMemRegion, size: int): pointer = zeroMem(result, size) proc dealloc(allocator: var TMemRegion, p: pointer) = + sysAssert(p != nil, "dealloc 0") var x = cast[pointer](cast[TAddress](p) -% sizeof(TFreeCell)) - sysAssert(cast[ptr TFreeCell](x).zeroField == 1, "dealloc 1") + sysAssert(x != nil, "dealloc 1") + sysAssert(isAccessible(allocator, x), "is not accessible") + sysAssert(cast[ptr TFreeCell](x).zeroField == 1, "dealloc 2") rawDealloc(allocator, x) - sysAssert(not isAllocatedPtr(allocator, x), "dealloc 2") + sysAssert(not isAllocatedPtr(allocator, x), "dealloc 3") proc realloc(allocator: var TMemRegion, p: pointer, newsize: int): pointer = if newsize > 0: diff --git a/lib/system/assign.nim b/lib/system/assign.nim index bed8820be..75c749633 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -7,10 +7,11 @@ # distribution, for details about the copyright. # -proc genericResetAux(dest: pointer, n: ptr TNimNode) +proc genericResetAux(dest: pointer, n: ptr TNimNode) {.gcsafe.} -proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) -proc genericAssignAux(dest, src: pointer, n: ptr TNimNode, shallow: bool) = +proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) {.gcsafe.} +proc genericAssignAux(dest, src: pointer, n: ptr TNimNode, + shallow: bool) {.gcsafe.} = var d = cast[TAddress](dest) s = cast[TAddress](src) @@ -139,8 +140,8 @@ proc genericAssignOpenArray(dest, src: pointer, len: int, genericAssign(cast[pointer](d +% i*% mt.base.size), cast[pointer](s +% i*% mt.base.size), mt.base) -proc objectInit(dest: pointer, typ: PNimType) {.compilerProc.} -proc objectInitAux(dest: pointer, n: ptr TNimNode) = +proc objectInit(dest: pointer, typ: PNimType) {.compilerProc, gcsafe.} +proc objectInitAux(dest: pointer, n: ptr TNimNode) {.gcsafe.} = var d = cast[TAddress](dest) case n.kind of nkNone: sysAssert(false, "objectInitAux") @@ -184,7 +185,7 @@ else: mixin destroy for i in countup(0, r.len - 1): destroy(r[i]) -proc genericReset(dest: pointer, mt: PNimType) {.compilerProc.} +proc genericReset(dest: pointer, mt: PNimType) {.compilerProc, gcsafe.} proc genericResetAux(dest: pointer, n: ptr TNimNode) = var d = cast[TAddress](dest) case n.kind diff --git a/lib/system/avltree.nim b/lib/system/avltree.nim index fc965d6aa..bced15d6a 100644 --- a/lib/system/avltree.nim +++ b/lib/system/avltree.nim @@ -51,7 +51,7 @@ proc split(t: var PAvlNode) = t.link[0] = temp inc t.level -proc add(a: var TMemRegion, t: var PAvlNode, key, upperBound: int) = +proc add(a: var TMemRegion, t: var PAvlNode, key, upperBound: int) {.gcsafe.} = if t == bottom: t = allocAvlNode(a, key, upperBound) else: @@ -64,7 +64,7 @@ proc add(a: var TMemRegion, t: var PAvlNode, key, upperBound: int) = skew(t) split(t) -proc del(a: var TMemRegion, t: var PAvlNode, x: int) = +proc del(a: var TMemRegion, t: var PAvlNode, x: int) {.gcsafe.} = if t == bottom: return a.last = t if x <% t.key: diff --git a/lib/system/cgprocs.nim b/lib/system/cgprocs.nim index e30cfa469..d483c61bd 100644 --- a/lib/system/cgprocs.nim +++ b/lib/system/cgprocs.nim @@ -9,7 +9,7 @@ # Headers for procs that the code generator depends on ("compilerprocs") -proc addChar(s: NimString, c: char): NimString {.compilerProc.} +proc addChar(s: NimString, c: char): NimString {.compilerProc, gcsafe.} type TLibHandle = pointer # private type @@ -21,5 +21,5 @@ proc nimGetProcAddr(lib: TLibHandle, name: cstring): TProcAddr {.compilerproc.} proc nimLoadLibraryError(path: string) {.compilerproc, noinline.} -proc setStackBottom(theStackBottom: pointer) {.compilerRtl, noinline.} +proc setStackBottom(theStackBottom: pointer) {.compilerRtl, noinline, gcsafe.} diff --git a/lib/system/channels.nim b/lib/system/channels.nim index bf949529b..e5535dbdc 100644 --- a/lib/system/channels.nim +++ b/lib/system/channels.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2014 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -29,7 +29,7 @@ type region: TMemRegion PRawChannel = ptr TRawChannel TLoadStoreMode = enum mStore, mLoad - TChannel*[TMsg] = TRawChannel ## a channel for thread communication + TChannel* {.gcsafe.}[TMsg] = TRawChannel ## a channel for thread communication const ChannelDeadMask = -2 @@ -49,9 +49,9 @@ proc deinitRawChannel(p: pointer) = deinitSysCond(c.cond) proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel, - mode: TLoadStoreMode) + mode: TLoadStoreMode) {.gcsafe.} proc storeAux(dest, src: pointer, n: ptr TNimNode, t: PRawChannel, - mode: TLoadStoreMode) = + mode: TLoadStoreMode) {.gcsafe.} = var d = cast[TAddress](dest) s = cast[TAddress](src) @@ -209,7 +209,6 @@ proc send*[TMsg](c: var TChannel[TMsg], msg: TMsg) = proc llRecv(q: PRawChannel, res: pointer, typ: PNimType) = # to save space, the generic is as small as possible - acquireSys(q.lock) q.ready = true while q.count <= 0: waitSysCond(q.cond, q.lock) @@ -218,17 +217,29 @@ proc llRecv(q: PRawChannel, res: pointer, typ: PNimType) = releaseSys(q.lock) sysFatal(EInvalidValue, "cannot receive message of wrong type") rawRecv(q, res, typ) - releaseSys(q.lock) proc recv*[TMsg](c: var TChannel[TMsg]): TMsg = ## receives a message from the channel `c`. This blocks until ## a message has arrived! You may use ``peek`` to avoid the blocking. var q = cast[PRawChannel](addr(c)) + acquireSys(q.lock) llRecv(q, addr(result), cast[PNimType](getTypeInfo(result))) + releaseSys(q.lock) + +proc tryRecv*[TMsg](c: var TChannel[TMsg]): tuple[dataAvaliable: bool, + msg: TMsg] = + ## try to receives a message from the channel `c` if available. Otherwise + ## it returns ``(false, default(msg))``. + var q = cast[PRawChannel](addr(c)) + if q.mask != ChannelDeadMask: + lockChannel(q): + llRecv(q, addr(result.msg), cast[PNimType](getTypeInfo(result.msg))) + result.dataAvaliable = true proc peek*[TMsg](c: var TChannel[TMsg]): int = ## returns the current number of messages in the channel `c`. Returns -1 - ## if the channel has been closed. + ## if the channel has been closed. **Note**: This is dangerous to use + ## as it encourages races. It's much better to use ``tryRecv`` instead. var q = cast[PRawChannel](addr(c)) if q.mask != ChannelDeadMask: lockChannel(q): diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index e50ba7b9f..612a9e729 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -11,7 +11,7 @@ # use the heap (and nor exceptions) do not include the GC or memory allocator. var - errorMessageWriter*: (proc(msg: string) {.tags: [FWriteIO].}) + errorMessageWriter*: (proc(msg: string) {.tags: [FWriteIO], gcsafe.}) ## Function that will be called ## instead of stdmsg.write when printing stacktrace. ## Unstable API. @@ -32,10 +32,10 @@ proc showErrorMessage(data: cstring) = else: writeToStdErr(data) -proc chckIndx(i, a, b: int): int {.inline, compilerproc.} -proc chckRange(i, a, b: int): int {.inline, compilerproc.} -proc chckRangeF(x, a, b: float): float {.inline, compilerproc.} -proc chckNil(p: pointer) {.noinline, compilerproc.} +proc chckIndx(i, a, b: int): int {.inline, compilerproc, gcsafe.} +proc chckRange(i, a, b: int): int {.inline, compilerproc, gcsafe.} +proc chckRangeF(x, a, b: float): float {.inline, compilerproc, gcsafe.} +proc chckNil(p: pointer) {.noinline, compilerproc, gcsafe.} var framePtr {.rtlThreadVar.}: PFrame @@ -322,5 +322,5 @@ when not defined(noSignalHandler): proc setControlCHook(hook: proc () {.noconv.}) = # ugly cast, but should work on all architectures: - type TSignalHandler = proc (sig: cint) {.noconv.} + type TSignalHandler = proc (sig: cint) {.noconv, gcsafe.} c_signal(SIGINT, cast[TSignalHandler](hook)) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index ec1760914..3b85fe600 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -51,7 +51,7 @@ type waZctDecRef, waPush, waCycleDecRef, waMarkGray, waScan, waScanBlack, waCollectWhite, - TFinalizer {.compilerproc.} = proc (self: pointer) {.nimcall.} + TFinalizer {.compilerproc.} = proc (self: pointer) {.nimcall, gcsafe.} # A ref type can have a finalizer that is called before the object's # storage is freed. @@ -152,11 +152,11 @@ template gcTrace(cell, state: expr): stmt {.immediate.} = when traceGC: traceCell(cell, state) # forward declarations: -proc collectCT(gch: var TGcHeap) -proc isOnStack*(p: pointer): bool {.noinline.} -proc forAllChildren(cell: PCell, op: TWalkOp) -proc doOperation(p: pointer, op: TWalkOp) -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: TWalkOp) +proc collectCT(gch: var TGcHeap) {.gcsafe.} +proc isOnStack*(p: pointer): bool {.noinline, gcsafe.} +proc forAllChildren(cell: PCell, op: TWalkOp) {.gcsafe.} +proc doOperation(p: pointer, op: TWalkOp) {.gcsafe.} +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: TWalkOp) {.gcsafe.} # we need the prototype here for debugging purposes when hasThreadSupport and hasSharedHeap: @@ -294,7 +294,7 @@ proc initGC() = when useMarkForDebug or useBackupGc: type - TGlobalMarkerProc = proc () {.nimcall.} + TGlobalMarkerProc = proc () {.nimcall, gcsafe.} var globalMarkersLen: int globalMarkers: array[0.. 7_000, TGlobalMarkerProc] @@ -311,7 +311,7 @@ proc cellsetReset(s: var TCellSet) = deinit(s) init(s) -proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: TWalkOp) = +proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: TWalkOp) {.gcsafe.} = var d = cast[TAddress](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) @@ -680,10 +680,11 @@ proc doOperation(p: pointer, op: TWalkOp) = proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = doOperation(d, TWalkOp(op)) -proc collectZCT(gch: var TGcHeap): bool +proc collectZCT(gch: var TGcHeap): bool {.gcsafe.} when useMarkForDebug or useBackupGc: - proc markStackAndRegistersForSweep(gch: var TGcHeap) {.noinline, cdecl.} + proc markStackAndRegistersForSweep(gch: var TGcHeap) {.noinline, cdecl, + gcsafe.} proc collectRoots(gch: var TGcHeap) = for s in elements(gch.cycleRoots): diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index e78a4e5cd..3c99a57e1 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -32,11 +32,11 @@ type # local waMarkPrecise # fast precise marking - TFinalizer {.compilerproc.} = proc (self: pointer) {.nimcall.} + TFinalizer {.compilerproc.} = proc (self: pointer) {.nimcall, gcsafe.} # A ref type can have a finalizer that is called before the object's # storage is freed. - TGlobalMarkerProc = proc () {.nimcall.} + TGlobalMarkerProc = proc () {.nimcall, gcsafe.} TGcStat = object collections: int # number of performed full collections @@ -113,11 +113,11 @@ when BitsPerPage mod (sizeof(int)*8) != 0: {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} # forward declarations: -proc collectCT(gch: var TGcHeap) -proc isOnStack*(p: pointer): bool {.noinline.} -proc forAllChildren(cell: PCell, op: TWalkOp) -proc doOperation(p: pointer, op: TWalkOp) -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: TWalkOp) +proc collectCT(gch: var TGcHeap) {.gcsafe.} +proc isOnStack*(p: pointer): bool {.noinline, gcsafe.} +proc forAllChildren(cell: PCell, op: TWalkOp) {.gcsafe.} +proc doOperation(p: pointer, op: TWalkOp) {.gcsafe.} +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: TWalkOp) {.gcsafe.} # we need the prototype here for debugging purposes proc prepareDealloc(cell: PCell) = @@ -150,7 +150,7 @@ proc initGC() = Init(gch.allocated) init(gch.marked) -proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: TWalkOp) = +proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: TWalkOp) {.gcsafe.} = var d = cast[TAddress](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) diff --git a/lib/system/hti.nim b/lib/system/hti.nim index 9d8ece7df..64174e60f 100644 --- a/lib/system/hti.nim +++ b/lib/system/hti.nim @@ -85,7 +85,7 @@ type base: ptr TNimType node: ptr TNimNode # valid for tyRecord, tyObject, tyTuple, tyEnum finalizer: pointer # the finalizer for the type - marker: proc (p: pointer, op: int) {.nimcall.} # marker proc for GC + marker: proc (p: pointer, op: int) {.nimcall, gcsafe.} # marker proc for GC PNimType = ptr TNimType # node.len may be the ``first`` element of a set diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim index 9831130e2..475a09686 100644 --- a/lib/system/inclrtl.nim +++ b/lib/system/inclrtl.nim @@ -16,6 +16,8 @@ # -> defined(useNimRtl) or appType == "lib" and not defined(createNimRtl) # 3) Exported into nimrtl. # -> appType == "lib" and defined(createNimRtl) +when not defined(nimNewShared): + {.pragma: gcsafe.} when defined(createNimRtl): when defined(useNimRtl): @@ -24,7 +26,7 @@ when defined(createNimRtl): {.error: "nimrtl must be built as a library!".} when defined(createNimRtl): - {.pragma: rtl, exportc: "nimrtl_$1", dynlib.} + {.pragma: rtl, exportc: "nimrtl_$1", dynlib, gcsafe.} {.pragma: inl.} {.pragma: compilerRtl, compilerproc, exportc: "nimrtl_$1", dynlib.} elif defined(useNimRtl): @@ -34,11 +36,11 @@ elif defined(useNimRtl): const nimrtl* = "nimrtl.dylib" else: const nimrtl* = "libnimrtl.so" - {.pragma: rtl, importc: "nimrtl_$1", dynlib: nimrtl.} + {.pragma: rtl, importc: "nimrtl_$1", dynlib: nimrtl, gcsafe.} {.pragma: inl.} {.pragma: compilerRtl, compilerproc, importc: "nimrtl_$1", dynlib: nimrtl.} else: - {.pragma: rtl.} + {.pragma: rtl, gcsafe.} {.pragma: inl, inline.} {.pragma: compilerRtl, compilerproc.} diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 7c1a68bc7..487bac052 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -10,7 +10,7 @@ # The generic ``repr`` procedure. It is an invaluable debugging tool. when not defined(useNimRtl): - proc reprAny(p: pointer, typ: PNimType): string {.compilerRtl.} + proc reprAny(p: pointer, typ: PNimType): string {.compilerRtl, gcsafe.} proc reprInt(x: int64): string {.compilerproc.} = return $x proc reprFloat(x: float): string {.compilerproc.} = return $x @@ -78,7 +78,7 @@ proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = type PByteArray = ptr array[0.. 0xffff, int8] -proc addSetElem(result: var string, elem: int, typ: PNimType) = +proc addSetElem(result: var string, elem: int, typ: PNimType) {.gcsafe.} = case typ.kind of tyEnum: add result, reprEnum(elem, typ) of tyBool: add result, reprBool(bool(elem)) @@ -147,7 +147,7 @@ when not defined(useNimRtl): for i in 0..cl.indent-1: add result, ' ' proc reprAux(result: var string, p: pointer, typ: PNimType, - cl: var TReprClosure) + cl: var TReprClosure) {.gcsafe.} proc reprArray(result: var string, p: pointer, typ: PNimType, cl: var TReprClosure) = @@ -172,7 +172,7 @@ when not defined(useNimRtl): add result, "]" proc reprRecordAux(result: var string, p: pointer, n: ptr TNimNode, - cl: var TReprClosure) = + cl: var TReprClosure) {.gcsafe.} = case n.kind of nkNone: sysAssert(false, "reprRecordAux") of nkSlot: diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index 5e3b04b7f..b8ed29cfc 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -52,7 +52,7 @@ when defined(Windows): proc closeHandle(hObject: THandle) {.stdcall, noSideEffect, dynlib: "kernel32", importc: "CloseHandle".} proc waitForSingleObject(hHandle: THandle, dwMilliseconds: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "WaitForSingleObject".} + stdcall, dynlib: "kernel32", importc: "WaitForSingleObject", noSideEffect.} proc signalSysCond(hEvent: TSysCond) {.stdcall, noSideEffect, dynlib: "kernel32", importc: "SetEvent".} @@ -89,16 +89,16 @@ else: proc releaseSys(L: var TSysLock) {.noSideEffect, importc: "pthread_mutex_unlock", header: "<pthread.h>".} - proc deinitSys(L: var TSysLock) {. + proc deinitSys(L: var TSysLock) {.noSideEffect, importc: "pthread_mutex_destroy", header: "<pthread.h>".} proc initSysCond(cond: var TSysCond, cond_attr: pointer = nil) {. - importc: "pthread_cond_init", header: "<pthread.h>".} + importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} proc waitSysCond(cond: var TSysCond, lock: var TSysLock) {. - importc: "pthread_cond_wait", header: "<pthread.h>".} + importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} proc signalSysCond(cond: var TSysCond) {. - importc: "pthread_cond_signal", header: "<pthread.h>".} + importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} - proc deinitSysCond(cond: var TSysCond) {. + proc deinitSysCond(cond: var TSysCond) {.noSideEffect, importc: "pthread_cond_destroy", header: "<pthread.h>".} diff --git a/lib/system/sysspawn.nim b/lib/system/sysspawn.nim new file mode 100644 index 000000000..dabf35a3e --- /dev/null +++ b/lib/system/sysspawn.nim @@ -0,0 +1,195 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2014 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Implements Nimrod's 'spawn'. + +when not defined(NimString): + {.error: "You must not import this module explicitly".} + +{.push stackTrace:off.} + +when (defined(x86) or defined(amd64)) and defined(gcc): + proc cpuRelax {.inline.} = + {.emit: """asm volatile("pause" ::: "memory");""".} +elif (defined(x86) or defined(amd64)) and defined(vcc): + proc cpuRelax {.importc: "YieldProcessor", header: "<windows.h>".} +elif defined(intelc): + proc cpuRelax {.importc: "_mm_pause", header: "xmmintrin.h".} +elif false: + from os import sleep + + proc cpuRelax {.inline.} = os.sleep(1) + +when defined(windows) and not defined(gcc): + proc interlockedCompareExchange(p: pointer; exchange, comparand: int32): int32 + {.importc: "InterlockedCompareExchange", header: "<windows.h>", cdecl.} + + proc cas(p: ptr bool; oldValue, newValue: bool): bool = + interlockedCompareExchange(p, newValue.int32, oldValue.int32) != 0 + +else: + # this is valid for GCC and Intel C++ + proc cas(p: ptr bool; oldValue, newValue: bool): bool + {.importc: "__sync_bool_compare_and_swap", nodecl.} + +# We declare our own condition variables here to get rid of the dummy lock +# on Windows: + +type + CondVar = object + c: TSysCond + when defined(posix): + stupidLock: TSysLock + counter: int + +proc createCondVar(): CondVar = + initSysCond(result.c) + when defined(posix): + initSysLock(result.stupidLock) + #acquireSys(result.stupidLock) + +proc await(cv: var CondVar) = + when defined(posix): + acquireSys(cv.stupidLock) + while cv.counter <= 0: + waitSysCond(cv.c, cv.stupidLock) + dec cv.counter + releaseSys(cv.stupidLock) + else: + waitSysCondWindows(cv.c) + +proc signal(cv: var CondVar) = + when defined(posix): + acquireSys(cv.stupidLock) + inc cv.counter + releaseSys(cv.stupidLock) + signalSysCond(cv.c) + +type + FastCondVar = object + event, slowPath: bool + slow: CondVar + +proc createFastCondVar(): FastCondVar = + initSysCond(result.slow.c) + when defined(posix): + initSysLock(result.slow.stupidLock) + #acquireSys(result.slow.stupidLock) + result.event = false + result.slowPath = false + +proc await(cv: var FastCondVar) = + #for i in 0 .. 50: + # if cas(addr cv.event, true, false): + # # this is a HIT: Triggers > 95% in my tests. + # return + # cpuRelax() + #cv.slowPath = true + # XXX For some reason this crashes some test programs + await(cv.slow) + cv.event = false + +proc signal(cv: var FastCondVar) = + cv.event = true + #if cas(addr cv.slowPath, true, false): + signal(cv.slow) + +{.pop.} + +# ---------------------------------------------------------------------------- + +type + WorkerProc = proc (thread, args: pointer) {.nimcall, gcsafe.} + Worker = object + taskArrived: CondVar + taskStarted: FastCondVar #\ + # task data: + f: WorkerProc + data: pointer + ready: bool # put it here for correct alignment! + +proc nimArgsPassingDone(p: pointer) {.compilerProc.} = + let w = cast[ptr Worker](p) + signal(w.taskStarted) + +var gSomeReady = createFastCondVar() + +proc slave(w: ptr Worker) {.thread.} = + while true: + w.ready = true # If we instead signal "workerReady" we need the scheduler + # to notice this. The scheduler could then optimize the + # layout of the worker threads (e.g. keep the list sorted) + # so that no search for a "ready" thread is necessary. + # This might be implemented later, but is more tricky than + # it looks because 'spawn' itself can run concurrently. + signal(gSomeReady) + await(w.taskArrived) + assert(not w.ready) + # shield against spurious wakeups: + if w.data != nil: + w.f(w, w.data) + w.data = nil + +const NumThreads = 4 + +var + workers: array[NumThreads, TThread[ptr Worker]] + workersData: array[NumThreads, Worker] + +proc setup() = + for i in 0.. <NumThreads: + workersData[i].taskArrived = createCondVar() + workersData[i].taskStarted = createFastCondVar() + createThread(workers[i], slave, addr(workersData[i])) + +proc preferSpawn*(): bool = + ## Use this proc to determine quickly if a 'spawn' or a direct call is + ## preferable. If it returns 'true' a 'spawn' may make sense. In general + ## it is not necessary to call this directly; use 'spawnX' instead. + result = gSomeReady.event + +proc spawn*(call: stmt) {.magic: "Spawn".} + ## always spawns a new task, so that the 'call' is never executed on + ## the calling thread. 'call' has to be proc call 'p(...)' where 'p' + ## is gcsafe and has 'void' as the return type. + +template spawnX*(call: stmt) = + ## spawns a new task if a CPU core is ready, otherwise executes the + ## call in the calling thread. Usually it is advised to + ## use 'spawn' in order to not block the producer for an unknown + ## amount of time. 'call' has to be proc call 'p(...)' where 'p' + ## is gcsafe and has 'void' as the return type. + if preferSpawn(): spawn call + else: call + +proc nimSpawn(fn: WorkerProc; data: pointer) {.compilerProc.} = + # implementation of 'spawn' that is used by the code generator. + while true: + for i in 0.. high(workers): + let w = addr(workersData[i]) + if cas(addr w.ready, true, false): + w.data = data + w.f = fn + signal(w.taskArrived) + await(w.taskStarted) + return + await(gSomeReady) + +proc sync*() = + ## a simple barrier to wait for all spawn'ed tasks. If you need more elaborate + ## waiting, you have to use an explicit barrier. + while true: + var allReady = true + for i in 0 .. high(workers): + if not allReady: break + allReady = allReady and workersData[i].ready + if allReady: break + await(gSomeReady) + +setup() diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index eb9d2000b..4244bae4c 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -252,10 +252,8 @@ proc nimIntToStr(x: int): string {.compilerRtl.} = proc nimFloatToStr(x: float): string {.compilerproc.} = var buf: array [0..59, char] - c_sprintf(buf, "%#.f", x) - result = $buf - if result[len(result)-1] == '.': - result.add("0") + c_sprintf(buf, "%#.16e", x) + return $buf proc nimInt64ToStr(x: int64): string {.compilerRtl.} = result = newString(sizeof(x)*4) diff --git a/lib/system/threads.nim b/lib/system/threads.nim index ff9ab6cc0..0d52e4d09 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -256,9 +256,9 @@ type ## that **must not** be part of a message! Use ## a ``TThreadId`` for that. when TArg is void: - dataFn: proc () {.nimcall.} + dataFn: proc () {.nimcall, gcsafe.} else: - dataFn: proc (m: TArg) {.nimcall.} + dataFn: proc (m: TArg) {.nimcall, gcsafe.} data: TArg TThreadId*[TArg] = ptr TThread[TArg] ## the current implementation uses ## a pointer as a thread ID. diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 3449a3eba..4ce2f11b4 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -522,7 +522,7 @@ proc inet_addr*(cp: cstring): int32 {. stdcall, importc: "inet_addr", dynlib: ws2dll.} proc WSAFDIsSet(s: TSocketHandle, FDSet: var TFdSet): bool {. - stdcall, importc: "__WSAFDIsSet", dynlib: ws2dll.} + stdcall, importc: "__WSAFDIsSet", dynlib: ws2dll, noSideEffect.} proc FD_ISSET*(Socket: TSocketHandle, FDSet: var TFdSet): cint = result = if WSAFDIsSet(Socket, FDSet): 1'i32 else: 0'i32 @@ -718,4 +718,4 @@ proc WSASend*(s: TSocketHandle, buf: ptr TWSABuf, bufCount: DWORD, stdcall, importc: "WSASend", dynlib: "Ws2_32.dll".} proc get_osfhandle*(fd:TFileHandle): THandle {. - importc:"_get_osfhandle", header:"<io.h>".} \ No newline at end of file + importc:"_get_osfhandle", header:"<io.h>".} diff --git a/lib/wrappers/sqlite3.nim b/lib/wrappers/sqlite3.nim index 8ff1da1d1..586f763ae 100644 --- a/lib/wrappers/sqlite3.nim +++ b/lib/wrappers/sqlite3.nim @@ -90,6 +90,7 @@ const SQLITE_IGNORE* = 2 # Original from sqlite3.h: ##define SQLITE_STATIC ((void(*)(void *))0) ##define SQLITE_TRANSIENT ((void(*)(void *))-1) + SQLITE_DETERMINISTIC* = 0x800 const SQLITE_STATIC* = nil diff --git a/tests/actiontable/tactiontable2.nim b/tests/actiontable/tactiontable2.nim index 878356321..bfeb1c169 100644 --- a/tests/actiontable/tactiontable2.nim +++ b/tests/actiontable/tactiontable2.nim @@ -1,6 +1,6 @@ discard """ line: 21 - errormsg: "invalid type: 'TTable[string, proc (string)]'" + errormsg: "invalid type: 'TTable[string, proc (string){.gcsafe.}]'" """ import tables diff --git a/tests/assign/tobjasgn.nim b/tests/assign/tobjasgn.nim index da12319cd..23a31252d 100644 --- a/tests/assign/tobjasgn.nim +++ b/tests/assign/tobjasgn.nim @@ -1,5 +1,5 @@ discard """ - output: '''0 0 + output: '''8 5 0 0 pre test a:test b:1 c:2 haha:3 assignment test a:test b:1 c:2 haha:3 ''' @@ -9,13 +9,13 @@ assignment test a:test b:1 c:2 haha:3 type TSomeObj = object of TObject - a: int + a, b: int PSomeObj = ref object - a: int + a, b: int -var a = TSomeObj() -var b = PSomeObj() -echo a.a, " ", b.a +var a = TSomeObj(a: 8) +var b = PSomeObj(a: 5) +echo a.a, " ", b.a, " ", a.b, " ", b.b # bug #575 diff --git a/tests/async/tasyncawait.nim b/tests/async/tasyncawait.nim index bd722842f..ffceeaee6 100644 --- a/tests/async/tasyncawait.nim +++ b/tests/async/tasyncawait.nim @@ -1,6 +1,5 @@ discard """ file: "tasyncawait.nim" - cmd: "nimrod cc --hints:on $# $#" output: "5000" """ import asyncdispatch, rawsockets, net, strutils, os diff --git a/tests/async/tasynciossl.nim b/tests/async/tasynciossl.nim index fbed46efb..26c4c587c 100644 --- a/tests/async/tasynciossl.nim +++ b/tests/async/tasynciossl.nim @@ -1,6 +1,6 @@ discard """ file: "tasynciossl.nim" - cmd: "nimrod cc --hints:on --define:ssl $# $#" + cmd: "nimrod $target --hints:on --define:ssl $options $file" output: "20000" """ import sockets, asyncio, strutils, times @@ -88,4 +88,4 @@ while true: break assert msgCount == (swarmSize * messagesToSend) * serverCount -echo(msgCount) \ No newline at end of file +echo(msgCount) diff --git a/tests/bind/tnicerrorforsymchoice.nim b/tests/bind/tnicerrorforsymchoice.nim index bc271dcab..4e7a271b3 100644 --- a/tests/bind/tnicerrorforsymchoice.nim +++ b/tests/bind/tnicerrorforsymchoice.nim @@ -1,18 +1,18 @@ discard """ line: 18 - errormsg: "type mismatch: got (proc (TScgi) | proc (PAsyncSocket, PStringTable, string))" + errormsg: "type mismatch: got (proc (TScgi) | proc (PAsyncSocket, PStringTable, string){.gcsafe.})" """ #bug #442 import scgi, sockets, asyncio, strtabs proc handleSCGIRequest[TScgi: TScgiState | PAsyncScgiState](s: TScgi) = - nil + discard proc handleSCGIRequest(client: PAsyncSocket, headers: PStringTable, input: string) = - nil + discard proc test(handle: proc (client: PAsyncSocket, headers: PStringTable, input: string), b: int) = - nil + discard test(handleSCGIRequest) diff --git a/tests/closure/tclosuremacro.nim b/tests/closure/tclosuremacro.nim new file mode 100644 index 000000000..12e463316 --- /dev/null +++ b/tests/closure/tclosuremacro.nim @@ -0,0 +1,43 @@ +discard """ + output: '''10 +10 +10 +3 +3 +noReturn +6 +''' +""" + +import future + +proc twoParams(x: (int, int) -> int): int = + result = x(5, 5) + +proc oneParam(x: int -> int): int = + x(5) + +proc noParams(x: () -> int): int = + result = x() + +proc noReturn(x: () -> void) = + x() + +proc doWithOneAndTwo(f: (int, int) -> int): int = + f(1,2) + +echo twoParams(proc (a, b): auto = a + b) +echo twoParams((x, y) => x + y) + +echo oneParam(x => x+5) + +echo noParams(() => 3) + +echo doWithOneAndTwo((x, y) => x + y) + +noReturn(() -> void => echo("noReturn")) + +proc pass2(f: (int, int) -> int): (int) -> int = + (x: int) -> int => f(2, x) + +echo pass2((x, y) => x + y)(4) diff --git a/tests/closure/tinvalidclosure.nim b/tests/closure/tinvalidclosure.nim index c06270bfa..06e19df3c 100644 --- a/tests/closure/tinvalidclosure.nim +++ b/tests/closure/tinvalidclosure.nim @@ -1,6 +1,6 @@ discard """ line: 12 - errormsg: "type mismatch: got (proc (int){.closure.})" + errormsg: "type mismatch: got (proc (int){.closure, gcsafe.})" """ proc ugh[T](x: T) {.closure.} = diff --git a/tests/concurrency/tnodeadlocks.nim b/tests/concurrency/tnodeadlocks.nim index 3f27e24f6..d44196039 100644 --- a/tests/concurrency/tnodeadlocks.nim +++ b/tests/concurrency/tnodeadlocks.nim @@ -1,6 +1,6 @@ discard """ outputsub: "101" - cmd: "nimrod cc --hints:on --threads:on $# $#" + cmd: "nimrod $target --hints:on --threads:on $options $file" """ import os, locks diff --git a/tests/dll/client.nim b/tests/dll/client.nim index a78cef1d4..45fa8f639 100644 --- a/tests/dll/client.nim +++ b/tests/dll/client.nim @@ -1,6 +1,6 @@ discard """ output: "Done" - cmd: "nimrod cc --debuginfo --hints:on --define:useNimRtl $# $#" + cmd: "nimrod $target --debuginfo --hints:on --define:useNimRtl $options $file" """ type diff --git a/tests/dll/server.nim b/tests/dll/server.nim index ae2acc893..b2fac9ecc 100644 --- a/tests/dll/server.nim +++ b/tests/dll/server.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nimrod cc --debuginfo --hints:on --define:useNimRtl --app:lib $# $#" + cmd: "nimrod $target --debuginfo --hints:on --define:useNimRtl --app:lib $options $file" """ type diff --git a/tests/generics/tmetafield.nim b/tests/generics/tmetafield.nim index f2fac8fdd..e1bc43ce3 100644 --- a/tests/generics/tmetafield.nim +++ b/tests/generics/tmetafield.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nimrod check $# $#" + cmd: "nimrod check $options $file" errormsg: "'proc' is not a concrete type" errormsg: "'Foo' is not a concrete type." errormsg: "invalid type: 'TBaseMed'" diff --git a/tests/generics/tthread_generic.nim b/tests/generics/tthread_generic.nim index beae4b652..7109bba18 100644 --- a/tests/generics/tthread_generic.nim +++ b/tests/generics/tthread_generic.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nimrod cc --hints:on --threads:on $# $#" + cmd: "nimrod $target --hints:on --threads:on $options $file" """ type diff --git a/tests/iter/titer2.nim b/tests/iter/titer2.nim index f8967109e..71c7327a7 100644 --- a/tests/iter/titer2.nim +++ b/tests/iter/titer2.nim @@ -1,6 +1,6 @@ discard """ output: '''true''' - cmd: "nimrod cc --gc:none --hints:on --warnings:off $# $#" + cmd: "nimrod $target --gc:none --hints:on --warnings:off $options $file" """ import hashes diff --git a/tests/js.nim b/tests/js.nim index f31bb10d9..becc17834 100644 --- a/tests/js.nim +++ b/tests/js.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nimrod js --hints:on $# $#" + cmd: "nimrod js --hints:on $options $file" """ # This file tests the JavaScript generator diff --git a/tests/js/test1.nim b/tests/js/test1.nim index 3f3d5f02c..09ba30676 100644 --- a/tests/js/test1.nim +++ b/tests/js/test1.nim @@ -1,5 +1,4 @@ discard """ - cmd: "nimrod js --hints:on $# $#" output: "1261129" """ diff --git a/tests/js/test2.nim b/tests/js/test2.nim index 1342ed15d..5a734358c 100644 --- a/tests/js/test2.nim +++ b/tests/js/test2.nim @@ -1,5 +1,4 @@ discard """ - cmd: "nimrod js --hints:on -r $# $#" output: '''foo js 3.14''' """ diff --git a/tests/js/testmagic.nim b/tests/js/testmagic.nim index 2c02d24be..5f793ae05 100644 --- a/tests/js/testmagic.nim +++ b/tests/js/testmagic.nim @@ -1,5 +1,4 @@ discard """ - cmd: "nimrod js --hints:on -r $# $#" output: '''true''' """ diff --git a/tests/macros/texprcolonexpr.nim b/tests/macros/texprcolonexpr.nim new file mode 100644 index 000000000..3b2c86b77 --- /dev/null +++ b/tests/macros/texprcolonexpr.nim @@ -0,0 +1,19 @@ +discard """ + msg: ''' +Infix + Ident !"=>" + Call + Ident !"name" + Ident !"a" + ExprColonExpr + Ident !"b" + Ident !"cint" + NilLit nil +''' +""" +import macros + +macro def(x: stmt): stmt {.immediate.} = + echo treeRepr(x) + +def name(a, b:cint) => nil diff --git a/tests/misc/tgtk.nim b/tests/misc/tgtk.nim index 7febb0ab8..82227689d 100644 --- a/tests/misc/tgtk.nim +++ b/tests/misc/tgtk.nim @@ -1,4 +1,6 @@ - +discard """ + disabled: true +""" import gtk2, glib2, atk, gdk2, gdk2pixbuf, libglade2, pango, pangoutils diff --git a/tests/misc/tmandelbrot.nim b/tests/misc/tmandelbrot.nim index 1e39c8756..bb1b46d89 100644 --- a/tests/misc/tmandelbrot.nim +++ b/tests/misc/tmandelbrot.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nimrod cc --hints:on -d:release $# $#" + cmd: "nimrod $target --hints:on -d:release $options $file" """ # -*- nimrod -*- diff --git a/tests/sets/tsets2.nim b/tests/sets/tsets2.nim index ac977096b..7f313e14f 100644 --- a/tests/sets/tsets2.nim +++ b/tests/sets/tsets2.nim @@ -1,6 +1,5 @@ discard """ output: '''true''' - cmd: "nimrod cc --hints:on $# $#" """ import hashes, sets diff --git a/tests/stdlib/tunidecode.nim b/tests/stdlib/tunidecode.nim index cb6589d60..647858825 100644 --- a/tests/stdlib/tunidecode.nim +++ b/tests/stdlib/tunidecode.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nimrod cc --hints:on -d:embedUnidecodeTable $# $#" + cmd: "nimrod $target --hints:on -d:embedUnidecodeTable $options $file" output: "Ausserst" """ diff --git a/tests/system/tsysspawnbadarg.nim b/tests/system/tsysspawnbadarg.nim new file mode 100644 index 000000000..ace074602 --- /dev/null +++ b/tests/system/tsysspawnbadarg.nim @@ -0,0 +1,7 @@ +discard """ + line: 7 + errormsg: "'spawn' takes a call expression of type void" + cmd: "nimrod $target --threads:on $options $file" +""" + +spawn(1) diff --git a/tests/table/ttables.nim b/tests/table/ttables.nim index 681ff5424..60446b5a3 100644 --- a/tests/table/ttables.nim +++ b/tests/table/ttables.nim @@ -1,6 +1,5 @@ discard """ output: '''true''' - cmd: "nimrod cc --hints:on $# $#" """ import hashes, tables diff --git a/tests/table/ttables2.nim b/tests/table/ttables2.nim index b88c8dfbf..611f3f8ec 100644 --- a/tests/table/ttables2.nim +++ b/tests/table/ttables2.nim @@ -1,6 +1,5 @@ discard """ output: '''true''' - cmd: "nimrod cc --hints:on $# $#" """ import tables diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index 9bb4838e0..bb9c90d2a 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -151,7 +151,7 @@ proc threadTests(r: var TResults, cat: Category, options: string) = #test "tthreadanalysis" #test "tthreadsort" test "tthreadanalysis2" - test "tthreadanalysis3" + #test "tthreadanalysis3" test "tthreadheapviolation1" # ------------------------- IO tests ------------------------------------------ @@ -179,10 +179,12 @@ proc jsTests(r: var TResults, cat: Category, options: string) = for t in os.walkFiles("tests/js/t*.nim"): test(t) - for testfile in ["texceptions", "texcpt1", "texcsub", "tfinally", - "tfinally2", "tfinally3", "tactiontable", "tmultim1", - "tmultim3", "tmultim4"]: - test "tests/run/" & testfile & ".nim" + for testfile in ["exception/texceptions", "exception/texcpt1", + "exception/texcsub", "exception/tfinally", + "exception/tfinally2", "exception/tfinally3", + "actiontable/tactiontable", "method/tmultim1", + "method/tmultim3", "method/tmultim4"]: + test "tests/" & testfile & ".nim" # ------------------------- manyloc ------------------------------------------- #proc runSpecialTests(r: var TResults, options: string) = diff --git a/tests/testament/specs.nim b/tests/testament/specs.nim index 65e17a453..225ea1891 100644 --- a/tests/testament/specs.nim +++ b/tests/testament/specs.nim @@ -10,7 +10,7 @@ import parseutils, strutils, os, osproc, streams, parsecfg const - cmdTemplate* = r"nimrod cc --hints:on $# $#" + cmdTemplate* = r"nimrod $target --hints:on $options $file" type TTestAction* = enum @@ -51,6 +51,7 @@ type const targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"] + targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"] when not defined(parseCfgBool): # candidate for the stdlib: diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index 6655b1b79..757e54889 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -53,8 +53,10 @@ let pegSuccess = peg"'Hint: operation successful'.*" pegOfInterest = pegLineError / pegOtherError -proc callCompiler(cmdTemplate, filename, options: string): TSpec = - let c = parseCmdLine(cmdTemplate % [options, filename]) +proc callCompiler(cmdTemplate, filename, options: string, + target: TTarget): TSpec = + let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], + "options", options, "file", filename]) var p = startProcess(command=c[0], args=c[1.. -1], options={poStdErrToStdOut, poUseShell}) let outp = p.outputStream @@ -111,10 +113,10 @@ proc addResult(r: var TResults, test: TTest, r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) if success notin {reSuccess, reIgnored}: styledEcho styleBright, name, fgRed, " [", $success, "]" - styledEcho styleDim, "EXPECTED:" - echo expected - styledEcho styleDim, "GIVEN:" - echo given + echo"Expected:" + styledEcho styleBright, expected + echo"Given:" + styledEcho styleBright, given proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) = if strip(expected.msg) notin strip(given.msg): @@ -152,7 +154,7 @@ proc testSpec(r: var TResults, test: TTest) = # major entry point for a single test let tname = test.name.addFileExt(".nim") inc(r.total) - echo extractFilename(tname) + styledEcho "Processing ", fgCyan, extractFilename(tname) var expected = parseSpec(tname) if expected.err == reIgnored: r.addResult(test, "", "", reIgnored) @@ -160,13 +162,15 @@ proc testSpec(r: var TResults, test: TTest) = else: case expected.action of actionCompile: - var given = callCompiler(expected.cmd, test.name, test.options) + var given = callCompiler(expected.cmd, test.name, test.options, + test.target) if given.err == reSuccess: codegenCheck(test, expected.ccodeCheck, given) r.addResult(test, "", given.msg, given.err) if given.err == reSuccess: inc(r.passed) of actionRun: - var given = callCompiler(expected.cmd, test.name, test.options) + var given = callCompiler(expected.cmd, test.name, test.options, + test.target) if given.err != reSuccess: r.addResult(test, "", given.msg, given.err) else: @@ -176,10 +180,13 @@ proc testSpec(r: var TResults, test: TTest) = exeFile = dir / "nimcache" / file & ".js" else: exeFile = changeFileExt(tname, ExeExt) - if existsFile(exeFile): + if test.target == targetJS and findExe("nodejs") == "": + r.addResult(test, expected.outp, "nodejs binary not in PATH", + reExeNotFound) + return var (buf, exitCode) = execCmdEx( - (if test.target==targetJS: "node " else: "") & exeFile) + (if test.target == targetJS: "nodejs " else: "") & exeFile) if exitCode != expected.ExitCode: r.addResult(test, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode, reExitCodesDiffer) @@ -194,7 +201,8 @@ proc testSpec(r: var TResults, test: TTest) = else: r.addResult(test, expected.outp, "executable not found", reExeNotFound) of actionReject: - var given = callCompiler(expected.cmd, test.name, test.options) + var given = callCompiler(expected.cmd, test.name, test.options, + test.target) cmpMsgs(r, expected, given, test) proc testNoSpec(r: var TResults, test: TTest) = @@ -202,7 +210,7 @@ proc testNoSpec(r: var TResults, test: TTest) = let tname = test.name.addFileExt(".nim") inc(r.total) echo extractFilename(tname) - let given = callCompiler(cmdTemplate, test.name, test.options) + let given = callCompiler(cmdTemplate, test.name, test.options, test.target) r.addResult(test, "", given.msg, given.err) if given.err == reSuccess: inc(r.passed) @@ -223,7 +231,7 @@ proc main() = os.putenv "NIMTEST_NO_COLOR", "1" os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES" - backend.open() + backend.open() var optPrintResults = false var p = initOptParser() p.next() @@ -238,9 +246,12 @@ proc main() = var r = initResults() case action of "all": - for kind, dir in walkDir("tests"): - if kind == pcDir and dir notin ["testament", "testdata", "nimcache"]: - processCategory(r, Category(dir), p.cmdLineRest.string) + let testsDir = "tests" & dirSep + for kind, dir in walkDir(testsDir): + assert testsDir.startsWith(testsDir) + let cat = dir[testsDir.len .. -1] + if kind == pcDir and cat notin ["testament", "testdata", "nimcache"]: + processCategory(r, Category(cat), p.cmdLineRest.string) for a in AdditionalCategories: processCategory(r, Category(a), p.cmdLineRest.string) of "c", "cat", "category": @@ -255,11 +266,11 @@ proc main() = else: quit usage - if optPrintResults: + if optPrintResults: if action == "html": openDefaultBrowser(resultsFile) else: echo r, r.data backend.close() - + if paramCount() == 0: quit usage main() diff --git a/tests/threads/tthreadanalysis.nim b/tests/threads/tthreadanalysis.nim index 3a46cd185..383680d81 100644 --- a/tests/threads/tthreadanalysis.nim +++ b/tests/threads/tthreadanalysis.nim @@ -2,7 +2,7 @@ discard """ outputsub: "101" msg: "Warning: write to foreign heap" line: 37 - cmd: "nimrod cc --hints:on --threads:on $# $#" + cmd: "nimrod $target --hints:on --threads:on $options $file" """ import os diff --git a/tests/threads/tthreadanalysis2.nim b/tests/threads/tthreadanalysis2.nim index 07f0e61fd..bcc09db98 100644 --- a/tests/threads/tthreadanalysis2.nim +++ b/tests/threads/tthreadanalysis2.nim @@ -1,8 +1,8 @@ discard """ file: "tthreadanalysis2.nim" - line: 42 - errormsg: "write to foreign heap" - cmd: "nimrod cc --hints:on --threads:on $# $#" + line: 37 + errormsg: "'threadFunc' is not GC-safe" + cmd: "nimrod $target --hints:on --threads:on $options $file" """ import os @@ -10,7 +10,7 @@ import os var thr: array [0..5, TThread[tuple[a, b: int]]] -proc doNothing() = nil +proc doNothing() = discard type PNode = ref TNode diff --git a/tests/threads/tthreadanalysis3.nim b/tests/threads/tthreadanalysis3.nim deleted file mode 100644 index d7a838fec..000000000 --- a/tests/threads/tthreadanalysis3.nim +++ /dev/null @@ -1,51 +0,0 @@ -discard """ - file: "tthreadanalysis3.nim" - line: 35 - errormsg: "write to foreign heap" - cmd: "nimrod cc --hints:on --threads:on $# $#" -""" - -import os - -var - thr: array [0..5, TThread[tuple[a, b: int]]] - -proc doNothing() = nil - -type - PNode = ref TNode - TNode = object {.pure.} - le, ri: PNode - data: string - -var - root: PNode - -proc buildTree(depth: int): PNode = - if depth == 3: return nil - new(result) - result.le = buildTree(depth-1) - result.ri = buildTree(depth-1) - result.data = $depth - -proc echoLeTree(n: PNode) = - var it = n - while it != nil: - echo it.data - it = it.le - -proc threadFunc(interval: tuple[a, b: int]) {.thread.} = - doNothing() - for i in interval.a..interval.b: - var r = buildTree(i) - echoLeTree(r) # for local data - echoLeTree(root) # and the same for foreign data :-) - -proc main = - root = buildTree(5) - for i in 0..high(thr): - createThread(thr[i], threadFunc, (i*100, i*100+50)) - joinThreads(thr) - -main() - diff --git a/tests/threads/tthreadheapviolation1.nim b/tests/threads/tthreadheapviolation1.nim index 7ca6f7928..f3a36e036 100644 --- a/tests/threads/tthreadheapviolation1.nim +++ b/tests/threads/tthreadheapviolation1.nim @@ -1,7 +1,7 @@ discard """ line: 12 errormsg: "write to foreign heap" - cmd: "nimrod cc --hints:on --threads:on $# $#" + cmd: "nimrod $target --hints:on --threads:on $options $file" """ var diff --git a/tests/typerel/tno_gcmem_in_shared.nim b/tests/typerel/tno_gcmem_in_shared.nim new file mode 100644 index 000000000..8c81e8db6 --- /dev/null +++ b/tests/typerel/tno_gcmem_in_shared.nim @@ -0,0 +1,22 @@ +discard """ + errormsg: "shared memory may not refer to GC'ed thread local memory" + line: 14 +""" + +type + Region = object + Foo = Region ptr int + + MyObject = object + a, b: string + + Bar[T] = shared ptr T + Bzar = Bar[MyObject] + +proc bar(x: Region ptr int) = + discard + +var + s: Foo + +bar s diff --git a/tests/types/tillegaltyperecursion.nim b/tests/types/tillegaltyperecursion.nim index 711f458bf..114e4d08e 100644 --- a/tests/types/tillegaltyperecursion.nim +++ b/tests/types/tillegaltyperecursion.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nimrod c --threads:on $# $#" + cmd: "nimrod $target --threads:on $options $file" errormsg: "illegal recursion in type 'TIRC'" line: 16 """ diff --git a/web/nimrod.ini b/web/nimrod.ini index b29bcff30..14701ecea 100644 --- a/web/nimrod.ini +++ b/web/nimrod.ini @@ -63,7 +63,7 @@ srcdoc2: "pure/asyncio;pure/actors;core/locks;pure/oids;pure/endians;pure/uri" srcdoc2: "pure/nimprof;pure/unittest;packages/docutils/highlite" srcdoc2: "packages/docutils/rst;packages/docutils/rstast" srcdoc2: "packages/docutils/rstgen;pure/logging;pure/asyncdispatch;pure/asyncnet" -srcdoc2: "pure/rawsockets;pure/asynchttpserver;pure/net;pure/selectors" +srcdoc2: "pure/rawsockets;pure/asynchttpserver;pure/net;pure/selectors;pure/future" webdoc: "wrappers/libcurl;pure/md5;wrappers/mysql;wrappers/iup" webdoc: "wrappers/sqlite3;wrappers/postgres;wrappers/tinyc" diff --git a/web/ticker.txt b/web/ticker.txt index 716fb90ad..a4ddddbbe 100644 --- a/web/ticker.txt +++ b/web/ticker.txt @@ -1,5 +1,5 @@ <a class="news" href="news.html#Z2014-XX-XX-version-0-9-4-released"> - <h3>April 20, 2014</h3> + <h3>Apr 20, 2014</h3> <p>Nimrod version 0.9.4 has been released!</p> </a> |