diff options
177 files changed, 7696 insertions, 870 deletions
diff --git a/bootstrap.sh b/bootstrap.sh index 7f19c2440..dd551b52d 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -18,5 +18,3 @@ set +x echo echo 'Install Nim using "./install.sh <dir>" or "sudo ./install.sh <dir>".' - -exit 0 diff --git a/compiler/ast.nim b/compiler/ast.nim index 73c99c0b1..4c98219f9 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -395,6 +395,9 @@ type # sons[1]: field type # .n: nkDotExpr storing the field name + tyVoid #\ + # now different from tyEmpty, hurray! + static: # remind us when TTypeKind stops to fit in a single 64-bit word assert TTypeKind.high.ord <= 63 @@ -528,8 +531,6 @@ const tfOldSchoolExprStmt* = tfVarargs # for now used to distinguish \ # 'varargs[expr]' from 'varargs[untyped]'. Eventually 'expr' will be # deprecated and this mess can be cleaned up. - tfVoid* = tfVarargs # for historical reasons we conflated 'void' with - # 'empty' ('@[]' has the type 'seq[empty]'). tfReturnsNew* = tfInheritable skError* = skUnknown @@ -544,7 +545,7 @@ type mEcho, mShallowCopy, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst, mExpandToAst, mQuoteAst, mUnaryLt, mInc, mDec, mOrd, - mNew, mNewFinalize, mNewSeq, + mNew, mNewFinalize, mNewSeq, mNewSeqOfCap, mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq, mXLenStr, mXLenSeq, mIncl, mExcl, mCard, mChr, @@ -590,6 +591,7 @@ type mNewString, mNewStringOfCap, mParseBiggestFloat, mReset, mArray, mOpenArray, mRange, mSet, mSeq, mVarargs, + mRef, mPtr, mVar, mDistinct, mVoid, mTuple, mOrdinal, mInt, mInt8, mInt16, mInt32, mInt64, mUInt, mUInt8, mUInt16, mUInt32, mUInt64, @@ -609,7 +611,7 @@ type mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, mNGenSym, - mNimvm + mNimvm, mIntDefine, mStrDefine # things that we can evaluate safely at compile time, even if not asked for it: const @@ -769,7 +771,7 @@ type procInstCache*: seq[PInstantiation] gcUnsafetyReason*: PSym # for better error messages wrt gcsafe #scope*: PScope # the scope where the proc was defined - of skModule: + of skModule, skPackage: # modules keep track of the generic symbols they use from other modules. # this is because in incremental compilation, when a module is about to # be replaced with a newer version, we must decrement the usage count @@ -848,6 +850,7 @@ type # see instantiateDestructor in semdestruct.nim deepCopy*: PSym # overriden 'deepCopy' operation assignment*: PSym # overriden '=' operator + methods*: seq[(int,PSym)] # attached methods size*: BiggestInt # the size of the type in bytes # -1 means that the size is unkwown align*: int16 # the type's alignment requirements @@ -963,7 +966,6 @@ const var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things var gMainPackageId*: int - gMainPackageNotes*: TNoteKinds proc isCallExpr*(n: PNode): bool = result = n.kind in nkCallKinds @@ -1587,7 +1589,7 @@ proc isAtom*(n: PNode): bool {.inline.} = proc isEmptyType*(t: PType): bool {.inline.} = ## 'void' and 'stmt' types are often equivalent to 'nil' these days: - result = t == nil or t.kind in {tyEmpty, tyStmt} + result = t == nil or t.kind in {tyVoid, tyStmt} proc makeStmtList*(n: PNode): PNode = if n.kind == nkStmtList: diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index bd17f85e4..dffb8a9a5 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -124,7 +124,7 @@ proc openArrayLoc(p: BProc, n: PNode): Rope = result = "(*$1)->data, (*$1)->$2" % [a.rdLoc, lenField(p)] of tyArray, tyArrayConstr: result = "$1, $2" % [rdLoc(a), rope(lengthOrd(lastSon(a.t)))] - else: + else: internalError("openArrayLoc: " & typeToString(a.t)) else: internalError("openArrayLoc: " & typeToString(a.t)) @@ -260,7 +260,11 @@ proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope = else: result = genArgNoParam(p, ri.sons[i]) #, typ.n.sons[i].sym) else: - result = genArgNoParam(p, ri.sons[i]) + if tfVarargs notin typ.flags: + localError(ri.info, "wrong argument count") + result = nil + else: + result = genArgNoParam(p, ri.sons[i]) discard """ Dot call syntax in C++ diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 8fe9cdb72..3f12fed2c 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -45,7 +45,7 @@ proc genHexLiteral(v: PNode): Rope = proc getStrLit(m: BModule, s: string): Rope = discard cgsym(m, "TGenericSeq") - result = "TMP" & rope(backendId()) + result = getTempName(m) addf(m.s[cfsData], "STRING_LITERAL($1, $2, $3);$n", [result, makeCString(s), rope(len(s))]) @@ -67,11 +67,11 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope = of nkNilLit: let t = skipTypes(ty, abstractVarRange) if t.kind == tyProc and t.callConv == ccClosure: - var id = nodeTableTestOrSet(p.module.dataCache, n, gBackendId) - result = "TMP" & rope(id) - if id == gBackendId: + let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) + result = p.module.tmpBase & rope(id) + if id == p.module.labels: # not found in cache: - inc(gBackendId) + inc(p.module.labels) addf(p.module.s[cfsData], "static NIM_CONST $1 $2 = {NIM_NIL,NIM_NIL};$n", [getTypeDesc(p.module, t), result]) @@ -81,13 +81,14 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope = if n.strVal.isNil: result = ropecg(p.module, "((#NimStringDesc*) NIM_NIL)", []) elif skipTypes(ty, abstractVarRange).kind == tyString: - var id = nodeTableTestOrSet(p.module.dataCache, n, gBackendId) - if id == gBackendId: + let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) + if id == p.module.labels: # string literal not found in the cache: result = ropecg(p.module, "((#NimStringDesc*) &$1)", [getStrLit(p.module, n.strVal)]) else: - result = ropecg(p.module, "((#NimStringDesc*) &TMP$1)", [rope(id)]) + result = ropecg(p.module, "((#NimStringDesc*) &$1$2)", + [p.module.tmpBase, rope(id)]) else: result = makeCString(n.strVal) of nkFloatLit..nkFloat64Lit: @@ -134,11 +135,11 @@ proc genSetNode(p: BProc, n: PNode): Rope = var size = int(getSize(n.typ)) toBitSet(n, cs) if size > 8: - var id = nodeTableTestOrSet(p.module.dataCache, n, gBackendId) - result = "TMP" & rope(id) - if id == gBackendId: + let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) + result = p.module.tmpBase & rope(id) + if id == p.module.labels: # not found in cache: - inc(gBackendId) + inc(p.module.labels) addf(p.module.s[cfsData], "static NIM_CONST $1 $2 = $3;$n", [getTypeDesc(p.module, n.typ), result, genRawSetData(cs, size)]) else: @@ -490,7 +491,7 @@ proc binaryArithOverflowRaw(p: BProc, t: PType, a, b: TLoc; var size = getSize(t) let storage = if size < platform.intSize: rope("NI") else: getTypeDesc(p.module, t) - result = getTempName() + result = getTempName(p.module) linefmt(p, cpsLocals, "$1 $2;$n", storage, result) lineCg(p, cpsStmts, frmt, result, rdCharLoc(a), rdCharLoc(b)) if size < platform.intSize or t.kind in {tyRange, tyEnum}: @@ -801,9 +802,9 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym; v.r.add(d.loc.r) genInExprAux(p, it, u, v, test) let id = nodeTableTestOrSet(p.module.dataCache, - newStrNode(nkStrLit, field.name.s), gBackendId) - let strLit = if id == gBackendId: getStrLit(p.module, field.name.s) - else: "TMP" & rope(id) + newStrNode(nkStrLit, field.name.s), p.module.labels) + let strLit = if id == p.module.labels: getStrLit(p.module, field.name.s) + else: p.module.tmpBase & rope(id) if op.magic == mNot: linefmt(p, cpsStmts, "if ($1) #raiseFieldError(((#NimStringDesc*) &$2));$n", @@ -1140,6 +1141,16 @@ proc genNewSeq(p: BProc, e: PNode) = genNewSeqAux(p, a, b.rdLoc) gcUsage(e) +proc genNewSeqOfCap(p: BProc; e: PNode; d: var TLoc) = + let seqtype = skipTypes(e.typ, abstractVarRange) + var a: TLoc + initLocExpr(p, e.sons[1], a) + putIntoDest(p, d, e.typ, ropecg(p.module, + "($1)#nimNewSeqOfCap($2, $3)", [ + getTypeDesc(p.module, seqtype), + genTypeInfo(p.module, seqtype), a.rdLoc])) + gcUsage(e) + proc genObjConstr(p: BProc, e: PNode, d: var TLoc) = var tmp: TLoc var t = e.typ.skipTypes(abstractInst) @@ -1319,7 +1330,7 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) = putIntoDest(p, d, e.typ, ropecg(p.module, "#reprAny($1, $2)", [ rdLoc(a), genTypeInfo(p.module, t)]), a.s) - of tyEmpty: + of tyEmpty, tyVoid: localError(e.info, "'repr' doesn't support 'void' type") else: putIntoDest(p, d, e.typ, ropecg(p.module, "#reprAny($1, $2)", @@ -1557,11 +1568,12 @@ proc genSomeCast(p: BProc, e: PNode, d: var TLoc) = [getTypeDesc(p.module, e.typ), rdCharLoc(a)], a.s) proc genCast(p: BProc, e: PNode, d: var TLoc) = - const floatTypes = {tyFloat..tyFloat128} + const ValueTypes = {tyFloat..tyFloat128, tyTuple, tyObject, + tyArray, tyArrayConstr} let destt = skipTypes(e.typ, abstractRange) srct = skipTypes(e.sons[1].typ, abstractRange) - if destt.kind in floatTypes or srct.kind in floatTypes: + if destt.kind in ValueTypes or srct.kind in ValueTypes: # 'cast' and some float type involved? --> use a union. inc(p.labels) var lbl = p.labels.rope @@ -1667,10 +1679,10 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = if optOverflowCheck notin p.options: unaryExpr(p, e, d, "($1 - 1)") else: unaryExpr(p, e, d, "#subInt($1, 1)") of mInc, mDec: - const opr: array [mInc..mDec, string] = ["$1 += $2;$n", "$1 -= $2;$n"] - const fun64: array [mInc..mDec, string] = ["$# = #addInt64($#, $#);$n", + const opr: array[mInc..mDec, string] = ["$1 += $2;$n", "$1 -= $2;$n"] + const fun64: array[mInc..mDec, string] = ["$# = #addInt64($#, $#);$n", "$# = #subInt64($#, $#);$n"] - const fun: array [mInc..mDec, string] = ["$# = #addInt($#, $#);$n", + const fun: array[mInc..mDec, string] = ["$# = #addInt($#, $#);$n", "$# = #subInt($#, $#);$n"] let underlying = skipTypes(e.sons[1].typ, {tyGenericInst, tyVar, tyRange}) if optOverflowCheck notin p.options or underlying.kind in {tyUInt..tyUInt64}: @@ -1711,6 +1723,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mNew: genNew(p, e) of mNewFinalize: genNewFinalize(p, e) of mNewSeq: genNewSeq(p, e) + of mNewSeqOfCap: genNewSeqOfCap(p, e, d) of mSizeOf: let t = e.sons[1].typ.skipTypes({tyTypeDesc}) putIntoDest(p, d, e.typ, "((NI)sizeof($1))" % [getTypeDesc(p.module, t)]) @@ -1761,11 +1774,11 @@ proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool = if nfAllConst in n.flags and d.k == locNone and n.len > 0 and n.isDeepConstExpr: var t = getUniqueType(n.typ) discard getTypeDesc(p.module, t) # so that any fields are initialized - var id = nodeTableTestOrSet(p.module.dataCache, n, gBackendId) - fillLoc(d, locData, t, "TMP" & rope(id), OnStatic) - if id == gBackendId: + let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) + fillLoc(d, locData, t, p.module.tmpBase & rope(id), OnStatic) + if id == p.module.labels: # expression not found in the cache: - inc(gBackendId) + inc(p.module.labels) addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n", [getTypeDesc(p.module, t), d.r, genConstExpr(p, n)]) result = true @@ -1952,12 +1965,12 @@ proc downConv(p: BProc, n: PNode, d: var TLoc) = proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) = var t = getUniqueType(n.typ) discard getTypeDesc(p.module, t) # so that any fields are initialized - var id = nodeTableTestOrSet(p.module.dataCache, n, gBackendId) - var tmp = "TMP" & rope(id) + let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) + let tmp = p.module.tmpBase & rope(id) - if id == gBackendId: + if id == p.module.labels: # expression not found in the cache: - inc(gBackendId) + inc(p.module.labels) addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n", [getTypeDesc(p.module, t), tmp, genConstExpr(p, n)]) @@ -2179,8 +2192,7 @@ proc genConstSeq(p: BProc, n: PNode, t: PType): Rope = data.add("}") data.add("}") - inc(gBackendId) - result = "CNSTSEQ" & gBackendId.rope + result = getTempName(p.module) appcg(p.module, cfsData, "NIM_CONST struct {$n" & diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 64933df78..70d50b060 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -793,7 +793,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = if not isEmptyType(t.typ) and d.k == locNone: getTemp(p, t.typ, d) genLineDir(p, t) - let exc = getTempName() + let exc = getTempName(p.module) if getCompilerProc("Exception") != nil: discard cgsym(p.module, "Exception") else: @@ -886,7 +886,7 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) = getTemp(p, t.typ, d) discard lists.includeStr(p.module.headerFiles, "<setjmp.h>") genLineDir(p, t) - var safePoint = getTempName() + var safePoint = getTempName(p.module) if getCompilerProc("Exception") != nil: discard cgsym(p.module, "Exception") else: diff --git a/compiler/ccgtrav.nim b/compiler/ccgtrav.nim index 0da6396ea..dfefc4b63 100644 --- a/compiler/ccgtrav.nim +++ b/compiler/ccgtrav.nim @@ -100,7 +100,7 @@ proc genTraverseProcSeq(c: var TTraversalClosure, accessor: Rope, typ: PType) = proc genTraverseProc(m: BModule, typ: PType, reason: TTypeInfoReason): Rope = var c: TTraversalClosure var p = newProc(nil, m) - result = getGlobalTempName() + result = getTempName(m) case reason of tiNew: c.visitorFrmt = "#nimGCvisit((void*)$1, op);$n" @@ -135,7 +135,7 @@ proc genTraverseProcForGlobal(m: BModule, s: PSym): Rope = var c: TTraversalClosure var p = newProc(nil, m) var sLoc = s.loc.r - result = getGlobalTempName() + result = getTempName(m) if sfThread in s.flags and emulatedThreadVars(): accessThreadLocalVar(p, s) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 4de196fe0..3d1c2affc 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -25,8 +25,9 @@ proc isKeyword(w: PIdent): bool = proc mangleField(name: PIdent): string = result = mangle(name.s) if isKeyword(name): - result[0] = result[0].toUpper # Mangling makes everything lowercase, - # but some identifiers are C keywords + result[0] = result[0].toUpperAscii + # Mangling makes everything lowercase, + # but some identifiers are C keywords proc hashOwner(s: PSym): FilenameHash = var m = s @@ -150,6 +151,9 @@ proc mapType(typ: PType): TCTypeKind = of tyCString: result = ctCString of tyInt..tyUInt64: result = TCTypeKind(ord(typ.kind) - ord(tyInt) + ord(ctInt)) + of tyStatic: + if typ.n != nil: result = mapType(lastSon typ) + else: internalError("mapType") else: internalError("mapType") proc mapReturnType(typ: PType): TCTypeKind = @@ -202,11 +206,9 @@ proc cacheGetType(tab: TIdTable, key: PType): Rope = # linear search is not necessary anymore: result = Rope(idTableGet(tab, key)) -proc getTempName(): Rope = - result = rfmt(nil, "TMP$1", rope(backendId())) - -proc getGlobalTempName(): Rope = - result = rfmt(nil, "TMP$1", rope(backendId())) +proc getTempName(m: BModule): Rope = + result = m.tmpBase & rope(m.labels) + inc m.labels proc ccgIntroducedPtr(s: PSym): bool = var pt = skipTypes(s.typ, typedescInst) @@ -258,6 +260,11 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = of tyInt..tyUInt64: result = typeNameOrLiteral(typ, NumericalTypeToStr[typ.kind]) of tyDistinct, tyRange, tyOrdinal: result = getSimpleTypeDesc(m, typ.sons[0]) + of tyStatic: + if typ.n != nil: result = getSimpleTypeDesc(m, lastSon typ) + else: internalError("tyStatic for getSimpleTypeDesc") + of tyGenericInst: + result = getSimpleTypeDesc(m, lastSon typ) else: result = nil proc pushType(m: BModule, typ: PType) = @@ -418,7 +425,7 @@ proc genRecordFieldsAux(m: BModule, n: PNode, addf(result, "union{$n$1} $2;$n", [unionBody, uname]) of nkSym: field = n.sym - if field.typ.kind == tyEmpty: return + if field.typ.kind == tyVoid: return #assert(field.ast == nil) sname = mangleRecFieldName(field, rectype) if accessExpr != nil: ae = "$1.$2" % [accessExpr, sname] @@ -665,7 +672,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = chunkStart = i let typeInSlot = resolveStarsInCppType(typ, idx + 1, stars) - if typeInSlot == nil or typeInSlot.kind == tyEmpty: + if typeInSlot == nil or typeInSlot.kind == tyVoid: result.add(~"void") else: result.add getTypeDescAux(m, typeInSlot, check) @@ -724,7 +731,7 @@ type proc getClosureType(m: BModule, t: PType, kind: TClosureTypeKind): Rope = assert t.kind == tyProc var check = initIntSet() - result = getTempName() + result = getTempName(m) var rettype, desc: Rope genProcParams(m, t, rettype, desc, check, declareEnvironment=kind != clHalf) if not isImportedType(t): @@ -839,7 +846,7 @@ proc genObjectFields(m: BModule, typ: PType, n: PNode, expr: Rope) = if L == 1: genObjectFields(m, typ, n.sons[0], expr) elif L > 0: - var tmp = getTempName() + var tmp = getTempName(m) addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(L)]) for i in countup(0, L-1): var tmp2 = getNimNode(m) @@ -911,7 +918,7 @@ proc genTupleInfo(m: BModule, typ: PType, name: Rope) = var expr = getNimNode(m) var length = sonsLen(typ) if length > 0: - var tmp = getTempName() + var tmp = getTempName(m) addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(length)]) for i in countup(0, length - 1): var a = typ.sons[i] @@ -935,7 +942,7 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope) = # anyway. We generate a cstring array and a loop over it. Exceptional # positions will be reset after the loop. genTypeInfoAux(m, typ, typ, name) - var nodePtrs = getTempName() + var nodePtrs = getTempName(m) var length = sonsLen(typ.n) addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [nodePtrs, rope(length)]) @@ -955,8 +962,8 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope) = if field.position != i or tfEnumHasHoles in typ.flags: addf(specialCases, "$1.offset = $2;$n", [elemNode, rope(field.position)]) hasHoles = true - var enumArray = getTempName() - var counter = getTempName() + var enumArray = getTempName(m) + var counter = getTempName(m) addf(m.s[cfsTypeInit1], "NI $1;$n", [counter]) addf(m.s[cfsTypeInit1], "static char* NIM_CONST $1[$2] = {$n$3};$n", [enumArray, rope(length), enumNames]) @@ -1024,9 +1031,12 @@ proc genTypeInfo(m: BModule, t: PType): Rope = [result, rope(typeToString(t))]) return "(&".rope & result & ")".rope case t.kind - of tyEmpty: result = rope"0" + of tyEmpty, tyVoid: result = rope"0" of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar: genTypeInfoAuxBase(m, t, t, result, rope"0") + of tyStatic: + if t.n != nil: result = genTypeInfo(m, lastSon t) + else: internalError("genTypeInfo(" & $t.kind & ')') of tyProc: if t.callConv != ccClosure: genTypeInfoAuxBase(m, t, t, result, rope"0") diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim index 6dfd7b52c..ecd98a2bf 100644 --- a/compiler/ccgutils.nim +++ b/compiler/ccgutils.nim @@ -93,7 +93,7 @@ proc getUniqueType*(key: PType): PType = # produced instead of ``NI``. result = key of tyEmpty, tyNil, tyExpr, tyStmt, tyPointer, tyString, - tyCString, tyNone, tyBigNum: + tyCString, tyNone, tyBigNum, tyVoid: result = gCanonicalTypes[k] if result == nil: gCanonicalTypes[k] = key @@ -188,7 +188,7 @@ proc mangle*(name: string): string = let c = name[i] case c of 'A'..'Z': - add(result, c.toLower) + add(result, c.toLowerAscii) of '_': discard of 'a'..'z', '0'..'9': diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 3334f047f..589fcb515 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -500,7 +500,7 @@ proc loadDynamicLib(m: BModule, lib: PLib) = assert(lib != nil) if not lib.generated: lib.generated = true - var tmp = getGlobalTempName() + var tmp = getTempName(m) assert(lib.name == nil) lib.name = tmp # BUGFIX: cgsym has awful side-effects addf(m.s[cfsVars], "static void* $1;$n", [tmp]) @@ -1084,6 +1084,7 @@ proc initProcOptions(m: BModule): TOptions = proc rawNewModule(module: PSym, filename: string): BModule = new(result) + result.tmpBase = rope("T" & $hashOwner(module) & "_") initLinkedList(result.headerFiles) result.declaredThings = initIntSet() result.declaredProtos = initIntSet() @@ -1100,8 +1101,8 @@ proc rawNewModule(module: PSym, filename: string): BModule = initNodeTable(result.dataCache) result.typeStack = @[] result.forwardedProcs = @[] - result.typeNodesName = getTempName() - result.nimTypesName = getTempName() + result.typeNodesName = getTempName(result) + result.nimTypesName = getTempName(result) # no line tracing for the init sections of the system module so that we # don't generate a TFrame which can confuse the stack botton initialization: if sfSystemModule in module.flags: @@ -1126,8 +1127,8 @@ proc resetModule*(m: BModule) = initNodeTable(m.dataCache) m.typeStack = @[] m.forwardedProcs = @[] - m.typeNodesName = getTempName() - m.nimTypesName = getTempName() + m.typeNodesName = getTempName(m) + m.nimTypesName = getTempName(m) if sfSystemModule in m.module.flags: incl m.flags, preventStackTrace else: diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index c098902a6..c027bb451 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -102,12 +102,13 @@ type includesStringh, # C source file already includes ``<string.h>`` objHasKidsValid # whether we can rely on tfObjHasKids TCGen = object of TPassContext # represents a C source file - module*: PSym - filename*: string s*: TCFileSections # sections of the C file flags*: set[Codegenflag] + module*: PSym + filename*: string cfilename*: string # filename of the module (including path, # without extension) + tmpBase*: Rope # base for temp identifier generation typeCache*: TIdTable # cache the generated types forwTypeCache*: TIdTable # cache for forward declarations of types declaredThings*: IntSet # things we have declared in this .c file diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index 312afec1a..624c5b183 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -63,19 +63,25 @@ proc sameMethodBucket(a, b: PSym): MethodResult = while true: aa = skipTypes(aa, {tyGenericInst}) bb = skipTypes(bb, {tyGenericInst}) - if (aa.kind == bb.kind) and (aa.kind in {tyVar, tyPtr, tyRef}): + if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef}: aa = aa.lastSon bb = bb.lastSon else: break if sameType(aa, bb): - if aa.kind == tyObject and result != Invalid: result = Yes + if aa.kind == tyObject and result != Invalid: + result = Yes elif aa.kind == tyObject and bb.kind == tyObject: let diff = inheritanceDiff(bb, aa) if diff < 0: - if result != Invalid: result = Yes + if result != Invalid: + result = Yes + else: + return No elif diff != high(int): result = Invalid + else: + return No else: return No @@ -181,7 +187,7 @@ proc cmpSignatures(a, b: PSym, relevantCols: IntSet): int = var aa = skipTypes(a.typ.sons[col], skipPtrs) var bb = skipTypes(b.typ.sons[col], skipPtrs) var d = inheritanceDiff(aa, bb) - if (d != high(int)): + if (d != high(int)) and d != 0: return d proc sortBucket(a: var TSymSeq, relevantCols: IntSet) = diff --git a/compiler/commands.nim b/compiler/commands.nim index 5601ef662..b468dd6b8 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -158,7 +158,7 @@ var enableNotes: TNoteKinds disableNotes: TNoteKinds -proc processSpecificNote(arg: string, state: TSpecialWord, pass: TCmdLinePass, +proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, info: TLineInfo; orig: string) = var id = "" # arg = "X]:on|off" var i = 0 @@ -181,10 +181,13 @@ proc processSpecificNote(arg: string, state: TSpecialWord, pass: TCmdLinePass, case whichKeyword(substr(arg, i)) of wOn: incl(gNotes, n) + incl(gMainPackageNotes, n) incl(enableNotes, n) of wOff: excl(gNotes, n) + excl(gMainPackageNotes, n) incl(disableNotes, n) + excl(ForeignPackageNotes, n) else: localError(info, errOnOrOffExpectedButXFound, arg) proc processCompile(filename: string) = @@ -341,7 +344,11 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = discard "allow for backwards compatibility, but don't do anything" of "define", "d": expectArg(switch, arg, pass, info) - defineSymbol(arg) + if {':', '='} in arg: + splitSwitch(arg, key, val, pass, info) + defineSymbol(key, val) + else: + defineSymbol(arg) of "undef", "u": expectArg(switch, arg, pass, info) undefSymbol(arg) @@ -543,6 +550,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = gNotes = NotesVerbosity[gVerbosity] incl(gNotes, enableNotes) excl(gNotes, disableNotes) + gMainPackageNotes = gNotes of "parallelbuild": expectArg(switch, arg, pass, info) gNumberOfProcessors = parseInt(arg) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 60e8f2826..02f7e764d 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -19,8 +19,8 @@ var gSymbols: StringTableRef const catNone = "false" -proc defineSymbol*(symbol: string) = - gSymbols[symbol] = "true" +proc defineSymbol*(symbol: string, value: string = "true") = + gSymbols[symbol] = value proc undefSymbol*(symbol: string) = gSymbols[symbol] = catNone @@ -62,6 +62,11 @@ proc isDefined*(symbol: string): bool = proc isDefined*(symbol: PIdent): bool = isDefined(symbol.s) +proc lookupSymbol*(symbol: string): string = + result = if isDefined(symbol): gSymbols[symbol] else: nil + +proc lookupSymbol*(symbol: PIdent): string = lookupSymbol(symbol.s) + iterator definedSymbolNames*: string = for key, val in pairs(gSymbols): if val != catNone: yield key diff --git a/compiler/docgen.nim b/compiler/docgen.nim index d954b897b..bbbec081a 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -14,7 +14,7 @@ import ast, strutils, strtabs, options, msgs, os, ropes, idents, wordrecg, syntaxes, renderer, lexer, rstast, rst, rstgen, times, highlite, - importer, sempass2, json, xmltree, cgi, typesrenderer + importer, sempass2, json, xmltree, cgi, typesrenderer, astalgo type TSections = array[TSymKind, Rope] @@ -25,9 +25,32 @@ type indexValFilename: string analytics: string # Google Analytics javascript, "" if doesn't exist seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML. + jArray: JsonNode + types: TStrTable PDoc* = ref TDocumentor ## Alias to type less. +proc whichType(d: PDoc; n: PNode): PSym = + if n.kind == nkSym: + if d.types.strTableContains(n.sym): + result = n.sym + else: + for i in 0..<safeLen(n): + let x = whichType(d, n[i]) + if x != nil: return x + +proc attachToType(d: PDoc; p: PSym): PSym = + let params = p.ast.sons[paramsPos] + # first check the first parameter, then the return type, + # then the other parameter: + template check(i) = + result = whichType(d, params[i]) + if result != nil: return result + + if params.len > 1: check(1) + if params.len > 0: check(0) + for i in 2..<params.len: check(i) + proc compilerMsgHandler(filename: string, line, col: int, msgKind: rst.MsgKind, arg: string) {.procvar.} = # translate msg kind: @@ -81,6 +104,8 @@ proc newDocumentor*(filename: string, config: StringTableRef): PDoc = result.seenSymbols = newStringTable(modeCaseInsensitive) result.id = 100 + result.jArray = newJArray() + initStrTable result.types proc dispA(dest: var Rope, xml, tex: string, args: openArray[Rope]) = if gCmd != cmdRst2tex: addf(dest, xml, args) @@ -226,10 +251,27 @@ proc getName(d: PDoc, n: PNode, splitAfter = -1): string = result = esc(d.target, "`") for i in 0.. <n.len: result.add(getName(d, n[i], splitAfter)) result.add esc(d.target, "`") + of nkOpenSymChoice, nkClosedSymChoice: + result = getName(d, n[0], splitAfter) else: internalError(n.info, "getName()") result = "" +proc getNameIdent(n: PNode): PIdent = + case n.kind + of nkPostfix: result = getNameIdent(n.sons[1]) + of nkPragmaExpr: result = getNameIdent(n.sons[0]) + of nkSym: result = n.sym.name + of nkIdent: result = n.ident + of nkAccQuoted: + var r = "" + for i in 0.. <n.len: r.add(getNameIdent(n[i]).s) + result = getIdent(r) + of nkOpenSymChoice, nkClosedSymChoice: + result = getNameIdent(n[0]) + else: + result = nil + proc getRstName(n: PNode): PRstNode = case n.kind of nkPostfix: result = getRstName(n.sons[1]) @@ -239,6 +281,8 @@ proc getRstName(n: PNode): PRstNode = of nkAccQuoted: result = getRstName(n.sons[0]) for i in 1 .. <n.len: result.text.add(getRstName(n[i]).text) + of nkOpenSymChoice, nkClosedSymChoice: + result = getRstName(n[0]) else: internalError(n.info, "getRstName()") result = nil @@ -311,7 +355,7 @@ proc docstringSummary(rstText: string): string = ## Also, we hope to not break the rst, but maybe we do. If there is any ## trimming done, an ellipsis unicode char is added. const maxDocstringChars = 100 - assert (rstText.len < 2 or (rstText[0] == '#' and rstText[1] == '#')) + assert(rstText.len < 2 or (rstText[0] == '#' and rstText[1] == '#')) result = rstText.substr(2).strip var pos = result.find('\L') if pos > 0: @@ -380,13 +424,27 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = "\\spanIdentifier{$1}", [rope(esc(d.target, literal))]) of tkSpaces, tkInvalid: add(result, literal) + of tkCurlyDotLe: + dispA(result, """<span class="Other pragmabegin">$1</span><div class="pragma">""", + "\\spanOther{$1}", + [rope(esc(d.target, literal))]) + of tkCurlyDotRi: + dispA(result, "</div><span class=\"Other pragmaend\">$1</span>", + "\\spanOther{$1}", + [rope(esc(d.target, literal))]) of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi, - tkBracketDotLe, tkBracketDotRi, tkCurlyDotLe, tkCurlyDotRi, tkParDotLe, + tkBracketDotLe, tkBracketDotRi, tkParDotLe, tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot, tkAccent, tkColonColon, tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr: dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}", [rope(esc(d.target, literal))]) + + if k in routineKinds and nameNode.kind == nkSym: + let att = attachToType(d, nameNode.sym) + if att != nil: + dispA(result, """<span class="attachedType" style="visibility:hidden">$1</span>""", "", + [rope esc(d.target, att.name.s)]) inc(d.id) let plainNameRope = rope(xmltree.escape(plainName.strip)) @@ -430,8 +488,10 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = setIndexTerm(d[], symbolOrId, name, linkTitle, xmltree.escape(plainDocstring.docstringSummary)) + if k == skType and nameNode.kind == nkSym: + d.types.strTableAdd nameNode.sym -proc genJSONItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = +proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = if not isVisible(nameNode): return var name = getName(d, nameNode) @@ -440,8 +500,8 @@ proc genJSONItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments}) - result = %{ "name": %name, "type": %($k) } - + result = %{ "name": %name, "type": %($k), "line": %n.info.line, + "col": %n.info.col} if comm != nil and comm != "": result["description"] = %comm if r.buf != nil: @@ -491,46 +551,45 @@ proc generateDoc*(d: PDoc, n: PNode) = of nkFromStmt, nkImportExceptStmt: traceDeps(d, n.sons[0]) else: discard -proc generateJson(d: PDoc, n: PNode, jArray: JsonNode = nil): JsonNode = +proc add(d: PDoc; j: JsonNode) = + if j != nil: d.jArray.add j + +proc generateJson*(d: PDoc, n: PNode) = case n.kind of nkCommentStmt: if n.comment != nil and startsWith(n.comment, "##"): let stripped = n.comment.substr(2).strip - result = %{ "comment": %stripped } + d.add %{ "comment": %stripped, "line": %n.info.line, + "col": %n.info.col } of nkProcDef: when useEffectSystem: documentRaises(n) - result = genJSONItem(d, n, n.sons[namePos], skProc) + d.add genJsonItem(d, n, n.sons[namePos], skProc) of nkMethodDef: when useEffectSystem: documentRaises(n) - result = genJSONItem(d, n, n.sons[namePos], skMethod) + d.add genJsonItem(d, n, n.sons[namePos], skMethod) of nkIteratorDef: when useEffectSystem: documentRaises(n) - result = genJSONItem(d, n, n.sons[namePos], skIterator) + d.add genJsonItem(d, n, n.sons[namePos], skIterator) of nkMacroDef: - result = genJSONItem(d, n, n.sons[namePos], skMacro) + d.add genJsonItem(d, n, n.sons[namePos], skMacro) of nkTemplateDef: - result = genJSONItem(d, n, n.sons[namePos], skTemplate) + d.add genJsonItem(d, n, n.sons[namePos], skTemplate) of nkConverterDef: when useEffectSystem: documentRaises(n) - result = genJSONItem(d, n, n.sons[namePos], skConverter) + d.add genJsonItem(d, n, n.sons[namePos], skConverter) of nkTypeSection, nkVarSection, nkLetSection, nkConstSection: for i in countup(0, sonsLen(n) - 1): if n.sons[i].kind != nkCommentStmt: # order is always 'type var let const': - result = genJSONItem(d, n.sons[i], n.sons[i].sons[0], + d.add genJsonItem(d, n.sons[i], n.sons[i].sons[0], succ(skType, ord(n.kind)-ord(nkTypeSection))) of nkStmtList: - result = if jArray != nil: jArray else: newJArray() - for i in countup(0, sonsLen(n) - 1): - var r = generateJson(d, n.sons[i], result) - if r != nil: - result.add(r) - + generateJson(d, n.sons[i]) of nkWhenStmt: # generate documentation for the first branch only: - if not checkForFalse(n.sons[0].sons[0]) and jArray != nil: - discard generateJson(d, lastSon(n.sons[0]), jArray) + if not checkForFalse(n.sons[0].sons[0]): + generateJson(d, lastSon(n.sons[0])) else: discard proc genSection(d: PDoc, kind: TSymKind) = @@ -592,12 +651,36 @@ proc generateIndex*(d: PDoc) = writeIndexFile(d[], splitFile(options.outFile).dir / splitFile(d.filename).name & IndexExt) +proc getOutFile2(filename, ext, dir: string): string = + if gWholeProject: + let d = if options.outFile != "": options.outFile else: dir + createDir(d) + result = d / changeFileExt(filename, ext) + else: + result = getOutFile(filename, ext) + proc writeOutput*(d: PDoc, filename, outExt: string, useWarning = false) = var content = genOutFile(d) if optStdout in gGlobalOptions: writeRope(stdout, content) else: - writeRope(content, getOutFile(filename, outExt), useWarning) + writeRope(content, getOutFile2(filename, outExt, "htmldocs"), useWarning) + +proc writeOutputJson*(d: PDoc, filename, outExt: string, + useWarning = false) = + let content = %*{"orig": d.filename, + "nimble": getPackageName(d.filename), + "entries": d.jArray} + if optStdout in gGlobalOptions: + write(stdout, $content) + else: + var f: File + if open(f, getOutFile2(splitFile(filename).name, + outExt, "jsondocs"), fmWrite): + write(f, $content) + close(f) + else: + discard "fixme: error report" proc commandDoc*() = var ast = parseFile(gProjectMainIdx) @@ -628,18 +711,19 @@ proc commandRst2TeX*() = splitter = "\\-" commandRstAux(gProjectFull, TexExt) -proc commandJSON*() = +proc commandJson*() = var ast = parseFile(gProjectMainIdx) if ast == nil: return var d = newDocumentor(gProjectFull, options.gConfigVars) d.hasToc = true - var json = generateJson(d, ast) - var content = rope(pretty(json)) + generateJson(d, ast) + let json = d.jArray + let content = rope(pretty(json)) if optStdout in gGlobalOptions: writeRope(stdout, content) else: - echo getOutFile(gProjectFull, JsonExt) + #echo getOutFile(gProjectFull, JsonExt) writeRope(content, getOutFile(gProjectFull, JsonExt), useWarning = false) proc commandBuildIndex*() = diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index 27de06811..d70d5406c 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -19,21 +19,36 @@ type module: PSym PGen = ref TGen -proc close(p: PPassContext, n: PNode): PNode = +template closeImpl(body: untyped) {.dirty.} = var g = PGen(p) let useWarning = sfMainModule notin g.module.flags - if gWholeProject or sfMainModule in g.module.flags: - writeOutput(g.doc, g.module.filename, HtmlExt, useWarning) + #echo g.module.name.s, " ", g.module.owner.id, " ", gMainPackageId + if (g.module.owner.id == gMainPackageId and gWholeProject) or + sfMainModule in g.module.flags: + body try: generateIndex(g.doc) except IOError: discard +proc close(p: PPassContext, n: PNode): PNode = + closeImpl: + writeOutput(g.doc, g.module.filename, HtmlExt, useWarning) + +proc closeJson(p: PPassContext, n: PNode): PNode = + closeImpl: + writeOutputJson(g.doc, g.module.filename, ".json", useWarning) + proc processNode(c: PPassContext, n: PNode): PNode = result = n var g = PGen(c) generateDoc(g.doc, n) +proc processNodeJson(c: PPassContext, n: PNode): PNode = + result = n + var g = PGen(c) + generateJson(g.doc, n) + proc myOpen(module: PSym): PPassContext = var g: PGen new(g) @@ -44,6 +59,8 @@ proc myOpen(module: PSym): PPassContext = result = g const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close) +const docgen2JsonPass* = makePass(open = myOpen, process = processNodeJson, + close = closeJson) proc finishDoc2Pass*(project: string) = discard diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index b2ee9c7f1..eb9aac490 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -754,7 +754,7 @@ proc callCCompiler*(projectfile: string) = "lib", quoteShell(libpath)]) if optCompileOnly notin gGlobalOptions: execExternalProgram(linkCmd, - if gVerbosity > 1: hintExecuting else: hintLinking) + if optListCmd in gGlobalOptions or gVerbosity > 1: hintExecuting else: hintLinking) else: linkCmd = "" if optGenScript in gGlobalOptions: diff --git a/compiler/idgen.nim b/compiler/idgen.nim index 906c16546..333772705 100644 --- a/compiler/idgen.nim +++ b/compiler/idgen.nim @@ -11,7 +11,7 @@ import idents, strutils, os, options -var gFrontEndId, gBackendId*: int +var gFrontEndId*: int const debugIds* = false @@ -30,10 +30,6 @@ proc getID*(): int {.inline.} = result = gFrontEndId inc(gFrontEndId) -proc backendId*(): int {.inline.} = - result = gBackendId - inc(gBackendId) - proc setId*(id: int) {.inline.} = gFrontEndId = max(gFrontEndId, id + 1) @@ -49,7 +45,6 @@ proc toGid(f: string): string = proc saveMaxIds*(project: string) = var f = open(project.toGid, fmWrite) f.writeLine($gFrontEndId) - f.writeLine($gBackendId) f.close() proc loadMaxIds*(project: string) = @@ -61,5 +56,4 @@ proc loadMaxIds*(project: string) = if f.readLine(line): var backEndId = parseInt(line) gFrontEndId = max(gFrontEndId, frontEndId) - gBackendId = max(gBackendId, backEndId) f.close() diff --git a/compiler/installer.ini b/compiler/installer.ini index 07fdb13b5..b802f08f1 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -8,7 +8,7 @@ Platforms: """ windows: i386;amd64 linux: i386;amd64;powerpc64;arm;sparc;mips;mipsel;powerpc;powerpc64el;arm64 macosx: i386;amd64;powerpc64 - solaris: i386;amd64;sparc + solaris: i386;amd64;sparc;sparc64 freebsd: i386;amd64 netbsd: i386;amd64 openbsd: i386;amd64 diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 41e42b825..a60b0cc5e 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -48,6 +48,7 @@ type etyNull, # null type etyProc, # proc type etyBool, # bool type + etySeq, # Nim seq or string type etyInt, # JavaScript's int etyFloat, # JavaScript's float etyString, # JavaScript's string @@ -156,15 +157,18 @@ proc mapType(typ: PType): TJSTypeKind = of tyBool: result = etyBool of tyFloat..tyFloat128: result = etyFloat of tySet: result = etyObject # map a set to a table - of tyString, tySequence: result = etyInt # little hack to get right semantics + of tyString, tySequence: result = etySeq of tyObject, tyArray, tyArrayConstr, tyTuple, tyOpenArray, tyBigNum, tyVarargs: result = etyObject of tyNil: result = etyNull of tyGenericInst, tyGenericParam, tyGenericBody, tyGenericInvocation, tyNone, tyFromExpr, tyForward, tyEmpty, tyFieldAccessor, - tyExpr, tyStmt, tyStatic, tyTypeDesc, tyTypeClasses: + tyExpr, tyStmt, tyTypeDesc, tyTypeClasses, tyVoid: result = etyNone + of tyStatic: + if t.n != nil: result = mapType(lastSon t) + else: result = etyNone of tyProc: result = etyProc of tyCString: result = etyString @@ -817,7 +821,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = addf(p.body, "$#[$#] = chr($#);$n", [a.rdLoc, b.rdLoc, c.rdLoc]) return - let xtyp = mapType(p, x.typ) + var xtyp = mapType(p, x.typ) if x.kind == nkHiddenDeref and x.sons[0].kind == nkCall and xtyp != etyObject: gen(p, x.sons[0], a) @@ -829,7 +833,18 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = gen(p, y, b) + # we don't care if it's an etyBaseIndex (global) of a string, it's + # still a string that needs to be copied properly: + if x.typ.skipTypes(abstractInst).kind in {tySequence, tyString}: + xtyp = etySeq case xtyp + of etySeq: + if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: + addf(p.body, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) + else: + useMagic(p, "nimCopy") + addf(p.body, "$1 = nimCopy(null, $2, $3);$n", + [a.rdLoc, b.res, genTypeInfo(p, y.typ)]) of etyObject: if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: addf(p.body, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) @@ -1049,6 +1064,8 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) = else: internalError(n.sons[0].info, "expr(nkBracketExpr, " & $kindOfIndexedExpr & ')') of nkObjDownConv: gen(p, n.sons[0], r) + of nkHiddenDeref: + gen(p, n.sons[0].sons[0], r) else: internalError(n.sons[0].info, "genAddr: " & $n.sons[0].kind) proc thisParam(p: PProc; typ: PType): PType = @@ -1215,11 +1232,13 @@ proc genOtherArg(p: PProc; n: PNode; i: int; typ: PType; genArgNoParam(p, it, r) else: genArg(p, it, paramType.sym, r) + inc generated proc genPatternCall(p: PProc; n: PNode; pat: string; typ: PType; r: var TCompRes) = var i = 0 var j = 1 + r.kind = resExpr while i < pat.len: case pat[i] of '@': @@ -1233,10 +1252,18 @@ proc genPatternCall(p: PProc; n: PNode; pat: string; typ: PType; genOtherArg(p, n, j, typ, generated, r) inc j inc i + of '\31': + # unit separator + add(r.res, "#") + inc i + of '\29': + # group separator + add(r.res, "@") + inc i else: let start = i while i < pat.len: - if pat[i] notin {'@', '#'}: inc(i) + if pat[i] notin {'@', '#', '\31', '\29'}: inc(i) else: break if i - 1 >= start: add(r.res, substr(pat, start, i - 1)) @@ -1403,6 +1430,12 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope = result = putToSeq("null", indirect) of tySequence, tyString, tyCString, tyPointer, tyProc: result = putToSeq("null", indirect) + of tyStatic: + if t.n != nil: + result = createVar(p, lastSon t, indirect) + else: + internalError("createVar: " & $t.kind) + result = nil else: internalError("createVar: " & $t.kind) result = nil @@ -1418,7 +1451,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = discard mangleName(v, p.target) gen(p, n, a) case mapType(p, v.typ) - of etyObject: + of etyObject, etySeq: if needsNoCopy(p, n): s = a.res else: @@ -1557,8 +1590,16 @@ proc genRepr(p: PProc, n: PNode, r: var TCompRes) = of tyEnum, tyOrdinal: gen(p, n.sons[1], r) useMagic(p, "cstrToNimstr") + var offset = "" + if t.kind == tyEnum: + let firstFieldOffset = t.n.sons[0].sym.position + if firstFieldOffset < 0: + offset = "+" & $(-firstFieldOffset) + elif firstFieldOffset > 0: + offset = "-" & $firstFieldOffset + r.kind = resExpr - r.res = "cstrToNimstr($1.node.sons[$2].name)" % [genTypeInfo(p, t), r.res] + r.res = "cstrToNimstr($1.node.sons[$2$3].name)" % [genTypeInfo(p, t), r.res, rope(offset)] else: # XXX: internalError(n.info, "genRepr: Not implemented") @@ -1678,8 +1719,9 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = else: if optOverflowCheck notin p.options: binaryExpr(p, n, r, "", "$1 -= $2") else: binaryExpr(p, n, r, "subInt", "$1 = subInt($1, $2)") - of mSetLengthStr: binaryExpr(p, n, r, "", "$1.length = $2+1; $1[$1.length-1] = 0" | "") - of mSetLengthSeq: binaryExpr(p, n, r, "", "$1.length = $2" | "") + of mSetLengthStr: + binaryExpr(p, n, r, "", "$1.length = $2+1; $1[$1.length-1] = 0" | "$1 = substr($1, 0, $2)") + of mSetLengthSeq: binaryExpr(p, n, r, "", "$1.length = $2" | "$1 = array_slice($1, 0, $2)") of mCard: unaryExpr(p, n, r, "SetCard", "SetCard($1)") of mLtSet: binaryExpr(p, n, r, "SetLt", "SetLt($1, $2)") of mLeSet: binaryExpr(p, n, r, "SetLe", "SetLe($1, $2)") @@ -1689,8 +1731,31 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = of mMinusSet: binaryExpr(p, n, r, "SetMinus", "SetMinus($1, $2)") of mIncl: binaryExpr(p, n, r, "", "$1[$2] = true") of mExcl: binaryExpr(p, n, r, "", "delete $1[$2]" | "unset $1[$2]") - of mInSet: binaryExpr(p, n, r, "", "($1[$2] != undefined)" | "isset($1[$2])") + of mInSet: + if p.target == targetJS: + binaryExpr(p, n, r, "", "($1[$2] != undefined)") + else: + let s = n.sons[1] + if s.kind == nkCurly: + var a, b, x: TCompRes + gen(p, n.sons[2], x) + r.res = rope("(") + r.kind = resExpr + for i in countup(0, sonsLen(s) - 1): + if i > 0: add(r.res, " || ") + var it = s.sons[i] + if it.kind == nkRange: + gen(p, it.sons[0], a) + gen(p, it.sons[1], b) + addf(r.res, "($1 >= $2 && $1 <= $3)", [x.res, a.res, b.res,]) + else: + gen(p, it, a) + addf(r.res, "($1 == $2)", [x.res, a.res]) + add(r.res, ")") + else: + binaryExpr(p, n, r, "", "isset($1[$2])") of mNewSeq: genNewSeq(p, n) + of mNewSeqOfCap: unaryExpr(p, n, r, "", "[]" | "array()") of mOf: genOf(p, n, r) of mReset: genReset(p, n) of mEcho: genEcho(p, n, r) @@ -1989,7 +2054,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = if (n.sons[0].kind == nkSym) and (n.sons[0].sym.magic != mNone): genMagic(p, n, r) elif n.sons[0].kind == nkSym and sfInfixCall in n.sons[0].sym.flags and - n.len >= 2: + n.len >= 1: genInfixCall(p, n, r) else: genCall(p, n, r) diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim index 8d109e48a..cf1679ee4 100644 --- a/compiler/jstypes.nim +++ b/compiler/jstypes.nim @@ -165,4 +165,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope = of tyEnum: genEnumInfo(p, t, result) of tyObject: genObjectInfo(p, t, result) of tyTuple: genTupleInfo(p, t, result) + of tyStatic: + if t.n != nil: result = genTypeInfo(p, lastSon t) + else: internalError("genTypeInfo(" & $t.kind & ')') else: internalError("genTypeInfo(" & $t.kind & ')') diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 8b201431e..0a96ed0ba 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -735,7 +735,8 @@ proc getSymbol(L: var TLexer, tok: var TToken) = if c == '\226' and buf[pos+1] == '\128' and buf[pos+2] == '\147': # It's a 'magic separator' en-dash Unicode - if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars: + if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars or + isMagicIdentSeparatorRune(buf, pos+magicIdentSeparatorRuneByteWidth) or pos == L.bufpos: lexMessage(L, errInvalidToken, "–") break inc(pos, magicIdentSeparatorRuneByteWidth) @@ -747,7 +748,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) = h = h !& ord(c) inc(pos) of '_': - if buf[pos+1] notin SymChars: + if buf[pos+1] notin SymChars or isMagicIdentSeparatorRune(buf, pos+1): lexMessage(L, errInvalidToken, "_") break inc(pos) @@ -1056,7 +1057,8 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = inc(L.bufpos) of '_': inc(L.bufpos) - if L.buf[L.bufpos] notin SymChars: + if L.buf[L.bufpos] notin SymChars+{'_'} and not + isMagicIdentSeparatorRune(L.buf, L.bufpos): tok.tokType = tkSymbol tok.ident = getIdent("_") else: @@ -1077,6 +1079,9 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = tok.tokType = tkCharLit of '0'..'9': getNumber(L, tok) + let c = L.buf[L.bufpos] + if c in SymChars+{'_'}: + lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')') else: if c in OpChars: getOperator(L, tok) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 962c28613..ba2358b86 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -153,7 +153,8 @@ proc ensureNoMissingOrUnusedSymbols(scope: PScope) = if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam}: # XXX: implicit type params are currently skTypes # maybe they can be made skGenericParam as well. - if s.typ != nil and tfImplicitTypeParam notin s.typ.flags: + if s.typ != nil and tfImplicitTypeParam notin s.typ.flags and + s.typ.kind != tyGenericParam: message(s.info, hintXDeclaredButNotUsed, getSymRepr(s)) s = nextIter(it, scope.symbols) diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 7a5c7f44b..cdea2a4ff 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -471,6 +471,7 @@ proc setupArgsForParallelism(n: PNode; objType: PType; scratchObj: PSym; let slice = newNodeI(nkCall, n.info, 4) slice.typ = n.typ slice.sons[0] = newSymNode(createMagic("slice", mSlice)) + slice.sons[0].typ = getSysType(tyInt) # fake type var fieldB = newSym(skField, tmpName, objType.owner, n.info) fieldB.typ = getSysType(tyInt) objType.addField(fieldB) diff --git a/compiler/main.nim b/compiler/main.nim index b1b9006bd..0db66b53e 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -46,10 +46,11 @@ proc commandCheck = rodPass() compileProject() -proc commandDoc2 = +proc commandDoc2(json: bool) = msgs.gErrorMax = high(int) # do not stop after first error semanticPasses() - registerPass(docgen2Pass) + if json: registerPass(docgen2JsonPass) + else: registerPass(docgen2Pass) #registerPass(cleanupPass()) compileProject() finishDoc2Pass(gProjectName) @@ -281,7 +282,7 @@ proc mainCommand* = gCmd = cmdDoc loadConfigs(DocConfig) defineSymbol("nimdoc") - commandDoc2() + commandDoc2(false) of "rst2html": gCmd = cmdRst2html loadConfigs(DocConfig) @@ -296,7 +297,13 @@ proc mainCommand* = loadConfigs(DocConfig) wantMainModule() defineSymbol("nimdoc") - commandJSON() + commandJson() + of "jsondoc2": + gCmd = cmdDoc + loadConfigs(DocConfig) + wantMainModule() + defineSymbol("nimdoc") + commandDoc2(true) of "buildindex": gCmd = cmdDoc loadConfigs(DocConfig) diff --git a/compiler/modules.nim b/compiler/modules.nim index 8ac964321..8df300f5f 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -29,6 +29,10 @@ var gMemCacheData*: seq[TModuleInMemory] = @[] ## XXX: we should implement recycling of file IDs ## if the user keeps renaming modules, the file IDs will keep growing + gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why. + packageSyms: TStrTable + +initStrTable(packageSyms) proc getModule*(fileIdx: int32): PSym = if fileIdx >= 0 and fileIdx < gCompiledModules.len: @@ -90,6 +94,7 @@ proc resetAllModules* = if gCompiledModules[i] != nil: resetModule(i.int32) resetPackageCache() + initStrTable(packageSyms) # for m in cgenModules(): echo "CGEN MODULE FOUND" proc resetAllModulesHard* = @@ -97,6 +102,7 @@ proc resetAllModulesHard* = gCompiledModules.setLen 0 gMemCacheData.setLen 0 magicsys.resetSysTypes() + initStrTable(packageSyms) # XXX #gOwners = @[] @@ -105,12 +111,16 @@ proc checkDepMem(fileIdx: int32): TNeedRecompile = resetModule(fileIdx) return Yes - if gMemCacheData[fileIdx].needsRecompile != Maybe: - return gMemCacheData[fileIdx].needsRecompile + if gFuzzyGraphChecking: + if gMemCacheData[fileIdx].needsRecompile != Maybe: + return gMemCacheData[fileIdx].needsRecompile + else: + # cycle detection: We claim that a cycle does no harm. + if gMemCacheData[fileIdx].needsRecompile == Probing: + return No - if optForceFullMake in gGlobalOptions or - hashChanged(fileIdx): - markDirty + if optForceFullMake in gGlobalOptions or hashChanged(fileIdx): + markDirty() if gMemCacheData[fileIdx].deps != nil: gMemCacheData[fileIdx].needsRecompile = Probing @@ -118,7 +128,7 @@ proc checkDepMem(fileIdx: int32): TNeedRecompile = let d = checkDepMem(dep) if d in {Yes, Recompiled}: # echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d - markDirty + markDirty() gMemCacheData[fileIdx].needsRecompile = No return No @@ -135,8 +145,16 @@ proc newModule(fileIdx: int32): PSym = rawMessage(errInvalidModuleName, result.name.s) result.info = newLineInfo(fileIdx, 1, 1) - result.owner = newSym(skPackage, getIdent(getPackageName(filename)), nil, - result.info) + let pack = getIdent(getPackageName(filename)) + var packSym = packageSyms.strTableGet(pack) + if packSym == nil: + let pck = getPackageName(filename) + let pck2 = if pck.len > 0: pck else: "unknown" + packSym = newSym(skPackage, getIdent(pck2), nil, result.info) + initStrTable(packSym.tab) + packageSyms.strTableAdd(packSym) + + result.owner = packSym result.position = fileIdx growCache gMemCacheData, fileIdx @@ -146,6 +164,11 @@ proc newModule(fileIdx: int32): PSym = incl(result.flags, sfUsed) initStrTable(result.tab) strTableAdd(result.tab, result) # a module knows itself + let existing = strTableGet(packSym.tab, result.name) + if existing != nil and existing.info.fileIndex != result.info.fileIndex: + localError(result.info, "module names need to be unique per Nimble package; module clashes with " & existing.info.fileIndex.toFullPath) + # strTableIncl() for error corrections: + discard strTableIncl(packSym.tab, result) proc compileModule*(fileIdx: int32, flags: TSymFlags): PSym = result = getModule(fileIdx) @@ -184,8 +207,8 @@ proc importModule*(s: PSym, fileIdx: int32): PSym {.procvar.} = # this is called by the semantic checking phase result = compileModule(fileIdx, {}) if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx) - if sfSystemModule in result.flags: - localError(result.info, errAttemptToRedefine, result.name.s) + #if sfSystemModule in result.flags: + # localError(result.info, errAttemptToRedefine, result.name.s) # restore the notes for outer module: gNotes = if s.owner.id == gMainPackageId: gMainPackageNotes else: ForeignPackageNotes diff --git a/compiler/msgs.nim b/compiler/msgs.nim index ce9d8c8b6..83914753f 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -502,8 +502,6 @@ type ESuggestDone* = object of Exception const - ForeignPackageNotes*: TNoteKinds = {hintProcessing, warnUnknownMagic, - hintQuitCalled} NotesVerbosity*: array[0..3, TNoteKinds] = [ {low(TNoteKind)..high(TNoteKind)} - {warnShadowIdent, warnUninit, warnProveField, warnProveIndex, @@ -520,7 +518,6 @@ const warnGcUnsafe, hintPath, hintDependency, - hintExecuting, hintCodeBegin, hintCodeEnd, hintSource, hintStackTrace, hintGCStats}, @@ -531,6 +528,8 @@ const InvalidFileIDX* = int32(-1) var + ForeignPackageNotes*: TNoteKinds = {hintProcessing, warnUnknownMagic, + hintQuitCalled, hintExecuting} filenameToIndexTbl = initTable[string, int32]() fileInfos*: seq[TFileInfo] = @[] systemFileIdx*: int32 @@ -621,6 +620,7 @@ var gHintCounter*: int = 0 gWarnCounter*: int = 0 gErrorMax*: int = 1 # stop after gErrorMax errors + gMainPackageNotes*: TNoteKinds = NotesVerbosity[1] proc unknownLineInfo*(): TLineInfo = result.line = int16(-1) diff --git a/compiler/nimfix/pretty.nim b/compiler/nimfix/pretty.nim index 1123afb9e..8ba922927 100644 --- a/compiler/nimfix/pretty.nim +++ b/compiler/nimfix/pretty.nim @@ -66,7 +66,7 @@ proc beautifyName(s: string, k: TSymKind): string = ]: result.add s[i] else: - result.add toUpper(s[i]) + result.add toUpperAscii(s[i]) of skConst, skEnumField: # for 'const' we keep how it's spelt; either upper case or lower case: result.add s[0] @@ -74,7 +74,7 @@ proc beautifyName(s: string, k: TSymKind): string = # as a special rule, don't transform 'L' to 'l' if s.len == 1 and s[0] == 'L': result.add 'L' elif '_' in s: result.add(s[i]) - else: result.add toLower(s[0]) + else: result.add toLowerAscii(s[0]) inc i while i < s.len: if s[i] == '_': @@ -85,9 +85,9 @@ proc beautifyName(s: string, k: TSymKind): string = result.add s[i] else: inc i - result.add toUpper(s[i]) + result.add toUpperAscii(s[i]) elif allUpper: - result.add toLower(s[i]) + result.add toLowerAscii(s[i]) else: result.add s[i] inc i diff --git a/compiler/options.nim b/compiler/options.nim index 7797a4c82..dbd8ca2b9 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -151,6 +151,7 @@ const var gConfigVars* = newStringTable(modeStyleInsensitive) gDllOverrides = newStringTable(modeCaseInsensitive) + gModuleOverrides* = newStringTable(modeStyleInsensitive) gPrefixDir* = "" # Overrides the default prefix dir in getPrefixDir proc. libpath* = "" gProjectName* = "" # holds a name like 'nim' @@ -211,9 +212,7 @@ proc setDefaultLibpath*() = # Special rule to support other tools (nimble) which import the compiler # modules and make use of them. - let realNimPath = # Make sure we expand the symlink - if symlinkExists(findExe("nim")): expandSymlink(findExe("nim")) - else: findExe("nim") + let realNimPath = findExe("nim") # Find out if $nim/../../lib/system.nim exists. let parentNimLibPath = realNimPath.parentDir().parentDir() / "lib" if not fileExists(libpath / "system.nim") and @@ -221,7 +220,7 @@ proc setDefaultLibpath*() = libpath = parentNimLibPath proc canonicalizePath*(path: string): string = - when not FileSystemCaseSensitive: result = path.expandFilename.toLower + when not FileSystemCaseSensitive: result = path.expandFilename.toLowerAscii else: result = path.expandFilename proc shortenDir*(dir: string): string = @@ -240,6 +239,8 @@ proc removeTrailingDirSep*(path: string): string = else: result = path +include packagehandling + proc getNimcacheDir*: string = result = if nimcacheDir.len > 0: nimcacheDir else: gProjectPath.shortenDir / genSubDir @@ -259,54 +260,6 @@ proc pathSubs*(p, config: string): string = if '~' in result: result = result.replace("~", home) -template newPackageCache(): expr = - newStringTable(when FileSystemCaseSensitive: - modeCaseInsensitive - else: - modeCaseSensitive) - -var packageCache = newPackageCache() - -proc resetPackageCache*() = packageCache = newPackageCache() - -iterator myParentDirs(p: string): string = - # XXX os's parentDirs is stupid (multiple yields) and triggers an old bug... - var current = p - while true: - current = current.parentDir - if current.len == 0: break - yield current - -proc getPackageName*(path: string): string = - var parents = 0 - block packageSearch: - for d in myParentDirs(path): - if packageCache.hasKey(d): - #echo "from cache ", d, " |", packageCache[d], "|", path.splitFile.name - return packageCache[d] - inc parents - for file in walkFiles(d / "*.nimble"): - result = file.splitFile.name - break packageSearch - for file in walkFiles(d / "*.babel"): - result = file.splitFile.name - break packageSearch - # we also store if we didn't find anything: - if result.isNil: result = "" - for d in myParentDirs(path): - #echo "set cache ", d, " |", result, "|", parents - packageCache[d] = result - dec parents - if parents <= 0: break - -proc withPackageName*(path: string): string = - let x = path.getPackageName - if x.len == 0: - result = path - else: - let (p, file, ext) = path.splitFile - result = (p / (x & '_' & file)) & ext - proc toGeneratedFile*(path, ext: string): string = ## converts "/home/a/mymodule.nim", "rod" to "/home/a/nimcache/mymodule.rod" var (head, tail) = splitPath(path) @@ -374,17 +327,25 @@ proc rawFindFile2(f: string): string = it = PStrEntry(it.next) result = "" +template patchModule() {.dirty.} = + if result.len > 0 and gModuleOverrides.len > 0: + let key = getPackageName(result) & "_" & splitFile(result).name + if gModuleOverrides.hasKey(key): + let ov = gModuleOverrides[key] + if ov.len > 0: result = ov + proc findFile*(f: string): string {.procvar.} = if f.isAbsolute: result = if f.existsFile: f else: "" else: result = f.rawFindFile if result.len == 0: - result = f.toLower.rawFindFile + result = f.toLowerAscii.rawFindFile if result.len == 0: result = f.rawFindFile2 if result.len == 0: - result = f.toLower.rawFindFile2 + result = f.toLowerAscii.rawFindFile2 + patchModule() proc findModule*(modulename, currentModule: string): string = # returns path to module @@ -403,6 +364,7 @@ proc findModule*(modulename, currentModule: string): string = result = currentPath / m if not existsFile(result): result = findFile(m) + patchModule() proc libCandidates*(s: string, dest: var seq[string]) = var le = strutils.find(s, '(') diff --git a/compiler/packagehandling.nim b/compiler/packagehandling.nim new file mode 100644 index 000000000..c42a4d763 --- /dev/null +++ b/compiler/packagehandling.nim @@ -0,0 +1,56 @@ +# +# +# The Nim Compiler +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +iterator myParentDirs(p: string): string = + # XXX os's parentDirs is stupid (multiple yields) and triggers an old bug... + var current = p + while true: + current = current.parentDir + if current.len == 0: break + yield current + +template newPackageCache(): expr = + newStringTable(when FileSystemCaseSensitive: + modeCaseInsensitive + else: + modeCaseSensitive) + +var packageCache = newPackageCache() + +proc resetPackageCache*() = packageCache = newPackageCache() + +proc getPackageName*(path: string): string = + var parents = 0 + block packageSearch: + for d in myParentDirs(path): + if packageCache.hasKey(d): + #echo "from cache ", d, " |", packageCache[d], "|", path.splitFile.name + return packageCache[d] + inc parents + for file in walkFiles(d / "*.nimble"): + result = file.splitFile.name + break packageSearch + for file in walkFiles(d / "*.babel"): + result = file.splitFile.name + break packageSearch + # we also store if we didn't find anything: + if result.isNil: result = "" + for d in myParentDirs(path): + #echo "set cache ", d, " |", result, "|", parents + packageCache[d] = result + dec parents + if parents <= 0: break + +proc withPackageName*(path: string): string = + let x = path.getPackageName + if x.len == 0: + result = path + else: + let (p, file, ext) = path.splitFile + result = (p / (x & '_' & file)) & ext diff --git a/compiler/parser.nim b/compiler/parser.nim index 6132216e1..8b20b56db 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -686,11 +686,19 @@ proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode = #| | '{' optInd indexExprList optPar '}' #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax result = r + + template somePar() = + if p.tok.strongSpaceA > 0: + if p.strongSpaces: + break + else: + parMessage(p, warnDeprecated, + "a [b] will be parsed as command syntax; spacing") while p.tok.indent < 0 or (p.tok.tokType == tkDot and p.tok.indent >= baseIndent): case p.tok.tokType of tkParLe: - if p.strongSpaces and p.tok.strongSpaceA > 0: break + somePar() result = namedParams(p, result, nkCall, tkParRi) if result.len > 1 and result.sons[1].kind == nkExprColonExpr: result.kind = nkObjConstr @@ -705,10 +713,10 @@ proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode = result = dotExpr(p, result) result = parseGStrLit(p, result) of tkBracketLe: - if p.strongSpaces and p.tok.strongSpaceA > 0: break + somePar() result = namedParams(p, result, nkBracketExpr, tkBracketRi) of tkCurlyLe: - if p.strongSpaces and p.tok.strongSpaceA > 0: break + somePar() result = namedParams(p, result, nkCurlyExpr, tkCurlyRi) of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast, tkAddr, tkType: if p.inPragma == 0: diff --git a/compiler/passaux.nim b/compiler/passaux.nim index 9b9fdca4e..d4361a671 100644 --- a/compiler/passaux.nim +++ b/compiler/passaux.nim @@ -24,7 +24,7 @@ proc verboseProcess(context: PPassContext, n: PNode): PNode = # system.nim deactivates all hints, for verbosity:3 we want the processing # messages nonetheless, so we activate them again unconditionally: incl(msgs.gNotes, hintProcessing) - message(n.info, hintProcessing, $idgen.gBackendId) + message(n.info, hintProcessing, $idgen.gFrontendId) const verbosePass* = makePass(open = verboseOpen, process = verboseProcess) diff --git a/compiler/patterns.nim b/compiler/patterns.nim index 2336e44e7..09b8d3305 100644 --- a/compiler/patterns.nim +++ b/compiler/patterns.nim @@ -75,7 +75,7 @@ proc checkTypes(c: PPatternContext, p: PSym, n: PNode): bool = result = matchNodeKinds(p.constraint, n) if not result: return if isNil(n.typ): - result = p.typ.kind in {tyEmpty, tyStmt} + result = p.typ.kind in {tyVoid, tyStmt} else: result = sigmatch.argtypeMatches(c.c, p.typ, n.typ) diff --git a/compiler/platform.nim b/compiler/platform.nim index dc414bfeb..36ec10477 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -159,7 +159,7 @@ type # alias conditionals to condsyms (end of module). cpuNone, cpuI386, cpuM68k, cpuAlpha, cpuPowerpc, cpuPowerpc64, cpuPowerpc64el, cpuSparc, cpuVm, cpuIa64, cpuAmd64, cpuMips, cpuMipsel, - cpuArm, cpuArm64, cpuJS, cpuNimrodVM, cpuAVR, cpuMSP430 + cpuArm, cpuArm64, cpuJS, cpuNimrodVM, cpuAVR, cpuMSP430, cpuSparc64 type TEndian* = enum @@ -187,7 +187,8 @@ const (name: "js", intSize: 32, endian: bigEndian,floatSize: 64,bit: 32), (name: "nimrodvm", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32), (name: "avr", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16), - (name: "msp430", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16)] + (name: "msp430", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16), + (name: "sparc64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64)] var targetCPU*, hostCPU*: TSystemCPU diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index dc618d9aa..d7d1ed838 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -63,11 +63,12 @@ const wImportCpp, wImportObjC, wError, wNoInit, wCompileTime, wGlobal, wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims} constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl, - wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject, wExportNims} + wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject, wExportNims, + wIntDefine, wStrDefine} letPragmas* = varPragmas procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNosideeffect, wThread, wRaises, wLocks, wTags, wGcSafe} - allRoutinePragmas* = procPragmas + iteratorPragmas + lambdaPragmas + allRoutinePragmas* = methodPragmas + iteratorPragmas + lambdaPragmas proc pragma*(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords) # implementation @@ -898,6 +899,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, of wBase: noVal(it) sym.flags.incl sfBase + of wIntDefine: + sym.magic = mIntDefine + of wStrDefine: + sym.magic = mStrDefine else: invalidPragma(it) else: invalidPragma(it) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index e1db327f4..0e733d643 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -704,7 +704,10 @@ proc gcase(g: var TSrcGen, n: PNode) = proc gproc(g: var TSrcGen, n: PNode) = var c: TContext if n.sons[namePos].kind == nkSym: - put(g, tkSymbol, renderDefinitionName(n.sons[namePos].sym)) + let s = n.sons[namePos].sym + put(g, tkSymbol, renderDefinitionName(s)) + if sfGenSym in s.flags: + put(g, tkIntLit, $s.id) else: gsub(g, n.sons[namePos]) @@ -798,7 +801,8 @@ proc gident(g: var TSrcGen, n: PNode) = else: t = tkOpr put(g, t, s) - if n.kind == nkSym and renderIds in g.flags: put(g, tkIntLit, $n.sym.id) + if n.kind == nkSym and (renderIds in g.flags or sfGenSym in n.sym.flags): + put(g, tkIntLit, $n.sym.id) proc doParamsAux(g: var TSrcGen, params: PNode) = if params.len > 1: diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim index 91f93d279..945c94bc6 100644 --- a/compiler/rodutils.nim +++ b/compiler/rodutils.nim @@ -21,7 +21,7 @@ proc toStrMaxPrecision*(f: BiggestFloat): string = if f > 0.0: result = "INF" else: result = "-INF" else: - var buf: array [0..80, char] + var buf: array[0..80, char] c_sprintf(buf, "%#.16e", f) result = $buf diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index d04fd5231..833444788 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -13,10 +13,10 @@ import ast, modules, passes, passaux, condsyms, options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs, - os, times, osproc + os, times, osproc, wordrecg, strtabs # we support 'cmpIgnoreStyle' natively for efficiency: -from strutils import cmpIgnoreStyle +from strutils import cmpIgnoreStyle, contains proc listDirs(a: VmArgs, filter: set[PathComponent]) = let dir = getString(a, 0) @@ -116,7 +116,20 @@ proc setupVM*(module: PSym; scriptName: string): PEvalContext = setResult(a, options.command) cbconf switch: processSwitch(a.getString 0, a.getString 1, passPP, unknownLineInfo()) - + cbconf hintImpl: + processSpecificNote(a.getString 0, wHint, passPP, unknownLineInfo(), + a.getString 1) + cbconf warningImpl: + processSpecificNote(a.getString 0, wWarning, passPP, unknownLineInfo(), + a.getString 1) + cbconf patchFile: + let key = a.getString(0) & "_" & a.getString(1) + var val = a.getString(2).addFileExt(NimExt) + if {'$', '~'} in val: + val = pathSubs(val, vthisDir) + elif not isAbsolute(val): + val = vthisDir / val + gModuleOverrides[key] = val proc runNimScript*(scriptName: string; freshDefines=true) = passes.gIncludeFile = includeModule diff --git a/compiler/sem.nim b/compiler/sem.nim index a8ec2229f..7db4ae47e 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -59,8 +59,7 @@ template semIdeForTemplateOrGeneric(c: PContext; n: PNode; # templates perform some quick check whether the cursor is actually in # the generic or template. when defined(nimsuggest): - assert gCmd == cmdIdeTools - if requiresCheck: + if gCmd == cmdIdeTools and requiresCheck: #if optIdeDebug in gGlobalOptions: # echo "passing to safeSemExpr: ", renderTree(n) discard safeSemExpr(c, n) @@ -165,7 +164,7 @@ proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym = result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info) proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym = - proc `$`(kind: TSymKind): string = substr(system.`$`(kind), 2).toLower + proc `$`(kind: TSymKind): string = substr(system.`$`(kind), 2).toLowerAscii # like newSymS, but considers gensym'ed symbols if n.kind == nkSym: @@ -307,6 +306,13 @@ proc semConstExpr(c: PContext, n: PNode): PNode = include hlo, seminst, semcall +when false: + # hopefully not required: + proc resetSemFlag(n: PNode) = + excl n.flags, nfSem + for i in 0 ..< n.safeLen: + resetSemFlag(n[i]) + proc semAfterMacroCall(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = ## Semantically check the output of a macro. @@ -320,6 +326,8 @@ proc semAfterMacroCall(c: PContext, n: PNode, s: PSym, c.friendModules.add(s.owner.getModule) result = n + excl(n.flags, nfSem) + #resetSemFlag n if s.typ.sons[0] == nil: result = semStmt(c, result) else: @@ -409,9 +417,6 @@ proc myOpen(module: PSym): PPassContext = c.importTable.addSym(module) # a module knows itself if sfSystemModule in module.flags: magicsys.systemModule = module # set global variable! - else: - c.importTable.addSym magicsys.systemModule # import the "System" identifier - importAllSymbols(c, magicsys.systemModule) c.topLevelScope = openScope(c) # don't be verbose unless the module belongs to the main package: if module.owner.id == gMainPackageId: @@ -425,7 +430,29 @@ proc myOpenCached(module: PSym, rd: PRodReader): PPassContext = result = myOpen(module) for m in items(rd.methods): methodDef(m, true) +proc isImportSystemStmt(n: PNode): bool = + if magicsys.systemModule == nil: return false + case n.kind + of nkImportStmt: + for x in n: + let f = checkModuleName(x) + if f == magicsys.systemModule.info.fileIndex: + return true + of nkImportExceptStmt, nkFromStmt: + let f = checkModuleName(n[0]) + if f == magicsys.systemModule.info.fileIndex: + return true + else: discard + proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = + if c.topStmts == 0 and not isImportSystemStmt(n): + if sfSystemModule notin c.module.flags and + n.kind notin {nkEmpty, nkCommentStmt}: + c.importTable.addSym magicsys.systemModule # import the "System" identifier + importAllSymbols(c, magicsys.systemModule) + inc c.topStmts + else: + inc c.topStmts if sfNoForward in c.module.flags: result = semAllTypeSections(c, n) else: diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index a1e209263..9702128ba 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -183,7 +183,7 @@ proc newSeqCall(c: PContext; x, y: PNode): PNode = proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = case t.kind - of tyNone, tyEmpty: discard + of tyNone, tyEmpty, tyVoid: discard of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString, tyPtr, tyString, tyRef: defaultOp(c, t, body, x, y) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 65aa5fd58..87a34ca4a 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -69,7 +69,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, # gDebug = true matches(c, n, orig, z) if errors != nil: - errors.safeAdd(sym) + errors.safeAdd((sym, int z.mutabilityProblem)) if z.errors != nil: for err in z.errors: errors.add(err) @@ -111,7 +111,7 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = let proto = describeArgs(c, n, 1, preferName) var prefer = preferName - for err in errors: + for err, mut in items(errors): var errProto = "" let n = err.typ.n for i in countup(1, n.len - 1): @@ -123,18 +123,21 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = if errProto == proto: prefer = preferModuleInfo break + # now use the information stored in 'prefer' to produce a nice error message: var result = msgKindToString(errTypeMismatch) add(result, describeArgs(c, n, 1, prefer)) add(result, ')') var candidates = "" - for err in errors: + for err, mut in items(errors): if err.kind in routineKinds and err.ast != nil: add(candidates, renderTree(err.ast, {renderNoBody, renderNoComments,renderNoPragmas})) else: add(candidates, err.getProcHeader(prefer)) add(candidates, "\n") + if mut != 0 and mut < n.len: + add(candidates, "for a 'var' type a variable needs to be passed, but '" & renderTree(n[mut]) & "' is immutable\n") if candidates != "": add(result, "\n" & msgKindToString(errButExpected) & "\n" & candidates) if c.compilesContextId > 0 and optReportConceptFailures in gGlobalOptions: @@ -142,6 +145,20 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = else: localError(n.info, errGenerated, result) +proc bracketNotFoundError(c: PContext; n: PNode) = + var errors: CandidateErrors = @[] + var o: TOverloadIter + let headSymbol = n[0] + var symx = initOverloadIter(o, c, headSymbol) + while symx != nil: + if symx.kind in routineKinds: + errors.add((symx, 0)) + symx = nextOverloadIter(o, c, headSymbol) + if errors.len == 0: + localError(n.info, "could not resolve: " & $n) + else: + notFoundError(c, n, errors) + proc resolveOverloads(c: PContext, n, orig: PNode, filter: TSymKinds; errors: var CandidateErrors): TCandidate = @@ -158,8 +175,6 @@ proc resolveOverloads(c: PContext, n, orig: PNode, template pickBest(headSymbol) = pickBestCandidate(c, headSymbol, n, orig, initialBinding, filter, result, alt, errors) - - pickBest(f) let overloadsState = result.state @@ -230,7 +245,6 @@ proc resolveOverloads(c: PContext, n, orig: PNode, #notFoundError(c, n, errors) return - if alt.state == csMatch and cmpCandidates(result, alt) == 0 and not sameMethodDispatcher(result.calleeSym, alt.calleeSym): internalAssert result.state == csMatch @@ -299,8 +313,7 @@ proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode = var finalCallee = x.calleeSym markUsed(n.sons[0].info, finalCallee) styleCheckUse(n.sons[0].info, finalCallee) - if finalCallee.ast == nil: - internalError(n.info, "calleeSym.ast is nil") # XXX: remove this check! + assert finalCallee.ast != nil if x.hasFauxMatch: result = x.call result.sons[0] = newSymNode(finalCallee, result.sons[0].info) @@ -319,12 +332,12 @@ proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode = # are added as normal params. for s in instantiateGenericParamList(c, gp, x.bindings): case s.kind - of skConst: - x.call.add s.ast - of skType: - x.call.add newSymNode(s, n.info) - else: - internalAssert false + of skConst: + x.call.add s.ast + of skType: + x.call.add newSymNode(s, n.info) + else: + internalAssert false result = x.call instGenericConvertersSons(c, result, x) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 91a1ebf86..30b6e261d 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -99,11 +99,12 @@ type unknownIdents*: IntSet # ids of all unknown identifiers to prevent # naming it multiple times generics*: seq[TInstantiationPair] # pending list of instantiated generics to compile + topStmts*: int # counts the number of encountered top level statements lastGenericIdx*: int # used for the generics stack hloLoopDetector*: int # used to prevent endless loops in the HLO inParallelStmt*: int instTypeBoundOp*: proc (c: PContext; dc: PSym; t: PType; info: TLineInfo; - op: TTypeAttachedOp): PSym {.nimcall.} + op: TTypeAttachedOp; col: int): PSym {.nimcall.} selfName*: PIdent signatures*: TStrTable diff --git a/compiler/semdestruct.nim b/compiler/semdestruct.nim index 1261dd460..18a1c262f 100644 --- a/compiler/semdestruct.nim +++ b/compiler/semdestruct.nim @@ -139,6 +139,7 @@ proc instantiateDestructor(c: PContext, typ: PType): PType = t = t.skipTypes({tyGenericInst}) case t.kind of tySequence, tyArray, tyArrayConstr, tyOpenArray, tyVarargs: + t.destructor = analyzingDestructor if instantiateDestructor(c, t.sons[0]) != nil: t.destructor = getCompilerProc"nimDestroyRange" return t diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 57fb3ceed..9f0696252 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -32,8 +32,7 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = # XXX tyGenericInst here? if result.typ.kind == tyVar: result = newDeref(result) elif {efWantStmt, efAllowStmt} * flags != {}: - result.typ = newTypeS(tyEmpty, c) - result.typ.flags.incl tfVoid + result.typ = newTypeS(tyVoid, c) else: localError(n.info, errExprXHasNoType, renderTree(result, {renderNoComments})) @@ -51,7 +50,6 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = renderTree(result, {renderNoComments})) result.typ = errorType(c) else: - # XXX tyGenericInst here? if efNoProcvarCheck notin flags: semProcvarCheck(c, result) if result.typ.kind == tyVar: result = newDeref(result) semDestructorCheck(c, result, flags) @@ -199,6 +197,8 @@ proc semConv(c: PContext, n: PNode): PNode = # separate proc from fitNode? if op.kind == nkSym and op.sym.isGenericRoutine: result.sons[1] = fitNode(c, result.typ, result.sons[1]) + elif op.kind == nkPar and targetType.kind == tyTuple: + op = fitNode(c, targetType, op) of convNotNeedeed: message(n.info, hintConvFromXtoItselfNotNeeded, result.typ.typeToString) of convNotLegal: @@ -601,6 +601,9 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode = result = n if n.kind notin nkCallKinds or n.sons[0].kind != nkSym: return var callee = n.sons[0].sym + # workaround for bug #537 (overly aggressive inlining leading to + # wrong NimNode semantics): + if n.typ != nil and tfTriggersCompileTime in n.typ.flags: return # constant folding that is necessary for correctness of semantic pass: if callee.magic != mNone and callee.magic in ctfeWhitelist and n.typ != nil: @@ -1063,8 +1066,11 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = result = newSymNode(s, n.info) result.typ = makeTypeDesc(c, s.typ) of skField: - if c.p != nil and c.p.selfSym != nil: - var ty = skipTypes(c.p.selfSym.typ, {tyGenericInst, tyVar, tyPtr, tyRef}) + var p = c.p + while p != nil and p.selfSym == nil: + p = p.next + if p != nil and p.selfSym != nil: + var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyPtr, tyRef}) while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct}) var check: PNode = nil if ty.kind == tyObject: @@ -1077,7 +1083,7 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = markUsed(n.info, f) styleCheckUse(n.info, f) result = newNodeIT(nkDotExpr, n.info, f.typ) - result.add makeDeref(newSymNode(c.p.selfSym)) + result.add makeDeref(newSymNode(p.selfSym)) result.add newSymNode(f) # we now have the correct field if check != nil: check.sons[0] = result @@ -1385,15 +1391,16 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = # --> `[]=`(a, i, x) let oldBracketExpr = c.p.bracketExpr a = semSubscript(c, a, {efLValue}) - if a == nil and mode != noOverloadedSubscript: + if a == nil: result = buildOverloadedSubscripts(n.sons[0], getIdent"[]=") add(result, n[1]) - result = semExprNoType(c, result) - c.p.bracketExpr = oldBracketExpr - return result - elif a == nil: - localError(n.info, "could not resolve: " & $n[0]) - return n + if mode == noOverloadedSubscript: + bracketNotFoundError(c, result) + return n + else: + result = semExprNoType(c, result) + c.p.bracketExpr = oldBracketExpr + return result c.p.bracketExpr = oldBracketExpr of nkCurlyExpr: # a{i} = x --> `{}=`(a, i, x) @@ -1629,8 +1636,8 @@ proc newAnonSym(kind: TSymKind, info: TLineInfo, result.flags = {sfGenSym} proc semExpandToAst(c: PContext, n: PNode): PNode = - var macroCall = n[1] - var expandedSym = expectMacroOrTemplateCall(c, macroCall) + let macroCall = n[1] + let expandedSym = expectMacroOrTemplateCall(c, macroCall) if expandedSym.kind == skError: return n macroCall.sons[0] = newSymNode(expandedSym, macroCall.info) @@ -1638,11 +1645,12 @@ proc semExpandToAst(c: PContext, n: PNode): PNode = styleCheckUse(n.info, expandedSym) for i in countup(1, macroCall.len-1): + #if macroCall.sons[0].typ.sons[i].kind != tyExpr: macroCall.sons[i] = semExprWithType(c, macroCall[i], {}) # Preserve the magic symbol in order to be handled in evals.nim internalAssert n.sons[0].sym.magic == mExpandToAst - #n.typ = getSysSym("PNimrodNode").typ # expandedSym.getReturnType + #n.typ = getSysSym("NimNode").typ # expandedSym.getReturnType n.typ = if getCompilerProc("NimNode") != nil: sysTypeFromName"NimNode" else: sysTypeFromName"PNimrodNode" result = n diff --git a/compiler/semfold.nim b/compiler/semfold.nim index feea6ce34..e9ee50d74 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -286,13 +286,14 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode = of mNot: result = newIntNodeT(1 - getInt(a), n) of mCard: result = newIntNodeT(nimsets.cardSet(a), n) of mBitnotI: result = newIntNodeT(not getInt(a), n) - of mLengthStr, mXLenStr: - if a.kind == nkNilLit: result = newIntNodeT(0, n) - else: result = newIntNodeT(len(getStr(a)), n) of mLengthArray: result = newIntNodeT(lengthOrd(a.typ), n) - of mLengthSeq, mLengthOpenArray, mXLenSeq: - if a.kind == nkNilLit: result = newIntNodeT(0, n) - else: result = newIntNodeT(sonsLen(a), n) # BUGFIX + of mLengthSeq, mLengthOpenArray, mXLenSeq, mLengthStr, mXLenStr: + if a.kind == nkNilLit: + result = newIntNodeT(0, n) + elif a.kind in {nkStrLit..nkTripleStrLit}: + result = newIntNodeT(len a.strVal, n) + else: + result = newIntNodeT(sonsLen(a), n) # BUGFIX of mUnaryPlusI, mUnaryPlusF64: result = a # throw `+` away of mToFloat, mToBiggestFloat: result = newFloatNodeT(toFloat(int(getInt(a))), n) @@ -634,12 +635,18 @@ proc getConstExpr(m: PSym, n: PNode): PNode = of mCompileDate: result = newStrNodeT(times.getDateStr(), n) of mCompileTime: result = newStrNodeT(times.getClockStr(), n) of mCpuEndian: result = newIntNodeT(ord(CPU[targetCPU].endian), n) - of mHostOS: result = newStrNodeT(toLower(platform.OS[targetOS].name), n) - of mHostCPU: result = newStrNodeT(platform.CPU[targetCPU].name.toLower, n) + of mHostOS: result = newStrNodeT(toLowerAscii(platform.OS[targetOS].name), n) + of mHostCPU: result = newStrNodeT(platform.CPU[targetCPU].name.toLowerAscii, n) of mAppType: result = getAppType(n) of mNaN: result = newFloatNodeT(NaN, n) of mInf: result = newFloatNodeT(Inf, n) of mNegInf: result = newFloatNodeT(NegInf, n) + of mIntDefine: + if isDefined(s.name): + result = newIntNodeT(lookupSymbol(s.name).parseInt, n) + of mStrDefine: + if isDefined(s.name): + result = newStrNodeT(lookupSymbol(s.name), n) else: if sfFakeConst notin s.flags: result = copyTree(s.ast) of {skProc, skMethod}: diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index 0ba76ccd3..b78679411 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -426,7 +426,12 @@ proc semGenericStmt(c: PContext, n: PNode, n.sons[paramsPos] = semGenericStmt(c, n.sons[paramsPos], flags, ctx) n.sons[pragmasPos] = semGenericStmt(c, n.sons[pragmasPos], flags, ctx) var body: PNode - if n.sons[namePos].kind == nkSym: body = n.sons[namePos].sym.getBody + if n.sons[namePos].kind == nkSym: + let s = n.sons[namePos].sym + if sfGenSym in s.flags and s.ast == nil: + body = n.sons[bodyPos] + else: + body = s.getBody else: body = n.sons[bodyPos] n.sons[bodyPos] = semGenericStmtScope(c, body, flags, ctx) closeScope(c) diff --git a/compiler/seminst.nim b/compiler/seminst.nim index e4ac56cd6..460db4f7c 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -205,7 +205,7 @@ proc instantiateProcType(c: PContext, pt: TIdTable, # The solution would be to move this logic into semtypinst, but # at this point semtypinst have to become part of sem, because it # will need to use openScope, addDecl, etc. - addDecl(c, prc) + #addDecl(c, prc) pushInfoContext(info) var cl = initTypeVars(c, pt, info, nil) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 1945d87eb..451745884 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -42,7 +42,10 @@ proc semArrGet(c: PContext; n: PNode; flags: TExprFlags): PNode = result = semSubscript(c, result, flags) c.p.bracketExpr = oldBracketExpr if result.isNil: - localError(n.info, "could not resolve: " & $n) + let x = copyTree(n) + x.sons[0] = newIdentNode(getIdent"[]", n.info) + bracketNotFoundError(c, x) + #localError(n.info, "could not resolve: " & $n) result = n proc semArrPut(c: PContext; n: PNode; flags: TExprFlags): PNode = diff --git a/compiler/semparallel.nim b/compiler/semparallel.nim index b04ba4657..a4ec14250 100644 --- a/compiler/semparallel.nim +++ b/compiler/semparallel.nim @@ -74,8 +74,6 @@ type currentSpawnId: int inLoop: int -let opSlice = createMagic("slice", mSlice) - proc initAnalysisCtx(): AnalysisCtx = result.locals = @[] result.slices = @[] @@ -399,7 +397,9 @@ proc transformSlices(n: PNode): PNode = let op = n[0].sym if op.name.s == "[]" and op.fromSystem: result = copyNode(n) - result.add opSlice.newSymNode + let opSlice = newSymNode(createMagic("slice", mSlice)) + opSlice.typ = getSysType(tyInt) + result.add opSlice result.add n[1] let slice = n[2].skipStmtList result.add slice[1] diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 7155f9c7a..1f076aa31 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -138,7 +138,7 @@ proc fixNilType(n: PNode) = proc discardCheck(c: PContext, result: PNode) = if c.inTypeClass > 0: return - if result.typ != nil and result.typ.kind notin {tyStmt, tyEmpty}: + if result.typ != nil and result.typ.kind notin {tyStmt, tyVoid}: if result.kind == nkNilLit: result.typ = nil message(result.info, warnNilStatement) @@ -411,6 +411,13 @@ proc semUsing(c: PContext; n: PNode): PNode = if a.sons[length-1].kind != nkEmpty: localError(a.info, "'using' sections cannot contain assignments") +proc hasEmpty(typ: PType): bool = + if typ.kind in {tySequence, tyArray, tySet}: + result = typ.lastSon.kind == tyEmpty + elif typ.kind == tyTuple: + for s in typ.sons: + result = result or hasEmpty(s) + proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = var b: PNode result = copyNode(n) @@ -445,10 +452,9 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = #changeType(def.skipConv, typ, check=true) else: typ = skipIntLit(def.typ) - if typ.kind in {tySequence, tyArray, tySet} and - typ.lastSon.kind == tyEmpty: + if hasEmpty(typ): localError(def.info, errCannotInferTypeOfTheLiteral, - ($typ.kind).substr(2).toLower) + ($typ.kind).substr(2).toLowerAscii) else: def = ast.emptyNode if symkind == skLet: localError(a.info, errLetNeedsInit) @@ -711,7 +717,7 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) = a.sons[1] = s.typ.n s.typ.size = -1 # could not be computed properly # we fill it out later. For magic generics like 'seq', it won't be filled - # so we use tyEmpty instead of nil to not crash for strange conversions + # so we use tyNone instead of nil to not crash for strange conversions # like: mydata.seq rawAddSon(s.typ, newTypeS(tyNone, c)) s.ast = a @@ -1246,6 +1252,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, pushOwner(s) s.options = gOptions if sfOverriden in s.flags or s.name.s[0] == '=': semOverride(c, s, n) + if s.name.s[0] in {'.', '('}: + if s.name.s in [".", ".()", ".=", "()"] and not experimentalMode(c): + message(n.info, warnDeprecated, "overloaded '.' and '()' operators are now .experimental; " & s.name.s) if n.sons[bodyPos].kind != nkEmpty: # for DLL generation it is annoying to check for sfImportc! if sfBorrow in s.flags: @@ -1343,7 +1352,19 @@ proc semMethod(c: PContext, n: PNode): PNode = # macros can transform methods to nothing: if namePos >= result.safeLen: return result var s = result.sons[namePos].sym - if not isGenericRoutine(s): + if isGenericRoutine(s): + let tt = s.typ + var foundObj = false + for col in countup(0, sonsLen(tt)-1): + let t = tt.sons[col] + if t != nil and t.kind == tyGenericInvocation: + var x = skipTypes(t.sons[0], {tyVar, tyPtr, tyRef, tyGenericInst, tyGenericInvocation, tyGenericBody}) + if x.kind == tyObject: + foundObj = true + x.methods.safeAdd((col,s)) + if not foundObj: + message(n.info, warnDeprecated, "generic method not attachable to object type") + else: # why check for the body? bug #2400 has none. Checking for sfForward makes # no sense either. # and result.sons[bodyPos].kind != nkEmpty: diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 13b283fe5..26ce28852 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -130,7 +130,7 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType = if n.len < 1: result = newConstraint(c, kind) else: - let isCall = ord(n.kind in nkCallKinds) + let isCall = ord(n.kind in nkCallKinds+{nkBracketExpr}) let n = if n[0].kind == nkBracket: n[0] else: n checkMinSonsLen(n, 1) var base = semTypeNode(c, n.lastSon, nil) @@ -370,7 +370,7 @@ proc semTuple(c: PContext, n: PNode, prev: PType): PType = result.n = newNodeI(nkRecList, n.info) var check = initIntSet() var counter = 0 - for i in countup(0, sonsLen(n) - 1): + for i in countup(ord(n.kind == nkBracketExpr), sonsLen(n) - 1): var a = n.sons[i] if (a.kind != nkIdentDefs): illFormedAst(a) checkMinSonsLen(a, 3) @@ -945,7 +945,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, if isType: localError(a.info, "':' expected") if kind in {skTemplate, skMacro}: typ = newTypeS(tyExpr, c) - elif skipTypes(typ, {tyGenericInst}).kind == tyEmpty: + elif skipTypes(typ, {tyGenericInst}).kind == tyVoid: continue for j in countup(0, length-3): var arg = newSymG(skParam, a.sons[j], c) @@ -977,7 +977,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, if r != nil: # turn explicit 'void' return type into 'nil' because the rest of the # compiler only checks for 'nil': - if skipTypes(r, {tyGenericInst}).kind != tyEmpty: + if skipTypes(r, {tyGenericInst}).kind != tyVoid: # 'auto' as a return type does not imply a generic: if r.kind == tyAnything: # 'p(): auto' and 'p(): expr' are equivalent, but the rest of the @@ -1038,7 +1038,7 @@ proc semGenericParamInInvocation(c: PContext, n: PNode): PType = proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType = if s.typ == nil: localError(n.info, "cannot instantiate the '$1' $2" % - [s.name.s, ($s.kind).substr(2).toLower]) + [s.name.s, ($s.kind).substr(2).toLowerAscii]) return newOrPrevType(tyError, prev, c) var t = s.typ @@ -1248,6 +1248,19 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result = copyType(result, getCurrOwner(), false) for i in countup(1, n.len - 1): result.rawAddSon(semTypeNode(c, n.sons[i], nil)) + of mDistinct: + result = newOrPrevType(tyDistinct, prev, c) + addSonSkipIntLit(result, semTypeNode(c, n[1], nil)) + of mVar: + result = newOrPrevType(tyVar, prev, c) + var base = semTypeNode(c, n.sons[1], nil) + if base.kind == tyVar: + localError(n.info, errVarVarTypeNotAllowed) + base = base.sons[0] + addSonSkipIntLit(result, base) + of mRef: result = semAnyRef(c, n, tyRef, prev) + of mPtr: result = semAnyRef(c, n, tyPtr, prev) + of mTuple: result = semTuple(c, n, prev) else: result = semGeneric(c, n, s, prev) of nkDotExpr: var typeExpr = semExpr(c, n) @@ -1377,10 +1390,7 @@ proc processMagicType(c: PContext, m: PSym) = setMagicType(m, tyTypeDesc, 0) rawAddSon(m.typ, newTypeS(tyNone, c)) of mVoidType: - setMagicType(m, tyEmpty, 0) - # for historical reasons we conflate 'void' with 'empty' so that '@[]' - # has the type 'seq[void]'. - m.typ.flags.incl tfVoid + setMagicType(m, tyVoid, 0) of mArray: setMagicType(m, tyArray, 0) of mOpenArray: diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 7ff33f918..076679bbf 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -162,7 +162,7 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode = discard of nkSym: result.sym = replaceTypeVarsS(cl, n.sym) - if result.sym.typ.kind == tyEmpty: + if result.sym.typ.kind == tyVoid: # don't add the 'void' field result = newNode(nkRecList, n.info) of nkRecWhen: @@ -289,7 +289,8 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = # but we already raised an error! rawAddSon(result, header.sons[i]) - var newbody = replaceTypeVarsT(cl, lastSon(body)) + let bbody = lastSon body + var newbody = replaceTypeVarsT(cl, bbody) cl.skipTypedesc = oldSkipTypedesc newbody.flags = newbody.flags + (t.flags + body.flags - tfInstClearedFlags) result.flags = result.flags + newbody.flags - tfInstClearedFlags @@ -306,25 +307,31 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = # 'deepCopy' needs to be instantiated for # generics *when the type is constructed*: newbody.deepCopy = cl.c.instTypeBoundOp(cl.c, dc, result, cl.info, - attachedDeepCopy) + attachedDeepCopy, 1) let asgn = newbody.assignment if asgn != nil and sfFromGeneric notin asgn.flags: # '=' needs to be instantiated for generics when the type is constructed: newbody.assignment = cl.c.instTypeBoundOp(cl.c, asgn, result, cl.info, - attachedAsgn) + attachedAsgn, 1) + let methods = skipTypes(bbody, abstractPtrs).methods + for col, meth in items(methods): + # we instantiate the known methods belonging to that type, this causes + # them to be registered and that's enough, so we 'discard' the result. + discard cl.c.instTypeBoundOp(cl.c, meth, result, cl.info, + attachedAsgn, col) proc eraseVoidParams*(t: PType) = # transform '(): void' into '()' because old parts of the compiler really # don't deal with '(): void': - if t.sons[0] != nil and t.sons[0].kind == tyEmpty: + if t.sons[0] != nil and t.sons[0].kind == tyVoid: t.sons[0] = nil for i in 1 .. <t.sonsLen: # don't touch any memory unless necessary - if t.sons[i].kind == tyEmpty: + if t.sons[i].kind == tyVoid: var pos = i for j in i+1 .. <t.sonsLen: - if t.sons[j].kind != tyEmpty: + if t.sons[j].kind != tyVoid: t.sons[pos] = t.sons[j] t.n.sons[pos] = t.n.sons[j] inc pos diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index df27e3c1d..9a8f865a6 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -22,7 +22,7 @@ type TCandidateState* = enum csEmpty, csMatch, csNoMatch - CandidateErrors* = seq[PSym] + CandidateErrors* = seq[(PSym,int)] TCandidate* = object c*: PContext exactMatches*: int # also misused to prefer iters over procs @@ -49,6 +49,7 @@ type # a distrinct type typedescMatched*: bool isNoCall*: bool # misused for generic type instantiations C[T] + mutabilityProblem*: uint8 # tyVar mismatch inheritancePenalty: int # to prefer closest father object type errors*: CandidateErrors # additional clarifications to be displayed to the # user if overload resolution fails @@ -363,6 +364,46 @@ proc isObjectSubtype(a, f: PType): int = if t != nil: result = depth +type + SkippedPtr = enum skippedNone, skippedRef, skippedPtr + +proc skipToGenericBody(t: PType; skipped: var SkippedPtr): PType = + var r = t + # we're allowed to skip one level of ptr/ref: + var ptrs = 0 + while r != nil: + case r.kind + of tyGenericInst, tyGenericInvocation: + result = r.sons[0] + break + of tyRef: + inc ptrs + skipped = skippedRef + r = r.lastSon + of tyPtr: + inc ptrs + skipped = skippedPtr + r = r.lastSon + of tyGenericBody, tyObject: + r = r.lastSon + else: + break + if ptrs > 1: result = nil + +proc isGenericSubtype(a, f: PType, d: var int): bool = + assert f.kind in {tyGenericInst, tyGenericInvocation, tyGenericBody} + var askip = skippedNone + var fskip = skippedNone + var t = if a.kind == tyGenericBody: a else: a.skipToGenericBody(askip) + var r = if f.kind == tyGenericBody: f else: f.skipToGenericBody(fskip) + var depth = 0 + while t != nil and not sameObjectTypes(r, t) and askip == fskip: + t = t.skipToGenericBody(askip) + inc depth + if t != nil and askip == fskip: + d = depth + result = true + proc minRel(a, b: TTypeRelation): TTypeRelation = if a <= b: result = a else: result = b @@ -647,7 +688,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = template bindingRet(res) = if doBind: let bound = aOrig.skipTypes({tyRange}).skipIntLit - if doBind: put(c.bindings, f, bound) + put(c.bindings, f, bound) return res template considerPreviousT(body: stmt) {.immediate.} = @@ -930,8 +971,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = result = isConvertible else: discard - of tyEmpty: - if a.kind == tyEmpty: result = isEqual + of tyEmpty, tyVoid: + if a.kind == f.kind: result = isEqual of tyGenericInst: result = typeRel(c, lastSon(f), a) @@ -945,17 +986,19 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = of tyGenericInvocation: var x = a.skipGenericAlias + var depth = 0 if x.kind == tyGenericInvocation or f.sons[0].kind != tyGenericBody: #InternalError("typeRel: tyGenericInvocation -> tyGenericInvocation") # simply no match for now: discard elif x.kind == tyGenericInst and - (f.sons[0] == x.sons[0]) and + ((f.sons[0] == x.sons[0]) or isGenericSubtype(x, f, depth)) and (sonsLen(x) - 1 == sonsLen(f)): for i in countup(1, sonsLen(f) - 1): if x.sons[i].kind == tyGenericParam: internalError("wrong instantiated type!") elif typeRel(c, f.sons[i], x.sons[i]) <= isSubtype: return + c.inheritancePenalty += depth result = isGeneric else: let genericBody = f.sons[0] @@ -1408,7 +1451,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType, if skipTypes(f, abstractVar-{tyTypeDesc}).kind in {tyTuple}: result = implicitConv(nkHiddenSubConv, f, arg, m, c) of isNone: - # do not do this in ``typeRel`` as it then can't infere T in ``ref T``: + # do not do this in ``typeRel`` as it then can't infer T in ``ref T``: if a.kind in {tyProxy, tyUnknown}: inc(m.genericMatches) m.fauxMatch = a.kind @@ -1429,6 +1472,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType, m.baseTypeMatch = true else: result = userConvMatch(c, m, base(f), a, arg) + if result != nil: m.baseTypeMatch = true proc paramTypesMatch*(m: var TCandidate, f, a: PType, arg, argOrig: PNode): PNode = @@ -1558,6 +1602,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, if formal.typ.kind == tyVar: if not n.isLValue: m.state = csNoMatch + m.mutabilityProblem = uint8(f-1) return var @@ -1577,6 +1622,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, while a < n.len: if a >= formalLen-1 and formal != nil and formal.typ.isVarargsUntyped: + incl(marker, formal.position) if container.isNil: container = newNodeIT(nkBracket, n.sons[a].info, arrayConstr(c, n.info)) setSon(m.call, formal.position + 1, container) @@ -1639,6 +1685,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, # beware of the side-effects in 'prepareOperand'! So only do it for # varargs matching. See tests/metatype/tstatic_overloading. m.baseTypeMatch = false + incl(marker, formal.position) n.sons[a] = prepareOperand(c, formal.typ, n.sons[a]) var arg = paramTypesMatch(m, formal.typ, n.sons[a].typ, n.sons[a], nOrig.sons[a]) @@ -1725,15 +1772,18 @@ proc matches*(c: PContext, n, nOrig: PNode, m: var TCandidate) = if formal.ast == nil: if formal.typ.kind == tyVarargs: var container = newNodeIT(nkBracket, n.info, arrayConstr(c, n.info)) - addSon(m.call, implicitConv(nkHiddenStdConv, formal.typ, - container, m, c)) + setSon(m.call, formal.position + 1, + implicitConv(nkHiddenStdConv, formal.typ, container, m, c)) else: # no default value m.state = csNoMatch break else: # use default value: - setSon(m.call, formal.position + 1, copyTree(formal.ast)) + var def = copyTree(formal.ast) + if def.kind == nkNilLit: + def = implicitConv(nkHiddenStdConv, formal.typ, def, m, c) + setSon(m.call, formal.position + 1, def) inc(f) proc argtypeMatches*(c: PContext, f, a: PType): bool = @@ -1746,16 +1796,19 @@ proc argtypeMatches*(c: PContext, f, a: PType): bool = result = res != nil proc instTypeBoundOp*(c: PContext; dc: PSym; t: PType; info: TLineInfo; - op: TTypeAttachedOp): PSym {.procvar.} = + op: TTypeAttachedOp; col: int): PSym {.procvar.} = var m: TCandidate initCandidate(c, m, dc.typ) - var f = dc.typ.sons[1] + if col >= dc.typ.len: + localError(info, errGenerated, "cannot instantiate '" & dc.name.s & "'") + return nil + var f = dc.typ.sons[col] if op == attachedDeepCopy: if f.kind in {tyRef, tyPtr}: f = f.lastSon else: if f.kind == tyVar: f = f.lastSon if typeRel(m, f, t) == isNone: - localError(info, errGenerated, "cannot instantiate 'deepCopy'") + localError(info, errGenerated, "cannot instantiate '" & dc.name.s & "'") else: result = c.semGenerateInstance(c, dc, m.bindings, info) assert sfFromGeneric in result.flags diff --git a/compiler/types.nim b/compiler/types.nim index bada47075..5fbab85b9 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -407,7 +407,8 @@ const "!", "varargs[$1]", "iter[$1]", "Error Type", "BuiltInTypeClass", "UserTypeClass", "UserTypeClassInst", "CompositeTypeClass", - "and", "or", "not", "any", "static", "TypeFromExpr", "FieldAccessor"] + "and", "or", "not", "any", "static", "TypeFromExpr", "FieldAccessor", + "void"] const preferToResolveSymbols = {preferName, preferModuleInfo, preferGenericArg} @@ -540,7 +541,9 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = else: result.add typeToString(t.sons[0]) of tyRange: - result = "range " & rangeToStr(t.n) + result = "range " + if t.n != nil and t.n.kind == nkRange: + result.add rangeToStr(t.n) if prefer != preferExported: result.add("(" & typeToString(t.sons[0]) & ")") of tyProc: @@ -727,6 +730,7 @@ proc equalParam(a, b: PSym): TParamsEquality = result = paramsNotEqual proc sameConstraints(a, b: PNode): bool = + if isNil(a) and isNil(b): return true internalAssert a.len == b.len for i in 1 .. <a.len: if not exprStructuralEquivalent(a[i].sym.constraint, @@ -799,6 +803,8 @@ proc sameTuple(a, b: PType, c: var TSameTypeClosure): bool = result = x.name.id == y.name.id if not result: break else: internalError(a.n.info, "sameTuple") + elif a.n != b.n and (a.n == nil or b.n == nil) and IgnoreTupleFields notin c.flags: + result = false template ifFastObjectTypeCheckFailed(a, b: PType, body: stmt) {.immediate.} = if tfFromGeneric notin a.flags + b.flags: @@ -922,7 +928,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = case a.kind of tyEmpty, tyChar, tyBool, tyNil, tyPointer, tyString, tyCString, - tyInt..tyBigNum, tyStmt, tyExpr: + tyInt..tyBigNum, tyStmt, tyExpr, tyVoid: result = sameFlags(a, b) of tyStatic, tyFromExpr: result = exprStructuralEquivalent(a.n, b.n) and sameFlags(a, b) @@ -1108,7 +1114,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = nil of tyExpr, tyStmt, tyStatic: if kind notin {skParam, skResult}: result = t - of tyEmpty: + of tyVoid: if taField notin flags: result = t of tyTypeClasses: if not (tfGenericTypeParam in t.flags or taField notin flags): result = t @@ -1153,7 +1159,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, if result != nil: break if result.isNil and t.n != nil: result = typeAllowedNode(marker, t.n, kind, flags) - of tyProxy: + of tyProxy, tyEmpty: # for now same as error node; we say it's a valid type as it should # prevent cascading errors: result = nil @@ -1313,6 +1319,9 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt = of tyTypeDesc: result = computeSizeAux(typ.base, a) of tyForward: return szIllegalRecursion + of tyStatic: + if typ.n != nil: result = computeSizeAux(lastSon(typ), a) + else: result = szUnknownSize else: #internalError("computeSizeAux()") result = szUnknownSize diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 678a765f4..a10e7c12e 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -63,8 +63,9 @@ proc opSlurp*(file: string, info: TLineInfo, module: PSym): string = localError(info, errCannotOpenFile, file) result = "" -proc atomicTypeX(name: string; t: PType; info: TLineInfo): PNode = +proc atomicTypeX(name: string; m: TMagic; t: PType; info: TLineInfo): PNode = let sym = newSym(skType, getIdent(name), t.owner, info) + sym.magic = m sym.typ = t result = newSymNode(sym) result.typ = t @@ -72,14 +73,14 @@ proc atomicTypeX(name: string; t: PType; info: TLineInfo): PNode = proc mapTypeToAstX(t: PType; info: TLineInfo; inst=false; allowRecursionX=false): PNode -proc mapTypeToBracketX(name: string; t: PType; info: TLineInfo; +proc mapTypeToBracketX(name: string; m: TMagic; t: PType; info: TLineInfo; inst=false): PNode = result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) - result.add atomicTypeX(name, t, info) + result.add atomicTypeX(name, m, t, info) for i in 0 .. < t.len: if t.sons[i] == nil: - let void = atomicTypeX("void", t, info) - void.typ = newType(tyEmpty, t.owner) + let void = atomicTypeX("void", mVoid, t, info) + void.typ = newType(tyVoid, t.owner) result.add void else: result.add mapTypeToAstX(t.sons[i], info, inst) @@ -87,14 +88,14 @@ proc mapTypeToBracketX(name: string; t: PType; info: TLineInfo; proc mapTypeToAstX(t: PType; info: TLineInfo; inst=false; allowRecursionX=false): PNode = var allowRecursion = allowRecursionX - template atomicType(name): expr = atomicTypeX(name, t, info) + template atomicType(name, m): expr = atomicTypeX(name, m, t, info) template mapTypeToAst(t,info): expr = mapTypeToAstX(t, info, inst) template mapTypeToAstR(t,info): expr = mapTypeToAstX(t, info, inst, true) template mapTypeToAst(t,i,info): expr = if i<t.len and t.sons[i]!=nil: mapTypeToAstX(t.sons[i], info, inst) else: ast.emptyNode - template mapTypeToBracket(name,t,info): expr = - mapTypeToBracketX(name, t, info, inst) + template mapTypeToBracket(name, m, t, info): expr = + mapTypeToBracketX(name, m, t, info, inst) template newNodeX(kind):expr = newNodeIT(kind, if t.n.isNil: info else: t.n.info, t) template newIdent(s):expr = @@ -114,19 +115,20 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; if allowRecursion: # getTypeImpl behavior: turn off recursion allowRecursion = false else: # getTypeInst behavior: return symbol - return atomicType(t.sym.name.s) + return atomicType(t.sym.name.s, t.sym.magic) case t.kind - of tyNone: result = atomicType("none") - of tyBool: result = atomicType("bool") - of tyChar: result = atomicType("char") - of tyNil: result = atomicType("nil") - of tyExpr: result = atomicType("expr") - of tyStmt: result = atomicType("stmt") - of tyEmpty: result = atomicType"void" + of tyNone: result = atomicType("none", mNone) + of tyBool: result = atomicType("bool", mBool) + of tyChar: result = atomicType("char", mChar) + of tyNil: result = atomicType("nil", mNil) + of tyExpr: result = atomicType("expr", mExpr) + of tyStmt: result = atomicType("stmt", mStmt) + of tyVoid: result = atomicType("void", mVoid) + of tyEmpty: result = atomicType("empty", mNone) of tyArrayConstr, tyArray: result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) - result.add atomicType("array") + result.add atomicType("array", mArray) if inst and t.sons[0].kind == tyRange: var rng = newNodeX(nkInfix) rng.add newIdentNode(getIdent(".."), info) @@ -139,10 +141,10 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; of tyTypeDesc: if t.base != nil: result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) - result.add atomicType("typeDesc") + result.add atomicType("typeDesc", mTypeDesc) result.add mapTypeToAst(t.base, info) else: - result = atomicType"typeDesc" + result = atomicType("typeDesc", mTypeDesc) of tyGenericInvocation: result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) for i in 0 .. < t.len: @@ -166,10 +168,11 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; result.add mapTypeToAst(t.sons[0], info) else: if allowRecursion or t.sym==nil: - result = mapTypeToBracket("distinct", t, info) + result = mapTypeToBracket("distinct", mDistinct, t, info) else: - result = atomicType(t.sym.name.s) - of tyGenericParam, tyForward: result = atomicType(t.sym.name.s) + result = atomicType(t.sym.name.s, t.sym.magic) + of tyGenericParam, tyForward: + result = atomicType(t.sym.name.s, t.sym.magic) of tyObject: if inst: result = newNodeX(nkObjectTy) @@ -196,7 +199,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; result.add mapTypeToAst(t.sons[0], info) result.add copyTree(t.n) else: - result = atomicType(t.sym.name.s) + result = atomicType(t.sym.name.s, t.sym.magic) of tyEnum: result = newNodeIT(nkEnumTy, if t.n.isNil: info else: t.n.info, t) result.add copyTree(t.n) @@ -206,22 +209,22 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; for s in t.n.sons: result.add newIdentDefs(s) else: - result = mapTypeToBracket("tuple", t, info) - of tySet: result = mapTypeToBracket("set", t, info) + result = mapTypeToBracket("tuple", mTuple, t, info) + of tySet: result = mapTypeToBracket("set", mSet, t, info) of tyPtr: if inst: result = newNodeX(nkPtrTy) result.add mapTypeToAst(t.sons[0], info) else: - result = mapTypeToBracket("ptr", t, info) + result = mapTypeToBracket("ptr", mPtr, t, info) of tyRef: if inst: result = newNodeX(nkRefTy) result.add mapTypeToAst(t.sons[0], info) else: - result = mapTypeToBracket("ref", t, info) - of tyVar: result = mapTypeToBracket("var", t, info) - of tySequence: result = mapTypeToBracket("seq", t, info) + result = mapTypeToBracket("ref", mRef, t, info) + of tyVar: result = mapTypeToBracket("var", mVar, t, info) + of tySequence: result = mapTypeToBracket("seq", mSeq, t, info) of tyProc: if inst: result = newNodeX(nkProcTy) @@ -235,52 +238,54 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; result.add fp result.add ast.emptyNode # pragmas aren't reconstructed yet else: - result = mapTypeToBracket("proc", t, info) - of tyOpenArray: result = mapTypeToBracket("openArray", t, info) + result = mapTypeToBracket("proc", mNone, t, info) + of tyOpenArray: result = mapTypeToBracket("openArray", mOpenArray, t, info) of tyRange: result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) - result.add atomicType("range") + result.add atomicType("range", mRange) result.add t.n.sons[0].copyTree result.add t.n.sons[1].copyTree - of tyPointer: result = atomicType"pointer" - of tyString: result = atomicType"string" - of tyCString: result = atomicType"cstring" - of tyInt: result = atomicType"int" - of tyInt8: result = atomicType"int8" - of tyInt16: result = atomicType"int16" - of tyInt32: result = atomicType"int32" - of tyInt64: result = atomicType"int64" - of tyFloat: result = atomicType"float" - of tyFloat32: result = atomicType"float32" - of tyFloat64: result = atomicType"float64" - of tyFloat128: result = atomicType"float128" - of tyUInt: result = atomicType"uint" - of tyUInt8: result = atomicType"uint8" - of tyUInt16: result = atomicType"uint16" - of tyUInt32: result = atomicType"uint32" - of tyUInt64: result = atomicType"uint64" - of tyBigNum: result = atomicType"bignum" - of tyConst: result = mapTypeToBracket("const", t, info) - of tyMutable: result = mapTypeToBracket("mutable", t, info) - of tyVarargs: result = mapTypeToBracket("varargs", t, info) - of tyIter: result = mapTypeToBracket("iter", t, info) - of tyProxy: result = atomicType"error" - of tyBuiltInTypeClass: result = mapTypeToBracket("builtinTypeClass", t, info) + of tyPointer: result = atomicType("pointer", mPointer) + of tyString: result = atomicType("string", mString) + of tyCString: result = atomicType("cstring", mCString) + of tyInt: result = atomicType("int", mInt) + of tyInt8: result = atomicType("int8", mInt8) + of tyInt16: result = atomicType("int16", mInt16) + of tyInt32: result = atomicType("int32", mInt32) + of tyInt64: result = atomicType("int64", mInt64) + of tyFloat: result = atomicType("float", mFloat) + of tyFloat32: result = atomicType("float32", mFloat32) + of tyFloat64: result = atomicType("float64", mFloat64) + of tyFloat128: result = atomicType("float128", mFloat128) + of tyUInt: result = atomicType("uint", mUint) + of tyUInt8: result = atomicType("uint8", mUint8) + of tyUInt16: result = atomicType("uint16", mUint16) + of tyUInt32: result = atomicType("uint32", mUint32) + of tyUInt64: result = atomicType("uint64", mUint64) + of tyBigNum: result = atomicType("bignum", mNone) + of tyConst: result = mapTypeToBracket("const", mNone, t, info) + of tyMutable: result = mapTypeToBracket("mutable", mNone, t, info) + of tyVarargs: result = mapTypeToBracket("varargs", mVarargs, t, info) + of tyIter: result = mapTypeToBracket("iter", mNone, t, info) + of tyProxy: result = atomicType("error", mNone) + of tyBuiltInTypeClass: + result = mapTypeToBracket("builtinTypeClass", mNone, t, info) of tyUserTypeClass: - result = mapTypeToBracket("concept", t, info) + result = mapTypeToBracket("concept", mNone, t, info) result.add t.n.copyTree - of tyCompositeTypeClass: result = mapTypeToBracket("compositeTypeClass", t, info) - of tyAnd: result = mapTypeToBracket("and", t, info) - of tyOr: result = mapTypeToBracket("or", t, info) - of tyNot: result = mapTypeToBracket("not", t, info) - of tyAnything: result = atomicType"anything" + of tyCompositeTypeClass: + result = mapTypeToBracket("compositeTypeClass", mNone, t, info) + of tyAnd: result = mapTypeToBracket("and", mAnd, t, info) + of tyOr: result = mapTypeToBracket("or", mOr, t, info) + of tyNot: result = mapTypeToBracket("not", mNot, t, info) + of tyAnything: result = atomicType("anything", mNone) of tyStatic, tyFromExpr, tyFieldAccessor: if inst: if t.n != nil: result = t.n.copyTree - else: result = atomicType "void" + else: result = atomicType("void", mVoid) else: result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) - result.add atomicType "static" + result.add atomicType("static", mNone) if t.n != nil: result.add t.n.copyTree diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index d4966b3e3..1c1d03a4f 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -267,7 +267,7 @@ proc genx(c: PCtx; n: PNode; flags: TGenFlags = {}): TRegister = proc clearDest(c: PCtx; n: PNode; dest: var TDest) {.inline.} = # stmt is different from 'void' in meta programming contexts. # So we only set dest to -1 if 'void': - if dest >= 0 and (n.typ.isNil or n.typ.kind == tyEmpty): + if dest >= 0 and (n.typ.isNil or n.typ.kind == tyVoid): c.freeTemp(dest) dest = -1 @@ -374,7 +374,7 @@ proc canonValue*(n: PNode): PNode = proc rawGenLiteral(c: PCtx; n: PNode): int = result = c.constants.len - assert(n.kind != nkCall) + #assert(n.kind != nkCall) n.flags.incl nfAllConst c.constants.add n.canonValue internalAssert result < 0x7fff @@ -497,12 +497,30 @@ proc genReturn(c: PCtx; n: PNode) = gen(c, n.sons[0]) c.gABC(n, opcRet) + +proc genLit(c: PCtx; n: PNode; dest: var TDest) = + # opcLdConst is now always valid. We produce the necessary copy in the + # assignments now: + #var opc = opcLdConst + if dest < 0: dest = c.getTemp(n.typ) + #elif c.prc.slots[dest].kind == slotFixedVar: opc = opcAsgnConst + let lit = genLiteral(c, n) + c.gABx(n, opcLdConst, dest, lit) + proc genCall(c: PCtx; n: PNode; dest: var TDest) = + # it can happen that due to inlining we have a 'n' that should be + # treated as a constant (see issue #537). + #if n.typ != nil and n.typ.sym != nil and n.typ.sym.magic == mPNimrodNode: + # genLit(c, n, dest) + # return if dest < 0 and not isEmptyType(n.typ): dest = getTemp(c, n.typ) let x = c.getTempRange(n.len, slotTempUnknown) # varargs need 'opcSetType' for the FFI support: - let fntyp = n.sons[0].typ + let fntyp = skipTypes(n.sons[0].typ, abstractInst) for i in 0.. <n.len: + #if i > 0 and i < sonsLen(fntyp): + # let paramType = fntyp.n.sons[i] + # if paramType.typ.isCompileTimeOnly: continue var r: TRegister = x+i c.gen(n.sons[i], r) if i >= fntyp.len: @@ -579,16 +597,27 @@ proc genNew(c: PCtx; n: PNode) = c.freeTemp(dest) proc genNewSeq(c: PCtx; n: PNode) = - let dest = if needsAsgnPatch(n.sons[1]): c.getTemp(n.sons[1].typ) + let t = n.sons[1].typ + let dest = if needsAsgnPatch(n.sons[1]): c.getTemp(t) else: c.genx(n.sons[1]) let tmp = c.genx(n.sons[2]) - c.gABx(n, opcNewSeq, dest, c.genType(n.sons[1].typ.skipTypes( + c.gABx(n, opcNewSeq, dest, c.genType(t.skipTypes( abstractVar-{tyTypeDesc}))) c.gABx(n, opcNewSeq, tmp, 0) c.freeTemp(tmp) c.genAsgnPatch(n.sons[1], dest) c.freeTemp(dest) +proc genNewSeqOfCap(c: PCtx; n: PNode; dest: var TDest) = + let t = n.typ + let tmp = c.getTemp(n.sons[1].typ) + c.gABx(n, opcLdNull, dest, c.genType(t)) + c.gABx(n, opcLdImmInt, tmp, 0) + c.gABx(n, opcNewSeq, dest, c.genType(t.skipTypes( + abstractVar-{tyTypeDesc}))) + c.gABx(n, opcNewSeq, tmp, 0) + c.freeTemp(tmp) + proc genUnaryABC(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = let tmp = c.genx(n.sons[1]) if dest < 0: dest = c.getTemp(n.typ) @@ -764,6 +793,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mNewSeq: unused(n, dest) c.genNewSeq(n) + of mNewSeqOfCap: c.genNewSeqOfCap(n, dest) of mNewString: genUnaryABC(c, n, dest, opcNewStr) # XXX buggy @@ -1307,15 +1337,6 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) = let dest = c.genx(le, {gfAddrOf}) genAsgn(c, dest, ri, requiresCopy) -proc genLit(c: PCtx; n: PNode; dest: var TDest) = - # opcLdConst is now always valid. We produce the necessary copy in the - # assignments now: - #var opc = opcLdConst - if dest < 0: dest = c.getTemp(n.typ) - #elif c.prc.slots[dest].kind == slotFixedVar: opc = opcAsgnConst - let lit = genLiteral(c, n) - c.gABx(n, opcLdConst, dest, lit) - proc genTypeLit(c: PCtx; t: PType; dest: var TDest) = var n = newNode(nkType) n.typ = t @@ -1677,10 +1698,10 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = of skType: genTypeLit(c, s.typ, dest) of skGenericParam: - if c.prc.sym.kind == skMacro: + if c.prc.sym != nil and c.prc.sym.kind == skMacro: genRdVar(c, n, dest, flags) else: - internalError(n.info, "cannot generate code for: " & s.name.s) + globalError(n.info, errGenerated, "cannot generate code for: " & s.name.s) else: globalError(n.info, errGenerated, "cannot generate code for: " & s.name.s) of nkCallKinds: @@ -1788,6 +1809,8 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = genConv(c, n, n.sons[1], dest, opcCast) else: globalError(n.info, errGenerated, "VM is not allowed to 'cast'") + of nkTypeOfExpr: + genTypeLit(c, n.typ, dest) else: globalError(n.info, errGenerated, "cannot generate VM code for " & $n) diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 3e0e05a94..b5ffd51c2 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -36,6 +36,7 @@ type wColon, wColonColon, wEquals, wDot, wDotDot, wStar, wMinus, wMagic, wThread, wFinal, wProfiler, wObjChecks, + wIntDefine, wStrDefine, wDestroy, @@ -121,7 +122,7 @@ const ":", "::", "=", ".", "..", "*", "-", - "magic", "thread", "final", "profiler", "objchecks", + "magic", "thread", "final", "profiler", "objchecks", "intdefine", "strdefine", "destroy", diff --git a/config/nim.cfg b/config/nim.cfg index 93b847784..0cc014a93 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -23,10 +23,6 @@ arm.linux.gcc.linkerexe = "arm-linux-gcc" mips.linux.gcc.exe = "mips-openwrt-linux-gcc" mips.linux.gcc.linkerexe = "mips-openwrt-linux-gcc" -@if not nimfix: - cs:partial -@end - path="$lib/deprecated/core" path="$lib/deprecated/pure" path="$lib/pure/collections" @@ -168,10 +164,5 @@ vcc.options.always = "/nologo" vcc.options.speed = "/O2 /arch:SSE2" vcc.options.size = "/O1" -# Configuration for the Digital Mars C/C++ compiler: -@if windows: - dmc.path = r"$nimrod\dist\dm\bin" -@end - # Configuration for the Tiny C Compiler: tcc.options.always = "-w" diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index d973b922a..112833e58 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -1,5 +1,5 @@ # This is the config file for the documentation generator. -# (c) 2012 Andreas Rumpf +# (c) 2016 Andreas Rumpf # Feel free to edit the templates as you need. If you modify this file, it # might be worth updating the hardcoded values in packages/docutils/rstgen.nim @@ -1235,10 +1235,46 @@ dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator background-repeat: no-repeat; background-image: url("data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="); margin-bottom: -5px; } + div.pragma { + display: none; + } + span.pragmabegin { + cursor: pointer; + } + span.pragmaend { + cursor: pointer; + } </style> + +<script type="text/javascript"> +function togglepragma(d) { + if (d.style.display != 'inline') + d.style.display = 'inline'; + else + d.style.display = 'none'; +} + +function main() { + var elements = document.getElementsByClassName("pragmabegin"); + for (var i = 0; i < elements.length; ++i) { + var e = elements[i]; + e.onclick = function(event) { + togglepragma(event.target.nextSibling); + }; + } + var elements = document.getElementsByClassName("pragmaend"); + for (var i = 0; i < elements.length; ++i) { + var e = elements[i]; + e.onclick = function(event) { + togglepragma(event.target.previousSibling); + }; + } +} +</script> + </head> -<body> +<body onload="main()"> <div class="document" id="documentId"> <div class="container"> <h1 class="title">$title</h1> diff --git a/doc/advopt.txt b/doc/advopt.txt index 02aada4fb..02e69c5b8 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -6,6 +6,7 @@ Advanced commands: //rst2html convert a reStructuredText file to HTML //rst2tex convert a reStructuredText file to TeX //jsondoc extract the documentation to a json file + //jsondoc2 extract documentation to a json file (uses doc2) //buildIndex build an index for the whole documentation //run run the project (with Tiny C backend; buggy!) //genDepend generate a DOT file containing the diff --git a/doc/basicopt.txt b/doc/basicopt.txt index 6a905bd53..9a1cfd956 100644 --- a/doc/basicopt.txt +++ b/doc/basicopt.txt @@ -11,7 +11,9 @@ Arguments: arguments are passed to the program being run (if --run option is selected) Options: -p, --path:PATH add path to search paths - -d, --define:SYMBOL define a conditional symbol + -d, --define:SYMBOL(:VAL) + define a conditional symbol + (Optionally: Define the value for that symbol) -u, --undef:SYMBOL undefine a conditional symbol -f, --forceBuild force rebuilding of all modules --stackTrace:on|off turn stack tracing on|off diff --git a/doc/manual/generics.txt b/doc/manual/generics.txt index 07d98b289..5f22e959a 100644 --- a/doc/manual/generics.txt +++ b/doc/manual/generics.txt @@ -232,16 +232,6 @@ type signatures of the required operations, but since type inference and default parameters are still applied in the provided block, it's also possible to encode usage protocols that do not reveal implementation details. -As a special rule providing further convenience when writing concepts, any -type value appearing in a callable expression will be treated as a variable of -the designated type for overload resolution purposes, unless the type value was -passed in its explicit ``typedesc[T]`` form: - -.. code-block:: nim - type - OutputStream = concept s - write(var s, string) - Much like generics, concepts are instantiated exactly once for each tested type and any static code included within them is also executed once. diff --git a/doc/manual/pragmas.txt b/doc/manual/pragmas.txt index 1a1f0b734..88ddabef8 100644 --- a/doc/manual/pragmas.txt +++ b/doc/manual/pragmas.txt @@ -1011,3 +1011,30 @@ debugging: # ... complex code here that produces crashes ... +compile time define pragmas +--------------------------- + +The pragmas listed here can be used to optionally accept values from +the -d/--define option at compile time. + +The implementation currently provides the following possible options (various +others may be added later). + +=============== ============================================ +pragma description +=============== ============================================ +intdefine Reads in a build-time define as an integer +strdefine Reads in a build-time define as a string +=============== ============================================ + +.. code-block:: nim + const FooBar {.intdefine.}: int = 5 + echo FooBar + +.. code-block:: bash + nim c -d:FooBar=42 foobar.c + +In the above example, providing the -d flag causes the symbol +``FooBar`` to be overwritten at compile time, printing out 42. If the +``-d:FooBar=42`` were to be omitted, the default value of 5 would be +used. diff --git a/doc/manual/procs.txt b/doc/manual/procs.txt index 9ce92de0d..ea6866845 100644 --- a/doc/manual/procs.txt +++ b/doc/manual/procs.txt @@ -215,6 +215,12 @@ the closure and its enclosing scope (i.e. any modifications made to them are visible in both places). The closure environment may be allocated on the heap or on the stack if the compiler determines that this would be safe. +Creating closures in loops +~~~~~~~~~~~~~~~~ + +Since closures capture local variables by reference it is often not wanted +behavior inside loop bodies. See `closureScope <system.html#closureScope>`_ +for details on how to change this behavior. Anonymous Procs --------------- @@ -223,7 +229,7 @@ Procs can also be treated as expressions, in which case it's allowed to omit the proc's name. .. code-block:: nim - var cities = @["Frankfurt", "Tokyo", "New York"] + var cities = @["Frankfurt", "Tokyo", "New York", "Kyiv"] cities.sort(proc (x,y: string): int = cmp(x.len, y.len)) @@ -619,7 +625,7 @@ Note that ``system.finished`` is error prone to use because it only returns 3 0 -Instead this code has be used: +Instead this code has to be used: .. code-block:: nim var c = mycount # instantiate the iterator diff --git a/doc/manual/special_ops.txt b/doc/manual/special_ops.txt index 702693423..1c7136bec 100644 --- a/doc/manual/special_ops.txt +++ b/doc/manual/special_ops.txt @@ -4,6 +4,9 @@ Special Operators dot operators ------------- +**Note**: Dot operators are still experimental and so need to be enabled +via ``{.experimental.}``. + Nim offers a special family of dot operators that can be used to intercept and rewrite proc call and field access attempts, referring to previously undeclared symbol names. They can be used to provide a diff --git a/doc/nep1.rst b/doc/nep1.rst index b4bd6309c..b5991ba9e 100644 --- a/doc/nep1.rst +++ b/doc/nep1.rst @@ -125,8 +125,8 @@ changed in the future. Coding Conventions ------------------ -- The 'return' statement should only be used when it's control-flow properties - are required. Use a procedures implicit 'result' variable instead. This +- The 'return' statement should only be used when its control-flow properties + are required. Use a procedure's implicit 'result' variable instead. This improves readability. - Prefer to return `[]` and `""` instead of `nil`, or throw an exception if @@ -150,7 +150,7 @@ Conventions for multi-line statements and expressions - Any tuple type declarations that are longer than one line should use the regular object type layout instead. This enhances the readability of the - tuple declaration by splitting its members information across multiple lines. + tuple declaration by splitting its members' information across multiple lines. .. code-block:: nim type diff --git a/doc/nimc.rst b/doc/nimc.rst index 48dbaeb21..eb1beb549 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -98,6 +98,11 @@ enable builds in release mode (``-d:release``) where certain safety checks are omitted for better performance. Another common use is the ``-d:ssl`` switch to activate `SSL sockets <sockets.html>`_. +Additionally, you may pass a value along with the symbol: ``-d:x=y`` +which may be used in conjunction with the `compile time define +pragmas<manual.html#implementation-specific-pragmas-compile-time-define-pragmas>`_ +to override symbols during build time. + Configuration files ------------------- @@ -370,7 +375,10 @@ For example, to generate code for an `AVR`:idx: processor use this command:: For the ``standalone`` target one needs to provide a file ``panicoverride.nim``. See ``tests/manyloc/standalone/panicoverride.nim`` for an example -implementation. +implementation. Additionally, users should specify the +amount of heap space to use with the ``-d:StandaloneHeapSize=<size>`` +command line switch. Note that the total heap size will be +``<size> * sizeof(float64)``. Nim for realtime systems diff --git a/koch.nim b/koch.nim index 9ccc752c0..04f6a4e4e 100644 --- a/koch.nim +++ b/koch.nim @@ -1,7 +1,7 @@ # # # Maintenance program for Nim -# (c) Copyright 2015 Andreas Rumpf +# (c) Copyright 2016 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -30,7 +30,7 @@ const +-----------------------------------------------------------------+ | Maintenance program for Nim | | Version $1| -| (c) 2015 Andreas Rumpf | +| (c) 2016 Andreas Rumpf | +-----------------------------------------------------------------+ Build time: $2, $3 @@ -80,17 +80,17 @@ proc findNim(): string = # assume there is a symlink to the exe or something: return nim -proc exec(cmd: string, errorcode: int = QuitFailure, additionalPATH = "") = - let prevPATH = getEnv("PATH") - if additionalPATH.len > 0: +proc exec(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = + let prevPath = getEnv("PATH") + if additionalPath.len > 0: var absolute = additionalPATH if not absolute.isAbsolute: absolute = getCurrentDir() / absolute echo("Adding to $PATH: ", absolute) - putEnv("PATH", prevPATH & PathSep & absolute) + putEnv("PATH", prevPath & PathSep & absolute) echo(cmd) if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) - putEnv("PATH", prevPATH) + putEnv("PATH", prevPath) proc execCleanPath(cmd: string, additionalPath = ""; errorcode: int = QuitFailure) = @@ -433,6 +433,7 @@ of cmdArgument: of "boot": boot(op.cmdLineRest) of "clean": clean(op.cmdLineRest) of "web": web(op.cmdLineRest) + of "json2": web("--json2 " & op.cmdLineRest) of "website": website(op.cmdLineRest & " --googleAnalytics:UA-48159761-1") of "web0": # undocumented command for Araq-the-merciful: diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 16a954611..9623bec28 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -692,6 +692,14 @@ proc `pragma=`*(someProc: NimNode; val: NimNode){.compileTime.}= assert val.kind in {nnkEmpty, nnkPragma} someProc[4] = val +proc addPragma*(someProc, pragma: NimNode) {.compileTime.} = + ## Adds pragma to routine definition + someProc.expectRoutine + var pragmaNode = someProc.pragma + if pragmaNode.isNil or pragmaNode.kind == nnkEmpty: + pragmaNode = newNimNode(nnkPragma) + someProc.pragma = pragmaNode + pragmaNode.add(pragma) template badNodeKind(k; f): stmt{.immediate.} = assert false, "Invalid node kind " & $k & " for macros.`" & $f & "`" @@ -758,12 +766,12 @@ template findChild*(n: NimNode; cond: expr): NimNode {. ## var res = findChild(n, it.kind == nnkPostfix and ## it.basename.ident == !"foo") block: - var result: NimNode + var res: NimNode for it in n.children: if cond: - result = it + res = it break - result + res proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} = ## Insert node B into A at pos @@ -884,7 +892,7 @@ proc boolVal*(n: NimNode): bool {.compileTime, noSideEffect.} = else: n == bindSym"true" # hacky solution for now when not defined(booting): - template emit*(e: static[string]): stmt = + template emit*(e: static[string]): stmt {.deprecated.} = ## accepts a single string argument and treats it as nim code ## that should be inserted verbatim in the program ## Example: @@ -892,6 +900,7 @@ when not defined(booting): ## .. code-block:: nim ## emit("echo " & '"' & "hello world".toUpper & '"') ## + ## Deprecated since version 0.15 since it's so rarely useful. macro payload: stmt {.gensym.} = result = parseStmt(e) payload() diff --git a/lib/deprecated/pure/sockets.nim b/lib/deprecated/pure/sockets.nim index 20e6d9364..34c2e7a7d 100644 --- a/lib/deprecated/pure/sockets.nim +++ b/lib/deprecated/pure/sockets.nim @@ -711,8 +711,13 @@ proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = cint(sockets.AF_INET)) if s == nil: raiseOSError(osLastError()) else: - var s = posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, - cint(posix.AF_INET)) + var s = + when defined(android4): + posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, + cint(posix.AF_INET)) + else: + posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, + cint(posix.AF_INET)) if s == nil: raiseOSError(osLastError(), $hstrerror(h_errno)) diff --git a/lib/nimbase.h b/lib/nimbase.h index d5a551352..3635ca006 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -102,7 +102,7 @@ __clang__ defined __ICL || \ defined __DMC__ || \ defined __BORLANDC__ ) -# define NIM_THREADVAR __declspec(thread) +# define NIM_THREADVAR __declspec(thread) /* note that ICC (linux) and Clang are covered by __GNUC__ */ #elif defined __GNUC__ || \ defined __SUNPRO_C || \ @@ -449,8 +449,8 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); } /* Test to see if Nim and the C compiler agree on the size of a pointer. On disagreement, your C compiler will say something like: - "error: 'assert_numbits' declared as an array with a negative size" */ -typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; + "error: 'Nim_and_C_compiler_disagree_on_target_architecture' declared as an array with a negative size" */ +typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; #endif #ifdef __cplusplus diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index ebe6fa19b..53699166f 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -49,7 +49,7 @@ type TMsgKind: MsgKind].} const - messages: array [MsgKind, string] = [ + messages: array[MsgKind, string] = [ meCannotOpenFile: "cannot open '$1'", meExpected: "'$1' expected", meGridTableNotImplemented: "grid table is not implemented", diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 6a13eec28..cd21b5c8b 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -110,7 +110,7 @@ type ## (not POSIX) when defined(linux) or defined(bsd): d_off*: Off ## Not an offset. Value that ``telldir()`` would return. - d_name*: array [0..255, char] ## Name of entry. + d_name*: array[0..255, char] ## Name of entry. Tflock* {.importc: "struct flock", final, pure, header: "<fcntl.h>".} = object ## flock type @@ -242,7 +242,7 @@ type ## network to which this node is attached, if any. release*, ## Current release level of this implementation. version*, ## Current version level of this release. - machine*: array [0..255, char] ## Name of the hardware type on which the + machine*: array[0..255, char] ## Name of the hardware type on which the ## system is running. Sem* {.importc: "sem_t", header: "<semaphore.h>", final, pure.} = object @@ -463,12 +463,12 @@ type SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", pure, final.} = object ## struct sockaddr sa_family*: TSa_Family ## Address family. - sa_data*: array [0..255, char] ## Socket address (variable-length data). + sa_data*: array[0..255, char] ## Socket address (variable-length data). Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>", pure, final.} = object ## struct sockaddr_un sun_family*: TSa_Family ## Address family. - sun_path*: array [0..Sockaddr_un_path_length-1, char] ## Socket path + sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path Sockaddr_storage* {.importc: "struct sockaddr_storage", header: "<sys/socket.h>", @@ -526,7 +526,7 @@ type In6Addr* {.importc: "struct in6_addr", pure, final, header: "<netinet/in.h>".} = object ## struct in6_addr - s6_addr*: array [0..15, char] + s6_addr*: array[0..15, char] Sockaddr_in6* {.importc: "struct sockaddr_in6", pure, final, header: "<netinet/in.h>".} = object ## struct sockaddr_in6 @@ -2616,8 +2616,12 @@ proc gai_strerror*(a1: cint): cstring {.importc:"(char *)$1", header: "<netdb.h> proc getaddrinfo*(a1, a2: cstring, a3: ptr AddrInfo, a4: var ptr AddrInfo): cint {.importc, header: "<netdb.h>".} -proc gethostbyaddr*(a1: pointer, a2: Socklen, a3: cint): ptr Hostent {. - importc, header: "<netdb.h>".} +when not defined(android4): + proc gethostbyaddr*(a1: pointer, a2: Socklen, a3: cint): ptr Hostent {. + importc, header: "<netdb.h>".} +else: + proc gethostbyaddr*(a1: cstring, a2: cint, a3: cint): ptr Hostent {. + importc, header: "<netdb.h>".} proc gethostbyname*(a1: cstring): ptr Hostent {.importc, header: "<netdb.h>".} proc gethostent*(): ptr Hostent {.importc, header: "<netdb.h>".} @@ -2649,7 +2653,7 @@ proc poll*(a1: ptr TPollfd, a2: Tnfds, a3: int): cint {. proc realpath*(name, resolved: cstring): cstring {. importc: "realpath", header: "<stdlib.h>".} -proc utimes*(path: cstring, times: ptr array [2, Timeval]): int {. +proc utimes*(path: cstring, times: ptr array[2, Timeval]): int {. importc: "utimes", header: "<sys/time.h>".} ## Sets file access and modification times. ## diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index c5b516b39..92a737a47 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -434,6 +434,9 @@ when defined(windows) or defined(nimdoc): fd*: AsyncFD # TODO: Rename this. cb*: proc (fd: AsyncFD, bytesTransferred: Dword, errcode: OSErrorCode) {.closure,gcsafe.} + cell*: ForeignCell # we need this `cell` to protect our `cb` environment, + # when using RegisterWaitForSingleObject, because + # waiting is done in different thread. PDispatcher* = ref object of PDispatcherBase ioPort: Handle @@ -517,6 +520,13 @@ when defined(windows) or defined(nimdoc): customOverlapped.data.cb(customOverlapped.data.fd, lpNumberOfBytesTransferred, OSErrorCode(-1)) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) else: let errCode = osLastError() @@ -524,6 +534,8 @@ when defined(windows) or defined(nimdoc): assert customOverlapped.data.fd == lpCompletionKey.AsyncFD customOverlapped.data.cb(customOverlapped.data.fd, lpNumberOfBytesTransferred, errCode) + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) GC_unref(customOverlapped) else: if errCode.int32 == WAIT_TIMEOUT: @@ -850,6 +862,101 @@ when defined(windows) or defined(nimdoc): # free ``ol``. return retFuture + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: Socklen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to specified destination ``saddr``, using + ## socket ``socket``. The returned future will complete once all data + ## has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("sendTo") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](data) + dataBuf.len = size.ULONG + var bytesSent = 0.Dword + var lowFlags = 0.Dword + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen: cint = cint(saddrLen) + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, + lowFlags, cast[ptr SockAddr](addr(staddr[0])), + stalen, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``buf``, which must + ## be at least of size ``size``, address of datagram's sender will be + ## stored into ``saddr`` and ``saddrLen``. Returned future will complete + ## once one datagram has been received, and will return size of packet + ## received. + verifyPresence(socket) + var retFuture = newFuture[int]("recvFromInto") + + var dataBuf = TWSABuf(buf: cast[cstring](data), len: size.ULONG) + + var bytesReceived = 0.Dword + var lowFlags = 0.Dword + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount <= size + retFuture.complete(bytesCount) + else: + # datagram sockets don't have disconnection, + # so we can just raise an exception + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, + addr bytesReceived, addr lowFlags, + saddr, cast[ptr cint](saddrLen), + cast[POVERLAPPED](ol), nil) + if res == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) + return retFuture + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): Future[tuple[address: string, client: AsyncFD]] = ## Accepts a new connection. Returns a future containing the client socket @@ -1026,6 +1133,10 @@ when defined(windows) or defined(nimdoc): # poll() GC_ref(pcd.ovl) ) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + # This is main part of `hacky way` is using WSAEventSelect, so `hEvent` # will be signaled when appropriate `mask` events will be triggered. if wsaEventSelect(fd.SocketHandle, hEvent, mask) != 0: @@ -1343,6 +1454,60 @@ else: addWrite(socket, cb) return retFuture + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: SockLen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` of size ``size`` in bytes to specified destination + ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``. + ## The returned future will complete once all data has been sent. + var retFuture = newFuture[void]("sendTo") + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen = saddrLen + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + proc cb(sock: AsyncFD): bool = + result = true + let res = sendto(sock.SocketHandle, data, size, MSG_NOSIGNAL, + cast[ptr SockAddr](addr(staddr[0])), stalen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete() + + addWrite(socket, cb) + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``data``, which must + ## be at least of size ``size`` in bytes, address of datagram's sender + ## will be stored into ``saddr`` and ``saddrLen``. Returned future will + ## complete once one datagram has been received, and will return size + ## of packet received. + var retFuture = newFuture[int]("recvFromInto") + proc cb(sock: AsyncFD): bool = + result = true + let res = recvfrom(sock.SocketHandle, data, size.cint, flags.toOSFlags(), + saddr, saddrLen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false + else: + retFuture.complete(res) + addRead(socket, cb) + return retFuture + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): Future[tuple[address: string, client: AsyncFD]] = var retFuture = newFuture[tuple[address: string, @@ -1377,6 +1542,24 @@ proc sleepAsync*(ms: int): Future[void] = p.timers.push((epochTime() + (ms / 1000), retFuture)) return retFuture +proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] = + ## Returns a future which will complete once ``fut`` completes or after + ## ``timeout`` milliseconds has elapsed. + ## + ## If ``fut`` completes first the returned future will hold true, + ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned + ## future will hold false. + + var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") + var timeoutFuture = sleepAsync(timeout) + fut.callback = + proc () = + if not retFuture.finished: retFuture.complete(true) + timeoutFuture.callback = + proc () = + if not retFuture.finished: retFuture.complete(false) + return retFuture + proc accept*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): Future[AsyncFD] = ## Accepts a new connection. Returns a future containing the client socket @@ -1516,15 +1699,17 @@ proc processBody(node, retFutureSym: NimNode, else: result.add newCall(newIdentNode("complete"), retFutureSym) else: - result.add newCall(newIdentNode("complete"), retFutureSym, - node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt)) + let x = node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt) + if x.kind == nnkYieldStmt: result.add x + else: + result.add newCall(newIdentNode("complete"), retFutureSym, x) result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt of nnkCommand, nnkCall: if node[0].kind == nnkIdent and node[0].ident == !"await": case node[1].kind - of nnkIdent, nnkInfix: + of nnkIdent, nnkInfix, nnkDotExpr: # await x # await x or y result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x @@ -1687,38 +1872,40 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # -> complete(retFuture, result) var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) - if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), - newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( - newIdentNode("warning"), newIdentNode("resultshadowed")), - newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} - - procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( - newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T - - procBody.insert(2, newNimNode(nnkPragma).add( - newIdentNode("pop"))) # -> {.pop.}) - - procBody.add( - newCall(newIdentNode("complete"), - retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) - else: - # -> complete(retFuture) - procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + # don't do anything with forward bodies (empty) + if procBody.kind != nnkEmpty: + if not subtypeIsVoid: + procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), + newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( + newIdentNode("warning"), newIdentNode("resultshadowed")), + newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} + + procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( + newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T + + procBody.insert(2, newNimNode(nnkPragma).add( + newIdentNode("pop"))) # -> {.pop.}) + + procBody.add( + newCall(newIdentNode("complete"), + retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) + else: + # -> complete(retFuture) + procBody.add(newCall(newIdentNode("complete"), retFutureSym)) - var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], - procBody, nnkIteratorDef) - closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) - outerProcBody.add(closureIterator) + var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], + procBody, nnkIteratorDef) + closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) + outerProcBody.add(closureIterator) - # -> createCb(retFuture) - #var cbName = newIdentNode("cb") - var procCb = newCall(bindSym"createCb", retFutureSym, iteratorNameSym, - newStrLitNode(prc[0].getName)) - outerProcBody.add procCb + # -> createCb(retFuture) + #var cbName = newIdentNode("cb") + var procCb = getAst createCb(retFutureSym, iteratorNameSym, + newStrLitNode(prc[0].getName)) + outerProcBody.add procCb - # -> return retFuture - outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) + # -> return retFuture + outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) result = prc @@ -1732,9 +1919,8 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = if returnType.kind == nnkEmpty: # Add Future[void] result[3][0] = parseExpr("Future[void]") - - result[6] = outerProcBody - + if procBody.kind != nnkEmpty: + result[6] = outerProcBody #echo(treeRepr(result)) #if prc[0].getName == "testInfix": # echo(toStrLit(result)) @@ -1748,6 +1934,8 @@ macro async*(prc: stmt): stmt {.immediate.} = result.add asyncSingleProc(oneProc) else: result = asyncSingleProc(prc) + when defined(nimDumpAsync): + echo repr result proc recvLine*(socket: AsyncFD): Future[string] {.async.} = ## Reads a line of data from ``socket``. Returned future will complete once diff --git a/lib/pure/collections/queues.nim b/lib/pure/collections/queues.nim index b9bf33bff..399e4d413 100644 --- a/lib/pure/collections/queues.nim +++ b/lib/pure/collections/queues.nim @@ -8,56 +8,142 @@ # ## Implementation of a `queue`:idx:. The underlying implementation uses a ``seq``. +## +## None of the procs that get an individual value from the queue can be used +## on an empty queue. +## If compiled with `boundChecks` option, those procs will raise an `IndexError` +## on such access. This should not be relied upon, as `-d:release` will +## disable those checks and may return garbage or crash the program. +## +## As such, a check to see if the queue is empty is needed before any +## access, unless your program logic guarantees it indirectly. +## +## .. code-block:: Nim +## proc foo(a, b: Positive) = # assume random positive values for `a` and `b` +## var q = initQueue[int]() # initializes the object +## for i in 1 ..< a: q.add i # populates the queue +## +## if b < q.len: # checking before indexed access +## echo "The element at index position ", b, " is ", q[b] +## +## # The following two lines don't need any checking on access due to the +## # logic of the program, but that would not be the case if `a` could be 0. +## assert q.front == 1 +## assert q.back == a +## +## while q.len > 0: # checking if the queue is empty +## echo q.pop() +## ## Note: For inter thread communication use ## a `Channel <channels.html>`_ instead. import math type - Queue*[T] = object ## a queue + Queue*[T] = object ## A queue. data: seq[T] rd, wr, count, mask: int {.deprecated: [TQueue: Queue].} -proc initQueue*[T](initialSize=4): Queue[T] = - ## creates a new queue. `initialSize` needs to be a power of 2. +proc initQueue*[T](initialSize: int = 4): Queue[T] = + ## Create a new queue. + ## Optionally, the initial capacity can be reserved via `initialSize` as a + ## performance optimization. The length of a newly created queue will still + ## be 0. + ## + ## `initialSize` needs to be a power of two. If you need to accept runtime + ## values for this you could use the ``nextPowerOfTwo`` proc from the + ## `math <math.html>`_ module. assert isPowerOfTwo(initialSize) result.mask = initialSize-1 newSeq(result.data, initialSize) -proc len*[T](q: Queue[T]): int = - ## returns the number of elements of `q`. +proc len*[T](q: Queue[T]): int {.inline.}= + ## Return the number of elements of `q`. result = q.count +template emptyCheck(q) = + # Bounds check for the regular queue access. + when compileOption("boundChecks"): + if unlikely(q.count < 1): + raise newException(IndexError, "Empty queue.") + +template xBoundsCheck(q, i) = + # Bounds check for the array like accesses. + when compileOption("boundChecks"): # d:release should disable this. + if unlikely(i >= q.count): # x < q.low is taken care by the Natural parameter + raise newException(IndexError, + "Out of bounds: " & $i & " > " & $(q.count - 1)) + +proc front*[T](q: Queue[T]): T {.inline.}= + ## Return the oldest element of `q`. Equivalent to `q.pop()` but does not + ## remove it from the queue. + emptyCheck(q) + result = q.data[q.rd] + +proc back*[T](q: Queue[T]): T {.inline.} = + ## Return the newest element of `q` but does not remove it from the queue. + emptyCheck(q) + result = q.data[q.wr - 1 and q.mask] + +proc `[]`*[T](q: Queue[T], i: Natural) : T {.inline.} = + ## Access the i-th element of `q` by order of insertion. + ## q[0] is the oldest (the next one q.pop() will extract), + ## q[^1] is the newest (last one added to the queue). + xBoundsCheck(q, i) + return q.data[q.rd + i and q.mask] + +proc `[]`*[T](q: var Queue[T], i: Natural): var T {.inline.} = + ## Access the i-th element of `q` and returns a mutable + ## reference to it. + xBoundsCheck(q, i) + return q.data[q.rd + i and q.mask] + +proc `[]=`* [T] (q: var Queue[T], i: Natural, val : T) {.inline.} = + ## Change the i-th element of `q`. + xBoundsCheck(q, i) + q.data[q.rd + i and q.mask] = val + iterator items*[T](q: Queue[T]): T = - ## yields every element of `q`. + ## Yield every element of `q`. var i = q.rd - var c = q.count - while c > 0: - dec c + for c in 0 ..< q.count: yield q.data[i] i = (i + 1) and q.mask iterator mitems*[T](q: var Queue[T]): var T = - ## yields every element of `q`. + ## Yield every element of `q`. var i = q.rd - var c = q.count - while c > 0: - dec c + for c in 0 ..< q.count: yield q.data[i] i = (i + 1) and q.mask +iterator pairs*[T](q: Queue[T]): tuple[key: int, val: T] = + ## Yield every (position, value) of `q`. + var i = q.rd + for c in 0 ..< q.count: + yield (c, q.data[i]) + i = (i + 1) and q.mask + +proc contains*[T](q: Queue[T], item: T): bool {.inline.} = + ## Return true if `item` is in `q` or false if not found. Usually used + ## via the ``in`` operator. It is the equivalent of ``q.find(item) >= 0``. + ## + ## .. code-block:: Nim + ## if x in q: + ## assert q.contains x + for e in q: + if e == item: return true + return false + proc add*[T](q: var Queue[T], item: T) = - ## adds an `item` to the end of the queue `q`. + ## Add an `item` to the end of the queue `q`. var cap = q.mask+1 - if q.count >= cap: - var n: seq[T] - newSeq(n, cap*2) - var i = 0 - for x in items(q): + if unlikely(q.count >= cap): + var n = newSeq[T](cap*2) + for i, x in q: # don't use copyMem because the GC and because it's slower. shallowCopy(n[i], x) - inc i shallowCopy(q.data, n) q.mask = cap*2 - 1 q.wr = q.count @@ -66,37 +152,104 @@ proc add*[T](q: var Queue[T], item: T) = q.data[q.wr] = item q.wr = (q.wr + 1) and q.mask -proc enqueue*[T](q: var Queue[T], item: T) = - ## alias for the ``add`` operation. - add(q, item) - -proc dequeue*[T](q: var Queue[T]): T = - ## removes and returns the first element of the queue `q`. - assert q.count > 0 +proc default[T](t: typedesc[T]): T {.inline.} = discard +proc pop*[T](q: var Queue[T]): T {.inline, discardable.} = + ## Remove and returns the first (oldest) element of the queue `q`. + emptyCheck(q) dec q.count result = q.data[q.rd] + q.data[q.rd] = default(type(result)) q.rd = (q.rd + 1) and q.mask +proc enqueue*[T](q: var Queue[T], item: T) = + ## Alias for the ``add`` operation. + q.add(item) + +proc dequeue*[T](q: var Queue[T]): T = + ## Alias for the ``pop`` operation. + q.pop() + proc `$`*[T](q: Queue[T]): string = - ## turns a queue into its string representation. + ## Turn a queue into its string representation. result = "[" - for x in items(q): + for x in items(q): # Don't remove the items here for reasons that don't fit in this margin. if result.len > 1: result.add(", ") result.add($x) result.add("]") when isMainModule: - var q = initQueue[int]() + var q = initQueue[int](1) q.add(123) q.add(9) - q.add(4) - var first = q.dequeue + q.enqueue(4) + var first = q.dequeue() q.add(56) q.add(6) - var second = q.dequeue + var second = q.pop() q.add(789) assert first == 123 assert second == 9 assert($q == "[4, 56, 6, 789]") + assert q[0] == q.front and q.front == 4 + assert q[^1] == q.back and q.back == 789 + q[0] = 42 + q[^1] = 7 + + assert 6 in q and 789 notin q + assert q.find(6) >= 0 + assert q.find(789) < 0 + + for i in -2 .. 10: + if i in q: + assert q.contains(i) and q.find(i) >= 0 + else: + assert(not q.contains(i) and q.find(i) < 0) + + when compileOption("boundChecks"): + try: + echo q[99] + assert false + except IndexError: + discard + + try: + assert q.len == 4 + for i in 0 ..< 5: q.pop() + assert false + except IndexError: + discard + + # grabs some types of resize error. + q = initQueue[int]() + for i in 1 .. 4: q.add i + q.pop() + q.pop() + for i in 5 .. 8: q.add i + assert $q == "[3, 4, 5, 6, 7, 8]" + + # Similar to proc from the documentation example + proc foo(a, b: Positive) = # assume random positive values for `a` and `b`. + var q = initQueue[int]() + assert q.len == 0 + for i in 1 .. a: q.add i + + if b < q.len: # checking before indexed access. + assert q[b] == b + 1 + + # The following two lines don't need any checking on access due to the logic + # of the program, but that would not be the case if `a` could be 0. + assert q.front == 1 + assert q.back == a + + while q.len > 0: # checking if the queue is empty + assert q.pop() > 0 + + #foo(0,0) + foo(8,5) + foo(10,9) + foo(1,1) + foo(2,1) + foo(1,5) + foo(3,2) diff --git a/lib/pure/collections/rtarrays.nim b/lib/pure/collections/rtarrays.nim index 9d8085643..2abe9d1f8 100644 --- a/lib/pure/collections/rtarrays.nim +++ b/lib/pure/collections/rtarrays.nim @@ -18,7 +18,7 @@ type RtArray*[T] = object ## L: Natural spart: seq[T] - apart: array [ArrayPartSize, T] + apart: array[ArrayPartSize, T] UncheckedArray* {.unchecked.}[T] = array[0..100_000_000, T] template usesSeqPart(x): expr = x.L > ArrayPartSize diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index e2081e5bf..20e06aaae 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -58,7 +58,7 @@ proc isValid*[A](s: HashSet[A]): bool = ## initialized. Example: ## ## .. code-block :: - ## proc savePreferences(options: Set[string]) = + ## proc savePreferences(options: HashSet[string]) = ## assert options.isValid, "Pass an initialized set!" ## # Do stuff here, may crash in release builds! result = not s.data.isNil @@ -72,7 +72,7 @@ proc len*[A](s: HashSet[A]): int = ## ## .. code-block:: ## - ## var values: Set[int] + ## var values: HashSet[int] ## assert(not values.isValid) ## assert values.len == 0 result = s.counter @@ -338,7 +338,7 @@ proc init*[A](s: var HashSet[A], initialSize=64) = ## existing values and calling `excl() <#excl,TSet[A],A>`_ on them. Example: ## ## .. code-block :: - ## var a: Set[int] + ## var a: HashSet[int] ## a.init(4) ## a.incl(2) ## a.init diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index e454a43cb..8d56669da 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -16,6 +16,39 @@ ## semantics, this means that ``=`` performs a copy of the hash table. ## For **reference** semantics use the ``Ref`` variant: ``TableRef``, ## ``OrderedTableRef``, ``CountTableRef``. +## To give an example, when `a` is a Table, then `var b = a` gives `b` +## as a new independent table. b is initialised with the contents of `a`. +## Changing `b` does not affect `a` and vice versa: +## +## .. code-block:: +## import tables +## +## var +## a = {1: "one", 2: "two"}.toTable # creates a Table +## b = a +## +## echo a, b # output: {1: one, 2: two}{1: one, 2: two} +## +## b[3] = "three" +## echo a, b # output: {1: one, 2: two}{1: one, 2: two, 3: three} +## echo a == b # output: false +## +## On the other hand, when `a` is a TableRef instead, then changes to `b` also affect `a`. +## Both `a` and `b` reference the same data structure: +## +## .. code-block:: +## import tables +## +## var +## a = {1: "one", 2: "two"}.newTable # creates a TableRef +## b = a +## +## echo a, b # output: {1: one, 2: two}{1: one, 2: two} +## +## b[3] = "three" +## echo a, b # output: {1: one, 2: two, 3: three}{1: one, 2: two, 3: three} +## echo a == b # output: true +## ## ## If you are using simple standard types like ``int`` or ``string`` for the ## keys of the table you won't have any problems, but as soon as you try to use @@ -298,18 +331,17 @@ proc hasKey*[A, B](t: TableRef[A, B], key: A): bool = ## returns true iff `key` is in the table `t`. result = t[].hasKey(key) -template equalsImpl() = +template equalsImpl(t) = if s.counter == t.counter: # different insertion orders mean different 'data' seqs, so we have # to use the slow route here: for key, val in s: - # prefix notation leads to automatic dereference in case of PTable if not t.hasKey(key): return false - if t[key] != val: return false + if t.getOrDefault(key) != val: return false return true proc `==`*[A, B](s, t: Table[A, B]): bool = - equalsImpl() + equalsImpl(t) proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = ## Index the collection with the proc provided. @@ -399,7 +431,7 @@ proc `$`*[A, B](t: TableRef[A, B]): string = proc `==`*[A, B](s, t: TableRef[A, B]): bool = if isNil(s): result = isNil(t) elif isNil(t): result = false - else: equalsImpl() + else: equalsImpl(t[]) proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index d620e816e..1fe0b297b 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -433,7 +433,7 @@ proc htmlTag*(n: XmlNode): HtmlTag = proc htmlTag*(s: string): HtmlTag = ## converts `s` to a ``HtmlTag``. If `s` is no HTML tag, ``tagUnknown`` is ## returned. - let s = if allLower(s): s else: s.toLower + let s = if allLower(s): s else: toLowerAscii(s) result = toHtmlTag(s) proc entityToUtf8*(entity: string): string = @@ -464,12 +464,18 @@ proc untilElementEnd(x: var XmlParser, result: XmlNode, case x.kind of xmlElementStart, xmlElementOpen: case result.htmlTag - of tagLi, tagP, tagDt, tagDd, tagInput, tagOption: - # some tags are common to have no ``</end>``, like ``<li>``: + of tagP, tagInput, tagOption: + # some tags are common to have no ``</end>``, like ``<li>`` but + # allow ``<p>`` in `<dd>`, `<dt>` and ``<li>`` in next case if htmlTag(x.elemName) in {tagLi, tagP, tagDt, tagDd, tagInput, tagOption}: errors.add(expected(x, result)) break + of tagDd, tagDt, tagLi: + if htmlTag(x.elemName) in {tagLi, tagDt, tagDd, tagInput, + tagOption}: + errors.add(expected(x, result)) + break of tagTd, tagTh: if htmlTag(x.elemName) in {tagTr, tagTd, tagTh, tagTfoot, tagThead}: errors.add(expected(x, result)) @@ -513,13 +519,13 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = errors.add(errorMsg(x)) next(x) of xmlElementStart: - result = newElement(x.elemName.toLower) + result = newElement(toLowerAscii(x.elemName)) next(x) untilElementEnd(x, result, errors) of xmlElementEnd: errors.add(errorMsg(x, "unexpected ending tag: " & x.elemName)) of xmlElementOpen: - result = newElement(x.elemName.toLower) + result = newElement(toLowerAscii(x.elemName)) next(x) result.attrs = newStringTable() while true: diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 41d4b0ee7..37bc5d8f4 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -167,7 +167,7 @@ proc parseChunks(s: Socket, timeout: int): string = # Trailer headers will only be sent if the request specifies that we want # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 -proc parseBody(s: Socket, headers: HttpHeaders, timeout: int): string = +proc parseBody(s: Socket, headers: HttpHeaders, httpVersion: string, timeout: int): string = result = "" if headers.getOrDefault"Transfer-Encoding" == "chunked": result = parseChunks(s, timeout) @@ -193,7 +193,7 @@ proc parseBody(s: Socket, headers: HttpHeaders, timeout: int): string = # -REGION- Connection: Close # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5 - if headers.getOrDefault"Connection" == "close": + if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0": var buf = "" while true: buf = newString(4000) @@ -249,7 +249,7 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response = if not fullyRead: httpError("Connection was closed before full request has been made") if getBody: - result.body = parseBody(s, result.headers, timeout) + result.body = parseBody(s, result.headers, result.version, timeout) else: result.body = "" @@ -685,7 +685,8 @@ proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} = # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 proc parseBody(client: AsyncHttpClient, - headers: HttpHeaders): Future[string] {.async.} = + headers: HttpHeaders, + httpVersion: string): Future[string] {.async.} = result = "" if headers.getOrDefault"Transfer-Encoding" == "chunked": result = await parseChunks(client) @@ -707,7 +708,7 @@ proc parseBody(client: AsyncHttpClient, # -REGION- Connection: Close # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5 - if headers.getOrDefault"Connection" == "close": + if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0": var buf = "" while true: buf = await client.socket.recvFull(4000) @@ -761,7 +762,7 @@ proc parseResponse(client: AsyncHttpClient, if not fullyRead: httpError("Connection was closed before full request has been made") if getBody: - result.body = await parseBody(client, result.headers) + result.body = await parseBody(client, result.headers, result.version) else: result.body = "" diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index c4d575833..562d16c19 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -24,6 +24,7 @@ type Http200 = "200 OK", Http201 = "201 Created", Http202 = "202 Accepted", + Http203 = "203 Non-Authoritative Information", Http204 = "204 No Content", Http205 = "205 Reset Content", Http206 = "206 Partial Content", @@ -52,6 +53,13 @@ type Http416 = "416 Requested Range Not Satisfiable", Http417 = "417 Expectation Failed", Http418 = "418 I'm a teapot", + Http421 = "421 Misdirected Request", + Http422 = "422 Unprocessable Entity", + Http426 = "426 Upgrade Required", + Http428 = "428 Precondition Required", + Http429 = "429 Too Many Requests", + Http431 = "431 Request Header Fields Too Large", + Http451 = "451 Unavailable For Legal Reasons", Http500 = "500 Internal Server Error", Http501 = "501 Not Implemented", Http502 = "502 Bad Gateway", diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim new file mode 100644 index 000000000..a5d5d2c01 --- /dev/null +++ b/lib/pure/ioselectors.nim @@ -0,0 +1,255 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module allows high-level and efficient I/O multiplexing. +## +## Supported OS primitives: ``epoll``, ``kqueue``, ``poll`` and +## Windows ``select``. +## +## To use threadsafe version of this module, it needs to be compiled +## with both ``-d:threadsafe`` and ``--threads:on`` options. +## +## Supported features: files, sockets, pipes, timers, processes, signals +## and user events. +## +## Fully supported OS: MacOSX, FreeBSD, OpenBSD, NetBSD, Linux. +## +## Partially supported OS: Windows (only sockets and user events), +## Solaris (files, sockets, handles and user events). +## +## TODO: ``/dev/poll``, ``event ports`` and filesystem events. + +import os + +const hasThreadSupport = compileOption("threads") and defined(threadsafe) + +const supportedPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(linux) + +const bsdPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) + + +when defined(nimdoc): + type + Selector*[T] = ref object + ## An object which holds descriptors to be checked for read/write status + + Event* {.pure.} = enum + ## An enum which hold event types + Read, ## Descriptor is available for read + Write, ## Descriptor is available for write + Timer, ## Timer descriptor is completed + Signal, ## Signal is raised + Process, ## Process is finished + Vnode, ## Currently not supported + User, ## User event is raised + Error ## Error happens while waiting, for descriptor + + ReadyKey*[T] = object + ## An object which holds result for descriptor + fd* : int ## file/socket descriptor + events*: set[Event] ## set of events + data*: T ## application-defined data + + SelectEvent* = object + ## An object which holds user defined event + + proc newSelector*[T](): Selector[T] = + ## Creates a new selector + + proc close*[T](s: Selector[T]) = + ## Closes selector + + proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], + data: T) = + ## Registers file/socket descriptor ``fd`` to selector ``s`` + ## with events set in ``events``. The ``data`` is application-defined + ## data, which to be passed when event happens. + + proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + ## Update file/socket descriptor ``fd``, registered in selector + ## ``s`` with new events set ``event``. + + proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + ## Registers timer notification with ``timeout`` in milliseconds + ## to selector ``s``. + ## If ``oneshot`` is ``true`` timer will be notified only once. + ## Set ``oneshot`` to ``false`` if your want periodic notifications. + ## The ``data`` is application-defined data, which to be passed, when + ## time limit expired. + + proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + ## Registers Unix signal notification with ``signal`` to selector + ## ``s``. The ``data`` is application-defined data, which to be + ## passed, when signal raises. + ## + ## This function is not supported for ``Windows``. + + proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + ## Registers process id (pid) notification when process has + ## exited to selector ``s``. + ## The ``data`` is application-defined data, which to be passed, when + ## process with ``pid`` has exited. + + proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + ## Registers selector event ``ev`` to selector ``s``. + ## ``data`` application-defined data, which to be passed, when + ## ``ev`` happens. + + proc newSelectEvent*(): SelectEvent = + ## Creates new event ``SelectEvent``. + + proc setEvent*(ev: SelectEvent) = + ## Trigger event ``ev``. + + proc close*(ev: SelectEvent) = + ## Closes selector event ``ev``. + + proc unregister*[T](s: Selector[T], ev: SelectEvent) = + ## Unregisters event ``ev`` from selector ``s``. + + proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = + ## Unregisters file/socket descriptor ``fd`` from selector ``s``. + + proc flush*[T](s: Selector[T]) = + ## Flushes all changes was made to kernel pool/queue. + ## This function is usefull only for BSD and MacOS, because + ## kqueue supports bulk changes to be made. + ## On Linux/Windows and other Posix compatible operation systems, + ## ``flush`` is alias for `discard`. + + proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + ## Process call waiting for events registered in selector ``s``. + ## The ``timeout`` argument specifies the minimum number of milliseconds + ## the function will be blocked, if no events are not ready. Specifying a + ## timeout of ``-1`` causes function to block indefinitely. + ## All available events will be stored in ``results`` array. + ## + ## Function returns number of triggered events. + + proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + ## Process call waiting for events registered in selector ``s``. + ## The ``timeout`` argument specifies the minimum number of milliseconds + ## the function will be blocked, if no events are not ready. Specifying a + ## timeout of -1 causes function to block indefinitely. + ## + ## Function returns sequence of triggered events. + + template isEmpty*[T](s: Selector[T]): bool = + ## Returns ``true``, if there no registered events or descriptors + ## in selector. + + template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + ## retrieves the application-data assigned with descriptor ``fd`` + ## to ``value``. This ``value`` can be modified in the scope of + ## the ``withData`` call. + ## + ## .. code-block:: nim + ## + ## s.withData(fd, value) do: + ## # block is executed only if ``fd`` registered in selector ``s`` + ## value.uid = 1000 + ## + + template withData*[T](s: Selector[T], fd: SocketHandle, value, + body1, body2: untyped) = + ## retrieves the application-data assigned with descriptor ``fd`` + ## to ``value``. This ``value`` can be modified in the scope of + ## the ``withData`` call. + ## + ## .. code-block:: nim + ## + ## s.withData(fd, value) do: + ## # block is executed only if ``fd`` registered in selector ``s``. + ## value.uid = 1000 + ## do: + ## # block is executed if ``fd`` not registered in selector ``s``. + ## raise + ## + +else: + when hasThreadSupport: + import locks + + type + SharedArray {.unchecked.}[T] = array[0..100, T] + + proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) + + proc deallocSharedArray[T](sa: ptr SharedArray[T]) = + deallocShared(cast[pointer](sa)) + type + Event* {.pure.} = enum + Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot + + ReadyKey*[T] = object + fd* : int + events*: set[Event] + data*: T + + SelectorKey[T] = object + ident: int + events: set[Event] + param: int + key: ReadyKey[T] + + when not defined(windows): + import posix + proc setNonBlocking(fd: cint) {.inline.} = + var x = fcntl(fd, F_GETFL, 0) + if x == -1: + raiseOSError(osLastError()) + else: + var mode = x or O_NONBLOCK + if fcntl(fd, F_SETFL, mode) == -1: + raiseOSError(osLastError()) + + template setKey(s, pident, pkeyfd, pevents, pparam, pdata) = + var skey = addr(s.fds[pident]) + skey.ident = pident + skey.events = pevents + skey.param = pparam + skey.key.fd = pkeyfd + skey.key.data = pdata + + when supportedPlatform: + template blockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + else: + if posix.sigprocmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + + template unblockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + else: + if posix.sigprocmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + + when defined(linux): + include ioselects/ioselectors_epoll + elif bsdPlatform: + include ioselects/ioselectors_kqueue + elif defined(windows): + include ioselects/ioselectors_select + elif defined(solaris): + include ioselects/ioselectors_poll # need to replace it with event ports + else: + include ioselects/ioselectors_poll diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim new file mode 100644 index 000000000..92b2cdc07 --- /dev/null +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -0,0 +1,461 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Linux epoll(). + +import posix, times + +# Maximum number of events that can be returned +const MAX_EPOLL_RESULT_EVENTS = 64 + +type + SignalFdInfo* {.importc: "struct signalfd_siginfo", + header: "<sys/signalfd.h>", pure, final.} = object + ssi_signo*: uint32 + ssi_errno*: int32 + ssi_code*: int32 + ssi_pid*: uint32 + ssi_uid*: uint32 + ssi_fd*: int32 + ssi_tid*: uint32 + ssi_band*: uint32 + ssi_overrun*: uint32 + ssi_trapno*: uint32 + ssi_status*: int32 + ssi_int*: int32 + ssi_ptr*: uint64 + ssi_utime*: uint64 + ssi_stime*: uint64 + ssi_addr*: uint64 + pad* {.importc: "__pad".}: array[0..47, uint8] + + eventFdData {.importc: "eventfd_t", + header: "<sys/eventfd.h>", pure, final.} = uint64 + epoll_data {.importc: "union epoll_data", header: "<sys/epoll.h>", + pure, final.} = object + u64 {.importc: "u64".}: uint64 + epoll_event {.importc: "struct epoll_event", + header: "<sys/epoll.h>", pure, final.} = object + events: uint32 # Epoll events + data: epoll_data # User data variable + +const + EPOLL_CTL_ADD = 1 # Add a file descriptor to the interface. + EPOLL_CTL_DEL = 2 # Remove a file descriptor from the interface. + EPOLL_CTL_MOD = 3 # Change file descriptor epoll_event structure. + EPOLLIN = 0x00000001 + EPOLLOUT = 0x00000004 + EPOLLERR = 0x00000008 + EPOLLHUP = 0x00000010 + EPOLLRDHUP = 0x00002000 + EPOLLONESHOT = 1 shl 30 + +proc epoll_create(size: cint): cint + {.importc: "epoll_create", header: "<sys/epoll.h>".} +proc epoll_ctl(epfd: cint; op: cint; fd: cint; event: ptr epoll_event): cint + {.importc: "epoll_ctl", header: "<sys/epoll.h>".} +proc epoll_wait(epfd: cint; events: ptr epoll_event; maxevents: cint; + timeout: cint): cint + {.importc: "epoll_wait", header: "<sys/epoll.h>".} +proc timerfd_create(clock_id: ClockId, flags: cint): cint + {.cdecl, importc: "timerfd_create", header: "<sys/timerfd.h>".} +proc timerfd_settime(ufd: cint, flags: cint, + utmr: var Itimerspec, otmr: var Itimerspec): cint + {.cdecl, importc: "timerfd_settime", header: "<sys/timerfd.h>".} +proc signalfd(fd: cint, mask: var Sigset, flags: cint): cint + {.cdecl, importc: "signalfd", header: "<sys/signalfd.h>".} +proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "<sys/eventfd.h>".} +proc ulimit(cmd: cint): clong + {.importc: "ulimit", header: "<ulimit.h>", varargs.} + +when hasThreadSupport: + type + SelectorImpl[T] = object + epollFD : cint + maxFD : int + fds: ptr SharedArray[SelectorKey[T]] + count: int + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + epollFD : cint + maxFD : int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] +type + SelectEventImpl = object + efd: cint + SelectEvent* = ptr SelectEventImpl + +proc newSelector*[T](): Selector[T] = + var maxFD = int(ulimit(4, 0)) + doAssert(maxFD > 0) + + var epollFD = epoll_create(MAX_EPOLL_RESULT_EVENTS) + if epollFD < 0: + raiseOsError(osLastError()) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.epollFD = epollFD + result.maxFD = maxFD + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + else: + result = Selector[T]() + result.epollFD = epollFD + result.maxFD = maxFD + result.fds = newSeq[SelectorKey[T]](maxFD) + +proc close*[T](s: Selector[T]) = + if posix.close(s.epollFD) != 0: + raiseOSError(osLastError()) + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +proc newSelectEvent*(): SelectEvent = + let fdci = eventfd(0, 0) + if fdci == -1: + raiseOSError(osLastError()) + setNonBlocking(fdci) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.efd = fdci + +proc setEvent*(ev: SelectEvent) = + var data : uint64 = 1 + if posix.write(ev.efd, addr data, sizeof(uint64)) == -1: + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(ev.efd) + deallocShared(cast[pointer](ev)) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: + var epv = epoll_event(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + var epv = epoll_event(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + + if pkey.events == {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + else: + if events != {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + else: + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + + if pkey.events != {}: + if pkey.events * {Event.Read, Event.Write} != {}: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + elif Event.Timer in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + discard posix.close(fdi.cint) + dec(s.count) + elif Event.Signal in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(s.fds[fdi].param)) + unblockSignals(nmask, omask) + discard posix.close(fdi.cint) + dec(s.count) + elif Event.Process in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, SIGCHLD) + unblockSignals(nmask, omask) + discard posix.close(fdi.cint) + dec(s.count) + pkey.ident = 0 + pkey.events = {} + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.efd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + var + new_ts: Itimerspec + old_ts: Itimerspec + let fdi = timerfd_create(CLOCK_MONOTONIC, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var events = {Event.Timer} + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + if oneshot: + new_ts.it_interval.tv_sec = 0.Time + new_ts.it_interval.tv_nsec = 0 + new_ts.it_value.tv_sec = (timeout div 1_000).Time + new_ts.it_value.tv_nsec = (timeout %% 1_000) * 1_000_000 + incl(events, Event.Oneshot) + epv.events = epv.events or EPOLLONESHOT + else: + new_ts.it_interval.tv_sec = (timeout div 1000).Time + new_ts.it_interval.tv_nsec = (timeout %% 1_000) * 1_000_000 + new_ts.it_value.tv_sec = new_ts.it_interval.tv_sec + new_ts.it_value.tv_nsec = new_ts.it_interval.tv_nsec + + if timerfd_settime(fdi.cint, cint(0), new_ts, old_ts) == -1: + raiseOSError(osLastError()) + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, fdi, events, 0, data) + inc(s.count) + result = fdi + +proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, signal, {Event.Signal}, signal, data) + inc(s.count) + result = fdi + +proc registerProcess*[T](s: Selector, pid: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, posix.SIGCHLD) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + epv.events = EPOLLIN or EPOLLRDHUP + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, pid, {Event.Process, Event.Oneshot}, pid, data) + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = int(ev.efd) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, {Event.User}, 0, data) + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = ev.efd.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, ev.efd, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + +proc flush*[T](s: Selector[T]) = + discard + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var + resTable: array[MAX_EPOLL_RESULT_EVENTS, epoll_event] + maxres = MAX_EPOLL_RESULT_EVENTS + events: set[Event] = {} + i, k: int + + if maxres > len(results): + maxres = len(results) + + let count = epoll_wait(s.epollFD, addr(resTable[0]), maxres.cint, + timeout.cint) + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + i = 0 + k = 0 + while i < count: + let fdi = int(resTable[i].data.u64) + let pevents = resTable[i].events + var skey = addr(s.fds[fdi]) + doAssert(skey.ident != 0) + events = {} + + if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0: + events.incl(Event.Error) + if (pevents and EPOLLOUT) != 0: + events.incl(Event.Write) + if (pevents and EPOLLIN) != 0: + if Event.Read in skey.events: + events.incl(Event.Read) + elif Event.Timer in skey.events: + var data: uint64 = 0 + if posix.read(fdi.cint, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + events = {Event.Timer} + elif Event.Signal in skey.events: + var data = SignalFdInfo() + if posix.read(fdi.cint, addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseOsError(osLastError()) + events = {Event.Signal} + elif Event.Process in skey.events: + var data = SignalFdInfo() + if posix.read(fdi.cint, addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseOsError(osLastError()) + if cast[int](data.ssi_pid) == skey.param: + events = {Event.Process} + else: + inc(i) + continue + elif Event.User in skey.events: + var data: uint = 0 + if posix.read(fdi.cint, addr data, sizeof(uint)) != sizeof(uint): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + inc(i) + continue + else: + raiseOSError(err) + events = {Event.User} + + skey.key.events = events + results[k] = skey.key + inc(k) + + if Event.Oneshot in skey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + discard posix.close(fdi.cint) + skey.ident = 0 + skey.events = {} + dec(s.count) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_EPOLL_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim new file mode 100644 index 000000000..29a201863 --- /dev/null +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -0,0 +1,439 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements BSD kqueue(). + +import posix, times, kqueue + +const + # Maximum number of cached changes. + MAX_KQUEUE_CHANGE_EVENTS = 64 + # Maximum number of events that can be returned. + MAX_KQUEUE_RESULT_EVENTS = 64 + +when defined(macosx) or defined(freebsd): + when defined(macosx): + const MAX_DESCRIPTORS_ID = 29 # KERN_MAXFILESPERPROC (MacOS) + else: + const MAX_DESCRIPTORS_ID = 27 # KERN_MAXFILESPERPROC (FreeBSD) + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, + newp: pointer, newplen: int): cint + {.importc: "sysctl",header: """#include <sys/types.h> + #include <sys/sysctl.h>"""} +elif defined(netbsd) or defined(openbsd): + # OpenBSD and NetBSD don't have KERN_MAXFILESPERPROC, so we are using + # KERN_MAXFILES, because KERN_MAXFILES is always bigger, + # than KERN_MAXFILESPERPROC. + const MAX_DESCRIPTORS_ID = 7 # KERN_MAXFILES + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, + newp: pointer, newplen: int): cint + {.importc: "sysctl",header: """#include <sys/param.h> + #include <sys/sysctl.h>"""} + +when hasThreadSupport: + type + SelectorImpl[T] = object + kqFD : cint + maxFD : int + changesTable: array[MAX_KQUEUE_CHANGE_EVENTS, KEvent] + changesCount: int + fds: ptr SharedArray[SelectorKey[T]] + count: int + changesLock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + kqFD : cint + maxFD : int + changesTable: array[MAX_KQUEUE_CHANGE_EVENTS, KEvent] + changesCount: int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint +# SelectEvent is declared as `ptr` to be placed in `shared memory`, +# so you can share one SelectEvent handle between threads. +type SelectEvent* = ptr SelectEventImpl + +proc newSelector*[T](): Selector[T] = + var maxFD = 0.cint + var size = sizeof(cint) + var namearr = [1.cint, MAX_DESCRIPTORS_ID.cint] + # Obtain maximum number of file descriptors for process + if sysctl(addr(namearr[0]), 2, cast[pointer](addr maxFD), addr size, + nil, 0) != 0: + raiseOsError(osLastError()) + + var kqFD = kqueue() + if kqFD < 0: + raiseOsError(osLastError()) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.kqFD = kqFD + result.maxFD = maxFD.int + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + initLock(result.changesLock) + else: + result = Selector[T]() + result.kqFD = kqFD + result.maxFD = maxFD.int + result.fds = newSeq[SelectorKey[T]](maxFD) + +proc close*[T](s: Selector[T]) = + if posix.close(s.kqFD) != 0: + raiseOSError(osLastError()) + when hasThreadSupport: + deinitLock(s.changesLock) + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + +proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rfd)) + discard posix.close(cint(ev.wfd)) + deallocShared(cast[pointer](ev)) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +when hasThreadSupport: + template withChangeLock[T](s: Selector[T], body: untyped) = + acquire(s.changesLock) + {.locks: [s.changesLock].}: + try: + body + finally: + release(s.changesLock) +else: + template withChangeLock(s, body: untyped) = + body + +template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, + nflags: cushort, nfflags: cuint, ndata: int, + nudata: pointer) = + mixin withChangeLock + s.withChangeLock(): + s.changesTable[s.changesCount] = KEvent(ident: nident, + filter: nfilter, flags: nflags, + fflags: nfflags, data: ndata, + udata: nudata) + inc(s.changesCount) + if s.changesCount == MAX_KQUEUE_CHANGE_EVENTS: + if kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + nil, 0, nil) == -1: + raiseOSError(osLastError()) + s.changesCount = 0 + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: + if Event.Read in events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if Event.Write in events: + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + pkey.events = events + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + let events = if oneshot: {Event.Timer, Event.Oneshot} else: {Event.Timer} + let flags: cushort = if oneshot: EV_ONESHOT or EV_ADD else: EV_ADD + + s.setKey(fdi, fdi, events, 0, data) + # EVFILT_TIMER on Open/Net(BSD) has granularity of only milliseconds, + # but MacOS and FreeBSD allow use `0` as `fflags` to use milliseconds + # too + modifyKQueue(s, fdi.uint, EVFILT_TIMER, flags, 0, cint(timeout), nil) + inc(s.count) + result = fdi + +proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + s.setKey(fdi, signal, {Event.Signal}, signal, data) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + # to be compatible with linux semantic we need to "eat" signals + posix.signal(cint(signal), SIG_IGN) + modifyKQueue(s, signal.uint, EVFILT_SIGNAL, EV_ADD, 0, 0, + cast[pointer](fdi)) + inc(s.count) + result = fdi + +proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var kflags: cushort = EV_ONESHOT or EV_ADD + setKey(s, fdi, pid, {Event.Process, Event.Oneshot}, pid, data) + modifyKQueue(s, pid.uint, EVFILT_PROC, kflags, NOTE_EXIT, 0, + cast[pointer](fdi)) + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = ev.rfd.int + doAssert(s.fds[fdi].ident == 0) + setKey(s, fdi, fdi, {Event.User}, 0, data) + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + + if pkey.events != {}: + if pkey.events * {Event.Read, Event.Write} != {}: + if Event.Read in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if Event.Write in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Timer in pkey.events: + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_TIMER, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Signal in pkey.events: + var nmask, omask: Sigset + var signal = cint(pkey.param) + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, signal) + unblockSignals(nmask, omask) + posix.signal(signal, SIG_DFL) + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_SIGNAL, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Process in pkey.events: + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_PROC, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.User in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + pkey.ident = 0 + pkey.events = {} + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + +proc flush*[T](s: Selector[T]) = + s.withChangeLock(): + var tv = Timespec() + if kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + nil, 0, addr tv) == -1: + raiseOSError(osLastError()) + s.changesCount = 0 + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var + tv: Timespec + resTable: array[MAX_KQUEUE_RESULT_EVENTS, KEvent] + ptv = addr tv + maxres = MAX_KQUEUE_RESULT_EVENTS + + if timeout != -1: + if timeout >= 1000: + tv.tv_sec = (timeout div 1_000).Time + tv.tv_nsec = (timeout %% 1_000) * 1_000_000 + else: + tv.tv_sec = 0.Time + tv.tv_nsec = timeout * 1_000_000 + else: + ptv = nil + + if maxres > len(results): + maxres = len(results) + + var count = 0 + s.withChangeLock(): + count = kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + addr(resTable[0]), cint(maxres), ptv) + s.changesCount = 0 + + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 + var pkey: ptr SelectorKey[T] + while i < count: + let kevent = addr(resTable[i]) + if (kevent.flags and EV_ERROR) == 0: + case kevent.filter: + of EVFILT_READ: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Read} + if Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(kevent.ident.cint, addr data, + sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + # someone already consumed event data + inc(i) + continue + else: + raiseOSError(osLastError()) + pkey.key.events = {Event.User} + of EVFILT_WRITE: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Write} + of EVFILT_TIMER: + pkey = addr(s.fds[kevent.ident.int]) + if Event.Oneshot in pkey.events: + if posix.close(cint(pkey.ident)) == -1: + raiseOSError(osLastError()) + pkey.ident = 0 + pkey.events = {} + dec(s.count) + pkey.key.events = {Event.Timer} + of EVFILT_VNODE: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Vnode} + of EVFILT_SIGNAL: + pkey = addr(s.fds[cast[int](kevent.udata)]) + pkey.key.events = {Event.Signal} + of EVFILT_PROC: + pkey = addr(s.fds[cast[int](kevent.udata)]) + if posix.close(cint(pkey.ident)) == -1: + raiseOSError(osLastError()) + pkey.ident = 0 + pkey.events = {} + dec(s.count) + pkey.key.events = {Event.Process} + else: + raise newException(ValueError, "Unsupported kqueue filter in queue") + + if (kevent.flags and EV_EOF) != 0: + pkey.key.events.incl(Event.Error) + + results[k] = pkey.key + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_KQUEUE_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim new file mode 100644 index 000000000..d2a0a1273 --- /dev/null +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -0,0 +1,295 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix poll(). + +import posix, times + +# Maximum number of events that can be returned +const MAX_POLL_RESULT_EVENTS = 64 + +when hasThreadSupport: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: ptr SharedArray[SelectorKey[T]] + pollfds: ptr SharedArray[TPollFd] + count: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: seq[SelectorKey[T]] + pollfds: seq[TPollFd] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint + SelectEvent* = ptr SelectEventImpl + +var RLIMIT_NOFILE {.importc: "RLIMIT_NOFILE", + header: "<sys/resource.h>".}: cint +type + rlimit {.importc: "struct rlimit", + header: "<sys/resource.h>", pure, final.} = object + rlim_cur: int + rlim_max: int +proc getrlimit(resource: cint, rlp: var rlimit): cint + {.importc: "getrlimit",header: "<sys/resource.h>".} + +when hasThreadSupport: + template withPollLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withPollLock(s, body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + var a = rlimit() + if getrlimit(RLIMIT_NOFILE, a) != 0: + raiseOsError(osLastError()) + var maxFD = int(a.rlim_max) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.maxFD = maxFD + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + result.pollfds = allocSharedArray[TPollFd](maxFD) + initLock(result.lock) + else: + result = Selector[T]() + result.maxFD = maxFD + result.fds = newSeq[SelectorKey[T]](maxFD) + result.pollfds = newSeq[TPollFd](maxFD) + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deinitLock(s.lock) + deallocSharedArray(s.fds) + deallocSharedArray(s.pollfds) + deallocShared(cast[pointer](s)) + +template pollAdd[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + s.pollfds[s.pollcnt].fd = cint(sock) + s.pollfds[s.pollcnt].events = pollev + inc(s.count) + inc(s.pollcnt) + +template pollUpdate[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var i = 0 + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + s.pollfds[i].events = pollev + break + inc(i) + + if i == s.pollcnt: + raise newException(ValueError, "Descriptor is not registered in queue") + +template pollRemove[T](s: Selector[T], sock: cint) = + withPollLock(s): + var i = 0 + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + if i == s.pollcnt - 1: + s.pollfds[i].fd = 0 + s.pollfds[i].events = 0 + s.pollfds[i].revents = 0 + else: + while i < (s.pollcnt - 1): + s.pollfds[i].fd = s.pollfds[i + 1].fd + s.pollfds[i].events = s.pollfds[i + 1].events + inc(i) + break + inc(i) + dec(s.pollcnt) + dec(s.count) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + var fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: s.pollAdd(fdi.cint, events) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if pkey.events == {}: + s.pollAdd(fd.cint, events) + else: + if events != {}: + s.pollUpdate(fd.cint, events) + else: + s.pollRemove(fd.cint) + pkey.events = events + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + var fdi = int(ev.rfd) + doAssert(s.fds[fdi].ident == 0) + var events = {Event.User} + setKey(s, fdi, fdi, events, 0, data) + events.incl(Event.Read) + s.pollAdd(fdi.cint, events) + +proc flush*[T](s: Selector[T]) = discard + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + pkey.ident = 0 + pkey.events = {} + s.pollRemove(fdi.cint) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + s.pollRemove(fdi.cint) + +proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + +proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rfd)) + discard posix.close(cint(ev.wfd)) + deallocShared(cast[pointer](ev)) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var maxres = MAX_POLL_RESULT_EVENTS + if maxres > len(results): + maxres = len(results) + + s.withPollLock(): + let count = posix.poll(addr(s.pollfds[0]), Tnfds(s.pollcnt), timeout) + if count < 0: + result = 0 + let err = osLastError() + if err.cint == EINTR: + discard + else: + raiseOSError(osLastError()) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 + var rindex = 0 + while (i < s.pollcnt) and (k < count) and (rindex < maxres): + let revents = s.pollfds[i].revents + if revents != 0: + let fd = s.pollfds[i].fd + var skey = addr(s.fds[fd]) + skey.key.events = {} + + if (revents and POLLIN) != 0: + skey.key.events.incl(Event.Read) + if Event.User in skey.events: + var data: uint64 = 0 + if posix.read(fd, addr data, sizeof(int)) != sizeof(int): + let err = osLastError() + if err != OSErrorCode(EAGAIN): + raiseOSError(osLastError()) + else: + # someone already consumed event data + inc(i) + continue + skey.key.events = {Event.User} + if (revents and POLLOUT) != 0: + skey.key.events.incl(Event.Write) + if (revents and POLLERR) != 0 or (revents and POLLHUP) != 0 or + (revents and POLLNVAL) != 0: + skey.key.events.incl(Event.Error) + results[rindex] = skey.key + s.pollfds[i].revents = 0 + inc(rindex) + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_POLL_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim new file mode 100644 index 000000000..f8099f9a0 --- /dev/null +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -0,0 +1,416 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix and Windows select(). + +import times, nativesockets + +when defined(windows): + import winlean + when defined(gcc): + {.passL: "-lws2_32".} + elif defined(vcc): + {.passL: "ws2_32.lib".} + const platformHeaders = """#include <winsock2.h> + #include <windows.h>""" + const EAGAIN = WSAEWOULDBLOCK +else: + const platformHeaders = """#include <sys/select.h> + #include <sys/time.h> + #include <sys/types.h> + #include <unistd.h>""" +type + Fdset {.importc: "fd_set", header: platformHeaders, pure, final.} = object +var + FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint + +proc IOFD_SET(fd: SocketHandle, fdset: ptr Fdset) + {.cdecl, importc: "FD_SET", header: platformHeaders, inline.} +proc IOFD_CLR(fd: SocketHandle, fdset: ptr Fdset) + {.cdecl, importc: "FD_CLR", header: platformHeaders, inline.} +proc IOFD_ZERO(fdset: ptr Fdset) + {.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.} + +when defined(windows): + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + {.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + timeout: ptr Timeval): cint + {.stdcall, importc: "select", header: platformHeaders.} +else: + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + {.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + timeout: ptr Timeval): cint + {.cdecl, importc: "select", header: platformHeaders.} + +when hasThreadSupport: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: ptr SharedArray[SelectorKey[T]] + count: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rsock: SocketHandle + wsock: SocketHandle + SelectEvent* = ptr SelectEventImpl + +when hasThreadSupport: + template withSelectLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withSelectLock[T](s: Selector[T], body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.fds = allocSharedArray[SelectorKey[T]](FD_SETSIZE) + initLock result.lock + else: + result = Selector[T]() + result.fds = newSeq[SelectorKey[T]](FD_SETSIZE) + + IOFD_ZERO(addr result.rSet) + IOFD_ZERO(addr result.wSet) + IOFD_ZERO(addr result.eSet) + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +when defined(windows): + proc newSelectEvent*(): SelectEvent = + var ssock = newNativeSocket() + var wsock = newNativeSocket() + var rsock: SocketHandle = INVALID_SOCKET + var saddr = Sockaddr_in() + + saddr.sin_family = winlean.AF_INET + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(ssock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) < 0'i32: + raiseOSError(osLastError()) + + if winlean.listen(ssock, 1) == -1: + raiseOSError(osLastError()) + + var namelen = sizeof(saddr).SockLen + if getsockname(ssock, cast[ptr SockAddr](addr(saddr)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + + saddr.sin_addr.s_addr = 0x0100007F + if winlean.connect(wsock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) == -1: + raiseOSError(osLastError()) + namelen = sizeof(saddr).SockLen + rsock = winlean.accept(ssock, cast[ptr SockAddr](addr(saddr)), + cast[ptr SockLen](addr(namelen))) + if rsock == SocketHandle(-1): + raiseOSError(osLastError()) + + if winlean.closesocket(ssock) == -1: + raiseOSError(osLastError()) + + var mode = clong(1) + if ioctlsocket(rsock, FIONBIO, addr(mode)) == -1: + raiseOSError(osLastError()) + mode = clong(1) + if ioctlsocket(wsock, FIONBIO, addr(mode)) == -1: + raiseOSError(osLastError()) + + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = rsock + result.wsock = wsock + + proc setEvent*(ev: SelectEvent) = + var data: int = 1 + if winlean.send(ev.wsock, cast[pointer](addr data), + cint(sizeof(int)), 0) != sizeof(int): + raiseOSError(osLastError()) + + proc close*(ev: SelectEvent) = + discard winlean.closesocket(ev.rsock) + discard winlean.closesocket(ev.wsock) + deallocShared(cast[pointer](ev)) + +else: + proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = SocketHandle(fds[0]) + result.wsock = SocketHandle(fds[1]) + + proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(cint(ev.wsock), addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + + proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rsock)) + discard posix.close(cint(ev.wsock)) + deallocShared(cast[pointer](ev)) + +proc setKey[T](s: Selector[T], fd: SocketHandle, events: set[Event], data: T) = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == 0: + var pkey = addr(s.fds[i]) + pkey.ident = fdi + pkey.events = events + pkey.key.fd = fd.int + pkey.key.events = {} + pkey.key.data = data + break + inc(i) + if i == FD_SETSIZE: + raise newException(ValueError, "Maximum numbers of fds exceeded") + +proc getKey[T](s: Selector[T], fd: SocketHandle): ptr SelectorKey[T] = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + result = addr(s.fds[i]) + break + inc(i) + doAssert(i < FD_SETSIZE, "Descriptor not registered in queue") + +proc delKey[T](s: Selector[T], fd: SocketHandle) = + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fd.int: + s.fds[i].ident = 0 + s.fds[i].events = {} + break + inc(i) + doAssert(i < FD_SETSIZE, "Descriptor not registered in queue") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + when not defined(windows): + let fdi = int(fd) + s.withSelectLock(): + s.setKey(fd, events, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + if Event.Read in events: + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if Event.Write in events: + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + when not defined(windows): + let fdi = int(ev.rsock) + s.withSelectLock(): + s.setKey(ev.rsock, {Event.User}, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + IOFD_SET(ev.rsock, addr s.rSet) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + s.withSelectLock(): + var pkey = s.getKey(fd) + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: SocketHandle) = + s.withSelectLock(): + var pkey = s.getKey(fd) + if Event.Read in pkey.events: + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if Event.Write in pkey.events: + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + s.delKey(fd) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fd = ev.rsock + s.withSelectLock(): + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + s.delKey(fd) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var tv = Timeval() + var ptv = addr tv + var rset, wset, eset: FdSet + + if timeout != -1: + tv.tv_sec = timeout.int32 div 1_000 + tv.tv_usec = (timeout.int32 %% 1_000) * 1_000 + else: + ptv = nil + + s.withSelectLock(): + rset = s.rSet + wset = s.wSet + eset = s.eSet + + var count = ioselect(cint(s.maxFD) + 1, addr(rset), addr(wset), + addr(eset), ptv) + if count < 0: + result = 0 + when defined(windows): + raiseOSError(osLastError()) + else: + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + var rindex = 0 + var i = 0 + var k = 0 + + while (i < FD_SETSIZE) and (k < count): + if s.fds[i].ident != 0: + var flag = false + var pkey = addr(s.fds[i]) + pkey.key.events = {} + let fd = SocketHandle(pkey.ident) + if IOFD_ISSET(fd, addr rset) != 0: + if Event.User in pkey.events: + var data: uint64 = 0 + if recv(fd, cast[pointer](addr(data)), + sizeof(uint64).cint, 0) != sizeof(uint64): + let err = osLastError() + if cint(err) != EAGAIN: + raiseOSError(err) + else: + inc(i) + inc(k) + continue + else: + flag = true + pkey.key.events = {Event.User} + else: + flag = true + pkey.key.events = {Event.Read} + if IOFD_ISSET(fd, addr wset) != 0: + pkey.key.events.incl(Event.Write) + if IOFD_ISSET(fd, addr eset) != 0: + pkey.key.events.incl(Event.Error) + flag = true + if flag: + results[rindex] = pkey.key + inc(rindex) + inc(k) + inc(i) + result = rindex + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](FD_SETSIZE) + var count = selectInto(s, timeout, result) + result.setLen(count) + +proc flush*[T](s: Selector[T]) = discard + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +when hasThreadSupport: + template withSelectLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withSelectLock[T](s: Selector[T], body: untyped) = + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin withSelectLock + s.withSelectLock(): + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].key.data) + break + inc(i) + if i != FD_SETSIZE: + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body1, body2: untyped) = + mixin withSelectLock + s.withSelectLock(): + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].key.data) + break + inc(i) + if i != FD_SETSIZE: + body1 + else: + body2 diff --git a/lib/pure/json.nim b/lib/pure/json.nim index e99a351a1..19947fbc2 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -126,7 +126,7 @@ type TJsonParser: JsonParser, TTokKind: TokKind].} const - errorMessages: array [JsonError, string] = [ + errorMessages: array[JsonError, string] = [ "no error", "invalid token", "string expected", @@ -139,7 +139,7 @@ const "EOF expected", "expression expected" ] - tokToStr: array [TokKind, string] = [ + tokToStr: array[TokKind, string] = [ "invalid token", "EOF", "string literal", @@ -712,6 +712,7 @@ proc `%`*(b: bool): JsonNode = proc `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode = ## Generic constructor for JSON data. Creates a new `JObject JsonNode` + if keyvals.len == 0: return newJArray() result = newJObject() for key, val in items(keyVals): result.fields[key] = val @@ -848,6 +849,16 @@ proc hasKey*(node: JsonNode, key: string): bool = assert(node.kind == JObject) result = node.fields.hasKey(key) +proc contains*(node: JsonNode, key: string): bool = + ## Checks if `key` exists in `node`. + assert(node.kind == JObject) + node.fields.hasKey(key) + +proc contains*(node: JsonNode, val: JsonNode): bool = + ## Checks if `val` exists in array `node`. + assert(node.kind == JArray) + find(node.elems, val) >= 0 + proc existsKey*(node: JsonNode, key: string): bool {.deprecated.} = node.hasKey(key) ## Deprecated for `hasKey` @@ -1116,7 +1127,7 @@ proc parseJson(p: var JsonParser): JsonNode = discard getTok(p) while p.tok != tkCurlyRi: if p.tok != tkString: - raiseParseErr(p, "string literal as key expected") + raiseParseErr(p, "string literal as key") var key = p.a discard getTok(p) eat(p, tkColon) diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index 8a0d2fedd..6a27e6af8 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -61,7 +61,7 @@ type lvlNone ## no levels active const - LevelNames*: array [Level, string] = [ + LevelNames*: array[Level, string] = [ "DEBUG", "DEBUG", "INFO", "NOTICE", "WARN", "ERROR", "FATAL", "NONE" ] diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 2b903bedb..c088e3d7d 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -138,11 +138,6 @@ when not defined(JS): proc exp*(x: float64): float64 {.importc: "exp", header: "<math.h>".} ## Computes the exponential function of `x` (pow(E, x)) - proc round0(x: float32): float32 {.importc: "roundf", header: "<math.h>".} - proc round0(x: float64): float64 {.importc: "round", header: "<math.h>".} - ## Converts a float to an int by rounding. Used internally by the round - ## function when the specified number of places is 0. - proc arccos*(x: float32): float32 {.importc: "acosf", header: "<math.h>".} proc arccos*(x: float64): float64 {.importc: "acos", header: "<math.h>".} ## Computes the arc cosine of `x` @@ -224,6 +219,17 @@ when not defined(JS): ## ## .. code-block:: nim ## echo ceil(-2.1) ## -2.0 + + when defined(windows) and defined(vcc): + proc round0[T: float32|float64](x: T): T = + ## Windows compilers prior to MSVC 2012 do not implement 'round', + ## 'roundl' or 'roundf'. + result = if x < 0.0: ceil(x - T(0.5)) else: floor(x + T(0.5)) + else: + proc round0(x: float32): float32 {.importc: "roundf", header: "<math.h>".} + proc round0(x: float64): float64 {.importc: "round", header: "<math.h>".} + ## Rounds a float to zero decimal places. Used internally by the round + ## function when the specified number of places is 0. proc fmod*(x, y: float32): float32 {.importc: "fmodf", header: "<math.h>".} proc fmod*(x, y: float64): float64 {.importc: "fmod", header: "<math.h>".} diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index afa343086..36b597767 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -1,3 +1,12 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + type MersenneTwister* = object mt: array[0..623, uint32] diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 15828ff35..4526afa49 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -326,8 +326,13 @@ proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = cint(AF_INET)) if s == nil: raiseOSError(osLastError()) else: - var s = posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, - cint(posix.AF_INET)) + var s = + when defined(android4): + posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, + cint(posix.AF_INET)) + else: + posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, + cint(posix.AF_INET)) if s == nil: raiseOSError(osLastError(), $hstrerror(h_errno)) diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 5a7deaab0..956c35b2f 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -27,7 +27,7 @@ const tickCountCorrection = 50_000 when not declared(system.StackTrace): - type StackTrace = array [0..20, cstring] + type StackTrace = array[0..20, cstring] {.deprecated: [TStackTrace: StackTrace].} # We use a simple hash table of bounded size to keep track of the stack traces: @@ -35,7 +35,7 @@ type ProfileEntry = object total: int st: StackTrace - ProfileData = array [0..64*1024-1, ptr ProfileEntry] + ProfileData = array[0..64*1024-1, ptr ProfileEntry] {.deprecated: [TProfileEntry: ProfileEntry, TProfileData: ProfileData].} proc `==`(a, b: StackTrace): bool = diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 2b1cf7409..9eb4cf57d 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -66,13 +66,13 @@ proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} = if err != 0'i32: when useWinUnicode: var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: result = $msgbuf if msgbuf != nil: localFree(cast[pointer](msgbuf)) else: var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200, + if formatMessageA(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: result = $msgbuf if msgbuf != nil: localFree(msgbuf) @@ -336,7 +336,7 @@ proc setCurrentDir*(newDir: string) {.inline, tags: [].} = proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", tags: [ReadDirEffect].} = - ## Returns the full path of `filename`, raises OSError in case of an error. + ## Returns the full (`absolute`:idx:) path of the file `filename`, raises OSError in case of an error. when defined(windows): const bufsize = 3072'i32 when useWinUnicode: @@ -778,12 +778,26 @@ iterator envPairs*(): tuple[key, value: TaintedString] {.tags: [ReadEnvEffect].} yield (TaintedString(substr(environment[i], 0, p-1)), TaintedString(substr(environment[i], p+1))) -iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = - ## Iterate over all the files that match the `pattern`. On POSIX this uses - ## the `glob`:idx: call. - ## - ## `pattern` is OS dependent, but at least the "\*.ext" - ## notation is supported. +# Templates for filtering directories and files +when defined(windows): + template isDir(f: WIN32_FIND_DATA): bool = + (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + template isFile(f: WIN32_FIND_DATA): bool = + not isDir(f) +else: + template isDir(f: string): bool = + dirExists(f) + template isFile(f: string): bool = + fileExists(f) + +template defaultWalkFilter(item): bool = + ## Walk filter used to return true on both + ## files and directories + true + +template walkCommon(pattern: string, filter) = + ## Common code for getting the files and directories with the + ## specified `pattern` when defined(windows): var f: WIN32_FIND_DATA @@ -792,8 +806,7 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = if res != -1: defer: findClose(res) while true: - if not skipFindData(f) and - (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) == 0'i32: + if not skipFindData(f) and filter(f): # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check # that the file extensions have the same length ... let ff = getFilename(f) @@ -815,7 +828,33 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = if res == 0: for i in 0.. f.gl_pathc - 1: assert(f.gl_pathv[i] != nil) - yield $f.gl_pathv[i] + let path = $f.gl_pathv[i] + if filter(path): + yield path + +iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect].} = + ## Iterate over all the files and directories that match the `pattern`. + ## On POSIX this uses the `glob`:idx: call. + ## + ## `pattern` is OS dependent, but at least the "\*.ext" + ## notation is supported. + walkCommon(pattern, defaultWalkFilter) + +iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = + ## Iterate over all the files that match the `pattern`. On POSIX this uses + ## the `glob`:idx: call. + ## + ## `pattern` is OS dependent, but at least the "\*.ext" + ## notation is supported. + walkCommon(pattern, isFile) + +iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect].} = + ## Iterate over all the directories that match the `pattern`. + ## On POSIX this uses the `glob`:idx: call. + ## + ## `pattern` is OS dependent, but at least the "\*.ext" + ## notation is supported. + walkCommon(pattern, isDir) type PathComponent* = enum ## Enumeration specifying a path component. @@ -1083,7 +1122,7 @@ proc parseCmdLine*(c: string): seq[string] {. while true: setLen(a, 0) # eat all delimiting whitespace - while c[i] == ' ' or c[i] == '\t' or c [i] == '\l' or c [i] == '\r' : inc(i) + while c[i] == ' ' or c[i] == '\t' or c[i] == '\l' or c[i] == '\r' : inc(i) when defined(windows): # parse a single argument according to the above rules: if c[i] == '\0': break diff --git a/lib/pure/ospaths.nim b/lib/pure/ospaths.nim index 9fc816f2f..56671ee62 100644 --- a/lib/pure/ospaths.nim +++ b/lib/pure/ospaths.nim @@ -10,7 +10,7 @@ # Included by the ``os`` module but a module in its own right for NimScript # support. -when isMainModule: +when not declared(os): {.pragma: rtl.} import strutils @@ -556,12 +556,20 @@ when declared(getEnv) or defined(nimscript): yield substr(s, first, last-1) inc(last) - proc findExe*(exe: string): string {. + when not defined(windows) and declared(os): + proc checkSymlink(path: string): bool = + var rawInfo: Stat + if lstat(path, rawInfo) < 0'i32: result = false + else: result = S_ISLNK(rawInfo.st_mode) + + proc findExe*(exe: string, followSymlinks: bool = true): string {. tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect].} = ## Searches for `exe` in the current working directory and then ## in directories listed in the ``PATH`` environment variable. ## Returns "" if the `exe` cannot be found. On DOS-like platforms, `exe` ## is added the `ExeExt <#ExeExt>`_ file extension if it has none. + ## If the system supports symlinks it also resolves them until it + ## meets the actual file. This behavior can be disabled if desired. result = addFileExt(exe, ExeExt) if existsFile(result): return var path = string(getEnv("PATH")) @@ -572,7 +580,25 @@ when declared(getEnv) or defined(nimscript): result else: var x = expandTilde(candidate) / result - if existsFile(x): return x + if existsFile(x): + when not defined(windows) and declared(os): + while followSymlinks: # doubles as if here + if x.checkSymlink: + var r = newString(256) + var len = readlink(x, r, 256) + if len < 0: + raiseOSError(osLastError()) + if len > 256: + r = newString(len+1) + len = readlink(x, r, len) + setLen(r, len) + if isAbsolute(r): + x = r + else: + x = parentDir(x) / r + else: + break + return x result = "" when defined(nimscript) or (defined(nimdoc) and not declared(os)): diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 1ec4d0d8d..7378520e3 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -175,7 +175,11 @@ proc startCmd*(command: string, options: set[ProcessOption] = { result = startProcess(command=command, options=options + {poEvalCommand}) proc close*(p: Process) {.rtl, extern: "nosp$1", tags: [].} - ## When the process has finished executing, cleanup related handles + ## When the process has finished executing, cleanup related handles. + ## + ## **Warning:** If the process has not finished executing, this will forcibly + ## terminate the process. Doing so may result in zombie processes and + ## `pty leaks <http://stackoverflow.com/questions/27021641/how-to-fix-request-failed-on-channel-0>`_. proc suspend*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Suspends the process `p`. @@ -400,15 +404,16 @@ when defined(Windows) and not defined(useNimRtl): result = cast[cstring](alloc0(res.len+1)) copyMem(result, cstring(res), res.len) - proc buildEnv(env: StringTableRef): cstring = + proc buildEnv(env: StringTableRef): tuple[str: cstring, len: int] = var L = 0 for key, val in pairs(env): inc(L, key.len + val.len + 2) - result = cast[cstring](alloc0(L+2)) + var str = cast[cstring](alloc0(L+2)) L = 0 for key, val in pairs(env): var x = key & "=" & val - copyMem(addr(result[L]), cstring(x), x.len+1) # copy \0 + copyMem(addr(str[L]), cstring(x), x.len+1) # copy \0 inc(L, x.len+1) + (str, L) #proc open_osfhandle(osh: Handle, mode: int): int {. # importc: "_open_osfhandle", header: "<fcntl.h>".} @@ -526,13 +531,15 @@ when defined(Windows) and not defined(useNimRtl): else: cmdl = buildCommandLine(command, args) var wd: cstring = nil - var e: cstring = nil + var e = (str: nil.cstring, len: -1) if len(workingDir) > 0: wd = workingDir if env != nil: e = buildEnv(env) if poEchoCmd in options: echo($cmdl) when useWinUnicode: var tmp = newWideCString(cmdl) - var ee = newWideCString(e) + var ee = + if e.str.isNil: nil + else: newWideCString(e.str, e.len) var wwd = newWideCString(wd) var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT if poDemon in options: flags = flags or CREATE_NO_WINDOW @@ -549,7 +556,7 @@ when defined(Windows) and not defined(useNimRtl): if poStdErrToStdOut notin options: fileClose(si.hStdError) - if e != nil: dealloc(e) + if e.str != nil: dealloc(e.str) if success == 0: if poInteractive in result.options: close(result) const errInvalidParameter = 87.int @@ -721,7 +728,7 @@ elif not defined(useNimRtl): env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): Process = var - pStdin, pStdout, pStderr: array [0..1, cint] + pStdin, pStdout, pStderr: array[0..1, cint] new(result) result.options = options result.exitCode = -3 # for ``waitForExit`` diff --git a/lib/pure/oswalkdir.nim b/lib/pure/oswalkdir.nim index 000fe25a3..23ca0566a 100644 --- a/lib/pure/oswalkdir.nim +++ b/lib/pure/oswalkdir.nim @@ -1,3 +1,11 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# ## Compile-time only version for walkDir if you need it at compile-time ## for JavaScript. diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 25879d2b7..c648b0703 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -131,7 +131,7 @@ type # implementation const - SymChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.', '/', '\\'} + SymChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.', '/', '\\', '-'} proc rawGetTok(c: var CfgParser, tok: var Token) {.gcsafe.} diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index 06daa3782..d16a55302 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -142,6 +142,9 @@ proc kind*(my: XmlParser): XmlEventKind {.inline.} = template charData*(my: XmlParser): string = ## returns the character data for the events: ``xmlCharData``, ## ``xmlWhitespace``, ``xmlComment``, ``xmlCData``, ``xmlSpecial`` + ## Raises an assertion in debug mode if ``my.kind`` is not one + ## of those events. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind in {xmlCharData, xmlWhitespace, xmlComment, xmlCData, xmlSpecial}) my.a @@ -149,31 +152,49 @@ template charData*(my: XmlParser): string = template elementName*(my: XmlParser): string = ## returns the element name for the events: ``xmlElementStart``, ## ``xmlElementEnd``, ``xmlElementOpen`` + ## Raises an assertion in debug mode if ``my.kind`` is not one + ## of those events. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind in {xmlElementStart, xmlElementEnd, xmlElementOpen}) my.a template entityName*(my: XmlParser): string = ## returns the entity name for the event: ``xmlEntity`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlEntity``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlEntity) my.a template attrKey*(my: XmlParser): string = ## returns the attribute key for the event ``xmlAttribute`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlAttribute``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.a template attrValue*(my: XmlParser): string = ## returns the attribute value for the event ``xmlAttribute`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlAttribute``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.b template piName*(my: XmlParser): string = ## returns the processing instruction name for the event ``xmlPI`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlPI``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.a template piRest*(my: XmlParser): string = ## returns the rest of the processing instruction for the event ``xmlPI`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlPI``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.b diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 7e1f50266..5c978a2f8 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1841,8 +1841,8 @@ when isMainModule: result.add ", " result.add case n: - of 2: c[0].toLower & ": '" & c[1] & "'" - of 1: c[0].toLower & ": ''" + of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'" + of 1: toLowerAscii(c[0]) & ": ''" else: "" assert("Var1=key1;var2=Key2; VAR3". diff --git a/lib/pure/punycode.nim b/lib/pure/punycode.nim index 4f35de487..ab6501ed1 100644 --- a/lib/pure/punycode.nim +++ b/lib/pure/punycode.nim @@ -1,3 +1,11 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# import strutils import unicode diff --git a/lib/pure/random.nim b/lib/pure/random.nim index c73f403eb..08da771dc 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -7,11 +7,10 @@ # distribution, for details about the copyright. # -##[Nim's standard random number generator. Based on - -| `http://xoroshiro.di.unimi.it/`_ -| `http://xoroshiro.di.unimi.it/xoroshiro128plus.c`_ -]## +## Nim's standard random number generator. Based on the ``xoroshiro128+`` (xor/rotate/shift/rotate) library. +## * More information: http://xoroshiro.di.unimi.it/ +## * C implementation: http://xoroshiro.di.unimi.it/xoroshiro128plus.c +## include "system/inclrtl" {.push debugger:off.} diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index b6edb834c..9b6cf45c5 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -26,6 +26,12 @@ include "system/inclrtl" {.pop.} +# Support old split with set[char] +when defined(nimOldSplit): + {.pragma: deprecatedSplit, deprecated.} +else: + {.pragma: deprecatedSplit.} + type CharSet* {.deprecated.} = set[char] # for compatibility with Nim {.deprecated: [TCharSet: CharSet].} @@ -64,8 +70,8 @@ const ## doAssert "01234".find(invalid) == -1 ## doAssert "01A34".find(invalid) == 2 -proc isAlpha*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaChar".}= +proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsAlphaAsciiChar".}= ## Checks whether or not `c` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -85,27 +91,27 @@ proc isDigit*(c: char): bool {.noSideEffect, procvar, ## This checks 0-9 ASCII characters only. return c in Digits -proc isSpace*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceChar".}= +proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsSpaceAsciiChar".}= ## Checks whether or not `c` is a whitespace character. return c in Whitespace -proc isLower*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerChar".}= +proc isLowerAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsLowerAsciiChar".}= ## Checks whether or not `c` is a lower case character. ## ## This checks ASCII characters only. return c in {'a'..'z'} -proc isUpper*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperChar".}= +proc isUpperAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsUpperAsciiChar".}= ## Checks whether or not `c` is an upper case character. ## ## This checks ASCII characters only. return c in {'A'..'Z'} -proc isAlpha*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaStr".}= +proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsAlphaAsciiStr".}= ## Checks whether or not `s` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -117,7 +123,7 @@ proc isAlpha*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isAlpha() and result + result = c.isAlphaAscii() and result proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsAlphaNumericStr".}= @@ -149,8 +155,8 @@ proc isDigit*(s: string): bool {.noSideEffect, procvar, for c in s: result = c.isDigit() and result -proc isSpace*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceStr".}= +proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsSpaceAsciiStr".}= ## Checks whether or not `s` is completely whitespace. ## ## Returns true if all characters in `s` are whitespace @@ -160,10 +166,11 @@ proc isSpace*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isSpace() and result + if not c.isSpaceAscii(): + return false -proc isLower*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerStr".}= +proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsLowerAsciiStr".}= ## Checks whether or not `s` contains all lower case characters. ## ## This checks ASCII characters only. @@ -174,10 +181,10 @@ proc isLower*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isLower() and result + result = c.isLowerAscii() and result -proc isUpper*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperStr".}= +proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsUpperAsciiStr".}= ## Checks whether or not `s` contains all upper case characters. ## ## This checks ASCII characters only. @@ -188,10 +195,10 @@ proc isUpper*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isUpper() and result + result = c.isUpperAscii() and result -proc toLower*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToLowerChar".} = +proc toLowerAscii*(c: char): char {.noSideEffect, procvar, + rtl, extern: "nsuToLowerAsciiChar".} = ## Converts `c` into lower case. ## ## This works only for the letters ``A-Z``. See `unicode.toLower @@ -202,8 +209,8 @@ proc toLower*(c: char): char {.noSideEffect, procvar, else: result = c -proc toLower*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToLowerStr".} = +proc toLowerAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuToLowerAsciiStr".} = ## Converts `s` into lower case. ## ## This works only for the letters ``A-Z``. See `unicode.toLower @@ -211,10 +218,10 @@ proc toLower*(s: string): string {.noSideEffect, procvar, ## character. result = newString(len(s)) for i in 0..len(s) - 1: - result[i] = toLower(s[i]) + result[i] = toLowerAscii(s[i]) -proc toUpper*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToUpperChar".} = +proc toUpperAscii*(c: char): char {.noSideEffect, procvar, + rtl, extern: "nsuToUpperAsciiChar".} = ## Converts `c` into upper case. ## ## This works only for the letters ``A-Z``. See `unicode.toUpper @@ -225,8 +232,8 @@ proc toUpper*(c: char): char {.noSideEffect, procvar, else: result = c -proc toUpper*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToUpperStr".} = +proc toUpperAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuToUpperAsciiStr".} = ## Converts `s` into upper case. ## ## This works only for the letters ``A-Z``. See `unicode.toUpper @@ -234,14 +241,145 @@ proc toUpper*(s: string): string {.noSideEffect, procvar, ## character. result = newString(len(s)) for i in 0..len(s) - 1: - result[i] = toUpper(s[i]) + result[i] = toUpperAscii(s[i]) + +proc capitalizeAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuCapitalizeAscii".} = + ## Converts the first character of `s` into upper case. + ## + ## This works only for the letters ``A-Z``. + result = toUpperAscii(s[0]) & substr(s, 1) + +proc isSpace*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsSpaceChar".}= + ## Checks whether or not `c` is a whitespace character. + ## + ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. + isSpaceAscii(c) + +proc isLower*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsLowerChar".}= + ## Checks whether or not `c` is a lower case character. + ## + ## This checks ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. + isLowerAscii(c) + +proc isUpper*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsUpperChar".}= + ## Checks whether or not `c` is an upper case character. + ## + ## This checks ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. + isUpperAscii(c) + +proc isAlpha*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsAlphaChar".}= + ## Checks whether or not `c` is alphabetical. + ## + ## This checks a-z, A-Z ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. + isAlphaAscii(c) + +proc isAlpha*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsAlphaStr".}= + ## Checks whether or not `s` is alphabetical. + ## + ## This checks a-z, A-Z ASCII characters only. + ## Returns true if all characters in `s` are + ## alphabetic and there is at least one character + ## in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. + isAlphaAscii(s) + +proc isSpace*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsSpaceStr".}= + ## Checks whether or not `s` is completely whitespace. + ## + ## Returns true if all characters in `s` are whitespace + ## characters and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. + isSpaceAscii(s) + +proc isLower*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsLowerStr".}= + ## Checks whether or not `s` contains all lower case characters. + ## + ## This checks ASCII characters only. + ## Returns true if all characters in `s` are lower case + ## and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. + isLowerAscii(s) + +proc isUpper*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsUpperStr".}= + ## Checks whether or not `s` contains all upper case characters. + ## + ## This checks ASCII characters only. + ## Returns true if all characters in `s` are upper case + ## and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. + isUpperAscii(s) + +proc toLower*(c: char): char {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToLowerChar".} = + ## Converts `c` into lower case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toLower + ## <unicode.html#toLower>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. + toLowerAscii(c) + +proc toLower*(s: string): string {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToLowerStr".} = + ## Converts `s` into lower case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toLower + ## <unicode.html#toLower>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. + toLowerAscii(s) + +proc toUpper*(c: char): char {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToUpperChar".} = + ## Converts `c` into upper case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toUpper + ## <unicode.html#toUpper>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. + toUpperAscii(c) + +proc toUpper*(s: string): string {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToUpperStr".} = + ## Converts `s` into upper case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toUpper + ## <unicode.html#toUpper>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. + toUpperAscii(s) proc capitalize*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuCapitalize".} = + rtl, deprecated, extern: "nsuCapitalize".} = ## Converts the first character of `s` into upper case. ## ## This works only for the letters ``A-Z``. - result = toUpper(s[0]) & substr(s, 1) + ## + ## **Deprecated since version 0.15.0**: use ``capitalizeAscii`` instead. + capitalizeAscii(s) proc normalize*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuNormalize".} = @@ -270,7 +408,7 @@ proc cmpIgnoreCase*(a, b: string): int {.noSideEffect, var i = 0 var m = min(a.len, b.len) while i < m: - result = ord(toLower(a[i])) - ord(toLower(b[i])) + result = ord(toLowerAscii(a[i])) - ord(toLowerAscii(b[i])) if result != 0: return inc(i) result = a.len - b.len @@ -291,8 +429,8 @@ proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, while true: while a[i] == '_': inc(i) while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLower(a[i]) - var bb = toLower(b[j]) + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[j]) result = ord(aa) - ord(bb) if result != 0 or aa == '\0': break inc(i) @@ -326,17 +464,77 @@ proc toOctal*(c: char): string {.noSideEffect, rtl, extern: "nsuToOctal".} = result[i] = chr(val mod 8 + ord('0')) val = val div 8 +proc isNilOrEmpty*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsNilOrEmpty".} = + ## Checks if `s` is nil or empty. + result = len(s) == 0 + +proc isNilOrWhitespace*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsNilOrWhitespace".} = + ## Checks if `s` is nil or consists entirely of whitespace characters. + if len(s) == 0: + return true + + result = true + for c in s: + if not c.isSpace(): + return false + +proc substrEq(s: string, pos: int, substr: string): bool = + var i = 0 + var length = substr.len + while i < length and s[pos+i] == substr[i]: + inc i + + return i == length + +# --------- Private templates for different split separators ----------- + +template stringHasSep(s: string, index: int, seps: set[char]): bool = + s[index] in seps + +template stringHasSep(s: string, index: int, sep: char): bool = + s[index] == sep + +template stringHasSep(s: string, index: int, sep: string): bool = + s.substrEq(index, sep) + +template splitCommon(s, sep, maxsplit, sepLen) = + ## Common code for split procedures + var last = 0 + var splits = maxsplit + + if len(s) > 0: + while last <= len(s): + var first = last + while last < len(s) and not stringHasSep(s, last, sep): + inc(last) + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + inc(last, sepLen) + +template oldSplit(s, seps, maxsplit) = + var last = 0 + var splits = maxsplit + assert(not ('\0' in seps)) + while last < len(s): + while s[last] in seps: inc(last) + var first = last + while last < len(s) and s[last] notin seps: inc(last) + if first <= last-1: + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + iterator split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a group of separators. ## - ## Substrings are separated by a substring containing only `seps`. Note - ## that whole sequences of characters found in ``seps`` will be counted as - ## a single split point and leading/trailing separators will be ignored. - ## The following example: + ## Substrings are separated by a substring containing only `seps`. ## ## .. code-block:: nim - ## for word in split(" this is an example "): + ## for word in split("this\lis an\texample"): ## writeLine(stdout, word) ## ## ...generates this output: @@ -350,7 +548,7 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## And the following code: ## ## .. code-block:: nim - ## for word in split(";;this;is;an;;example;;;", {';'}): + ## for word in split("this:is;an$example", {';', ':', '$'}): ## writeLine(stdout, word) ## ## ...produces the same output as the first example. The code: @@ -371,26 +569,26 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## "08" ## "08.398990" ## - var last = 0 - var splits = maxsplit - assert(not ('\0' in seps)) - while last < len(s): - while s[last] in seps: inc(last) - var first = last - while last < len(s) and s[last] notin seps: inc(last) # BUGFIX! - if first <= last-1: - if splits == 0: last = len(s) - yield substr(s, first, last-1) - if splits == 0: break - dec(splits) + when defined(nimOldSplit): + oldSplit(s, seps, maxsplit) + else: + splitCommon(s, seps, maxsplit, 1) + +iterator splitWhitespace*(s: string): string = + ## Splits at whitespace. + oldSplit(s, Whitespace, -1) + +proc splitWhitespace*(s: string): seq[string] {.noSideEffect, + rtl, extern: "nsuSplitWhitespace".} = + ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ + ## iterator, but is a proc that returns a sequence of substrings. + accumulateResult(splitWhitespace(s)) iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a single separator. ## ## Substrings are separated by the character `sep`. - ## Unlike the version of the iterator which accepts a set of separator - ## characters, this proc will not coalesce groups of the - ## separator, returning a string for each found character. The code: + ## The code: ## ## .. code-block:: nim ## for word in split(";;this;is;an;;example;;;", ';'): @@ -410,56 +608,27 @@ iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## "" ## "" ## - var last = 0 - var splits = maxsplit - assert('\0' != sep) - if len(s) > 0: - # `<=` is correct here for the edge cases! - while last <= len(s): - var first = last - while last < len(s) and s[last] != sep: inc(last) - if splits == 0: last = len(s) - yield substr(s, first, last-1) - if splits == 0: break - dec(splits) - inc(last) - -proc substrEq(s: string, pos: int, substr: string): bool = - var i = 0 - var length = substr.len - while i < length and s[pos+i] == substr[i]: - inc i - - return i == length + splitCommon(s, sep, maxsplit, 1) iterator split*(s: string, sep: string, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a string separator. ## ## Substrings are separated by the string `sep`. - var last = 0 - var splits = maxsplit - - if len(s) > 0: - while last <= len(s): - var first = last - while last < len(s) and not s.substrEq(last, sep): - inc(last) - if splits == 0: last = len(s) - yield substr(s, first, last-1) - if splits == 0: break - dec(splits) - inc(last, sep.len) - -# --------- Private templates for different rsplit separators ----------- - -template stringHasSep(s: string, index: int, seps: set[char]): bool = - s[index] in seps - -template stringHasSep(s: string, index: int, sep: char): bool = - s[index] == sep + ## The code: + ## + ## .. code-block:: nim + ## for word in split("thisDATAisDATAcorrupted", "DATA"): + ## writeLine(stdout, word) + ## + ## Results in: + ## + ## .. code-block:: + ## "this" + ## "is" + ## "corrupted" + ## -template stringHasSep(s: string, index: int, sep: string): bool = - s.substrEq(index, sep) + splitCommon(s, sep, maxsplit, sep.len) template rsplitCommon(s, sep, maxsplit, sepLen) = ## Common code for rsplit functions @@ -2123,13 +2292,13 @@ when isMainModule: doAssert " foo\n bar".indent(4, "Q") == "QQQQ foo\nQQQQ bar" - doAssert isAlpha('r') - doAssert isAlpha('A') - doAssert(not isAlpha('$')) + doAssert isAlphaAscii('r') + doAssert isAlphaAscii('A') + doAssert(not isAlphaAscii('$')) - doAssert isAlpha("Rasp") - doAssert isAlpha("Args") - doAssert(not isAlpha("$Tomato")) + doAssert isAlphaAscii("Rasp") + doAssert isAlphaAscii("Args") + doAssert(not isAlphaAscii("$Tomato")) doAssert isAlphaNumeric('3') doAssert isAlphaNumeric('R') @@ -2148,32 +2317,43 @@ when isMainModule: doAssert(not isDigit("12.33")) doAssert(not isDigit("A45b")) - doAssert isSpace('\t') - doAssert isSpace('\l') - doAssert(not isSpace('A')) + doAssert isSpaceAscii('\t') + doAssert isSpaceAscii('\l') + doAssert(not isSpaceAscii('A')) - doAssert isSpace("\t\l \v\r\f") - doAssert isSpace(" ") - doAssert(not isSpace("ABc \td")) + doAssert isSpaceAscii("\t\l \v\r\f") + doAssert isSpaceAscii(" ") + doAssert(not isSpaceAscii("ABc \td")) - doAssert isLower('a') - doAssert isLower('z') - doAssert(not isLower('A')) - doAssert(not isLower('5')) - doAssert(not isLower('&')) + doAssert(isNilOrEmpty("")) + doAssert(isNilOrEmpty(nil)) + doAssert(not isNilOrEmpty("test")) + doAssert(not isNilOrEmpty(" ")) - doAssert isLower("abcd") - doAssert(not isLower("abCD")) - doAssert(not isLower("33aa")) + doAssert(isNilOrWhitespace("")) + doAssert(isNilOrWhitespace(nil)) + doAssert(isNilOrWhitespace(" ")) + doAssert(isNilOrWhitespace("\t\l \v\r\f")) + doAssert(not isNilOrWhitespace("ABc \td")) - doAssert isUpper('A') - doAssert(not isUpper('b')) - doAssert(not isUpper('5')) - doAssert(not isUpper('%')) + doAssert isLowerAscii('a') + doAssert isLowerAscii('z') + doAssert(not isLowerAscii('A')) + doAssert(not isLowerAscii('5')) + doAssert(not isLowerAscii('&')) - doAssert isUpper("ABC") - doAssert(not isUpper("AAcc")) - doAssert(not isUpper("A#$")) + doAssert isLowerAscii("abcd") + doAssert(not isLowerAscii("abCD")) + doAssert(not isLowerAscii("33aa")) + + doAssert isUpperAscii('A') + doAssert(not isUpperAscii('b')) + doAssert(not isUpperAscii('5')) + doAssert(not isUpperAscii('%')) + + doAssert isUpperAscii("ABC") + doAssert(not isUpperAscii("AAcc")) + doAssert(not isUpperAscii("A#$")) doAssert rsplit("foo bar", seps=Whitespace) == @["foo", "bar"] doAssert rsplit(" foo bar", seps=Whitespace, maxsplit=1) == @[" foo", "bar"] @@ -2218,11 +2398,14 @@ bar bar """.unindent() == "foo\nfoo\nbar\n" - let s = " this is an example " - doAssert s.split() == @["this", "is", "an", "example"] - doAssert s.split(maxsplit=4) == @["this", "is", "an", "example"] - doAssert s.split(' ', maxsplit=4) == @["", "this", "", "", "is an example "] - doAssert s.split(" ", maxsplit=4) == @["", "this", "", "", "is an example "] + let s = " this is an example " + let s2 = ":this;is;an:example;;" + + doAssert s.split() == @["", "this", "is", "an", "example", "", ""] + doAssert s2.split(seps={':', ';'}) == @["", "this", "is", "an", "example", "", ""] + doAssert s.split(maxsplit=4) == @["", "this", "is", "an", "example "] + doAssert s.split(' ', maxsplit=1) == @["", "this is an example "] + doAssert s.split(" ", maxsplit=4) == @["", "this", "is", "an", "example "] block: # formatEng tests doAssert formatEng(0, 2, trim=false) == "0.00" diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 60f064e7c..1f34ec07e 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -384,7 +384,7 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright=false) = var old = getAttributes(h) and not 0x0007 if bright: old = old or FOREGROUND_INTENSITY - const lookup: array [ForegroundColor, int] = [ + const lookup: array[ForegroundColor, int] = [ 0, (FOREGROUND_RED), (FOREGROUND_GREEN), @@ -406,7 +406,7 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright=false) = var old = getAttributes(h) and not 0x0070 if bright: old = old or BACKGROUND_INTENSITY - const lookup: array [BackgroundColor, int] = [ + const lookup: array[BackgroundColor, int] = [ 0, (BACKGROUND_RED), (BACKGROUND_GREEN), @@ -493,17 +493,21 @@ template styledEcho*(args: varargs[expr]): expr = ## Echoes styles arguments to stdout using ``styledWriteLine``. callStyledEcho(args) -when defined(nimdoc): - proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. - ## The character is not printed to the terminal. This is not available for - ## Windows. - discard -elif not defined(windows): - proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. - ## The character is not printed to the terminal. This is not available for - ## Windows. +proc getch*(): char = + ## Read a single character from the terminal, blocking until it is entered. + ## The character is not printed to the terminal. + when defined(windows): + let fd = getStdHandle(STD_INPUT_HANDLE) + var keyEvent = KEY_EVENT_RECORD() + var numRead: cint + while true: + # Block until character is entered + doAssert(waitForSingleObject(fd, INFINITE) == WAIT_OBJECT_0) + doAssert(readConsoleInput(fd, addr(keyEvent), 1, addr(numRead)) != 0) + if numRead == 0 or keyEvent.eventType != 1 or keyEvent.bKeyDown == 0: + continue + return char(keyEvent.uChar) + else: let fd = getFileHandle(stdin) var oldMode: Termios discard fd.tcgetattr(addr oldMode) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index ac8dc93ad..d6cb5c9da 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -66,7 +66,7 @@ when defined(posix) and not defined(JS): when not defined(freebsd) and not defined(netbsd) and not defined(openbsd): var timezone {.importc, header: "<time.h>".}: int - var + var tzname {.importc, header: "<time.h>" .}: array[0..1, cstring] # we also need tzset() to make sure that tzname is initialized proc tzset() {.importc, header: "<time.h>".} @@ -369,7 +369,10 @@ proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## very accurate. let t = toSeconds(toTime(a)) let secs = toSeconds(a, interval) - result = getLocalTime(fromSeconds(t + secs)) + if a.tzname == "UTC": + result = getGMTime(fromSeconds(t + secs)) + else: + result = getLocalTime(fromSeconds(t + secs)) proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## subtracts ``interval`` time from TimeInfo ``a``. @@ -386,7 +389,10 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = intval.months = - interval.months intval.years = - interval.years let secs = toSeconds(a, intval) - result = getLocalTime(fromSeconds(t + secs)) + if a.tzname == "UTC": + result = getGMTime(fromSeconds(t + secs)) + else: + result = getLocalTime(fromSeconds(t + secs)) proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds @@ -471,7 +477,7 @@ when not defined(JS): # our own procs on top of that: proc tmToTimeInfo(tm: StructTM, local: bool): TimeInfo = const - weekDays: array [0..6, WeekDay] = [ + weekDays: array[0..6, WeekDay] = [ dSun, dMon, dTue, dWed, dThu, dFri, dSat] when defined(freebsd) or defined(netbsd) or defined(openbsd): TimeInfo(second: int(tm.second), @@ -517,7 +523,7 @@ when not defined(JS): proc timeInfoToTM(t: TimeInfo): StructTM = const - weekDays: array [WeekDay, int8] = [1'i8,2'i8,3'i8,4'i8,5'i8,6'i8,0'i8] + weekDays: array[WeekDay, int8] = [1'i8,2'i8,3'i8,4'i8,5'i8,6'i8,0'i8] result.second = t.second result.minute = t.minute result.hour = t.hour @@ -646,7 +652,7 @@ elif defined(JS): return newDate() const - weekDays: array [0..6, WeekDay] = [ + weekDays: array[0..6, WeekDay] = [ dSun, dMon, dTue, dWed, dThu, dFri, dSat] proc getLocalTime(t: Time): TimeInfo = diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index ac25dccef..0cbe8de7a 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -1369,6 +1369,64 @@ proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = (c >= 0x20d0 and c <= 0x20ff) or (c >= 0xfe20 and c <= 0xfe2f)) +template runeCheck(s, runeProc) = + ## Common code for rune.isLower, rune.isUpper, etc + result = if len(s) == 0: false else: true + + var + i = 0 + rune: Rune + + while i < len(s) and result: + fastRuneAt(s, i, rune, doInc=true) + result = runeProc(rune) and result + +proc isUpper*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all upper case unicode characters. + runeCheck(s, isUpper) + +proc isLower*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all lower case unicode characters. + runeCheck(s, isLower) + +proc isAlpha*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all alphabetic unicode characters. + runeCheck(s, isAlpha) + +proc isSpace*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all whitespace unicode characters. + runeCheck(s, isWhiteSpace) + +template convertRune(s, runeProc) = + ## Convert runes in `s` using `runeProc` as the converter. + result = newString(len(s)) + + var + i = 0 + lastIndex = 0 + rune: Rune + + while i < len(s): + lastIndex = i + fastRuneAt(s, i, rune, doInc=true) + rune = runeProc(rune) + + rune.fastToUTF8Copy(result, lastIndex) + +proc toUpper*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Converts `s` into upper-case unicode characters. + convertRune(s, toUpper) + +proc toLower*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Converts `s` into lower-case unicode characters. + convertRune(s, toLower) + proc swapCase*(s: string): string {.noSideEffect, procvar, rtl, extern: "nuc$1".} = ## Swaps the case of unicode characters in `s` @@ -1395,6 +1453,20 @@ proc swapCase*(s: string): string {.noSideEffect, procvar, rune.fastToUTF8Copy(result, lastIndex) +proc capitalize*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1".} = + ## Converts the first character of `s` into an upper-case unicode character. + if len(s) == 0: + return s + + var + rune: Rune + i = 0 + + fastRuneAt(s, i, rune, doInc=true) + + result = $toUpper(rune) & substr(s, i) + proc translate*(s: string, replacements: proc(key: string): string): string {. rtl, extern: "nuc$1".} = ## Translates words in a string using the `replacements` proc to substitute @@ -1500,7 +1572,7 @@ proc isTitle*(s: string): bool {.noSideEffect, procvar, firstRune = true iterator runes*(s: string): Rune = - ## Iterates over any unicode character of the string ``s`` + ## Iterates over any unicode character of the string ``s`` returning runes var i = 0 result: Rune @@ -1508,6 +1580,14 @@ iterator runes*(s: string): Rune = fastRuneAt(s, i, result, true) yield result +iterator utf8*(s: string): string = + ## Iterates over any unicode character of the string ``s`` returning utf8 values + var o = 0 + while o < s.len: + let n = runeLenAt(s, o) + yield s[o.. (o+n-1)] + o += n + proc toRunes*(s: string): seq[Rune] = ## Obtains a sequence containing the Runes in ``s`` result = newSeq[Rune]() @@ -1618,6 +1698,10 @@ when isMainModule: doAssert title("αlpha βeta γamma") == "Αlpha Βeta Γamma" doAssert title("") == "" + doAssert capitalize("βeta") == "Βeta" + doAssert capitalize("foo") == "Foo" + doAssert capitalize("") == "" + doAssert isTitle("Foo") doAssert(not isTitle("Foo bar")) doAssert(not isTitle("αlpha Βeta")) @@ -1630,6 +1714,64 @@ when isMainModule: doAssert swapCase("a✓B") == "A✓b" doAssert swapCase("") == "" + doAssert isAlpha("r") + doAssert isAlpha("α") + doAssert(not isAlpha("$")) + doAssert(not isAlpha("")) + + doAssert isAlpha("Βeta") + doAssert isAlpha("Args") + doAssert(not isAlpha("$Foo✓")) + + doAssert isSpace("\t") + doAssert isSpace("\l") + doAssert(not isSpace("Β")) + doAssert(not isSpace("Βeta")) + + doAssert isSpace("\t\l \v\r\f") + doAssert isSpace(" ") + doAssert(not isSpace("")) + doAssert(not isSpace("ΑΓc \td")) + + doAssert isLower("a") + doAssert isLower("γ") + doAssert(not isLower("Γ")) + doAssert(not isLower("4")) + doAssert(not isLower("")) + + doAssert isLower("abcdγ") + doAssert(not isLower("abCDΓ")) + doAssert(not isLower("33aaΓ")) + + doAssert isUpper("Γ") + doAssert(not isUpper("b")) + doAssert(not isUpper("α")) + doAssert(not isUpper("✓")) + doAssert(not isUpper("")) + + doAssert isUpper("ΑΒΓ") + doAssert(not isUpper("AAccβ")) + doAssert(not isUpper("A#$β")) + + doAssert toUpper("Γ") == "Γ" + doAssert toUpper("b") == "B" + doAssert toUpper("α") == "Α" + doAssert toUpper("✓") == "✓" + doAssert toUpper("") == "" + + doAssert toUpper("ΑΒΓ") == "ΑΒΓ" + doAssert toUpper("AAccβ") == "AACCΒ" + doAssert toUpper("A✓$β") == "A✓$Β" + + doAssert toLower("a") == "a" + doAssert toLower("γ") == "γ" + doAssert toLower("Γ") == "γ" + doAssert toLower("4") == "4" + doAssert toLower("") == "" + + doAssert toLower("abcdγ") == "abcdγ" + doAssert toLower("abCDΓ") == "abcdγ" + doAssert toLower("33aaΓ") == "33aaγ" doAssert reversed("Reverse this!") == "!siht esreveR" doAssert reversed("先秦兩漢") == "漢兩秦先" @@ -1643,6 +1785,12 @@ when isMainModule: # test for rune positioning and runeSubStr() let s = "Hänsel ««: 10,00€" + var t = "" + for c in s.utf8: + t.add c + + doAssert(s == t) + doAssert(runeReverseOffset(s, 1) == (20, 18)) doAssert(runeReverseOffset(s, 19) == (-1, 18)) diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index e8a23f27e..7e3983a0d 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -259,31 +259,34 @@ macro check*(conditions: stmt): stmt {.immediate.} = proc inspectArgs(exp: NimNode): NimNode = result = copyNimTree(exp) - for i in countup(1, exp.len - 1): - if exp[i].kind notin nnkLiterals: - inc counter - var arg = newIdentNode(":p" & $counter) - var argStr = exp[i].toStrLit - var paramAst = exp[i] - if exp[i].kind == nnkIdent: - argsPrintOuts.add getAst(print(argStr, paramAst)) - if exp[i].kind in nnkCallKinds: - var callVar = newIdentNode(":c" & $counter) - argsAsgns.add getAst(asgn(callVar, paramAst)) - result[i] = callVar - argsPrintOuts.add getAst(print(argStr, callVar)) - if exp[i].kind == nnkExprEqExpr: - # ExprEqExpr - # Ident !"v" - # IntLit 2 - result[i] = exp[i][1] - if exp[i].typekind notin {ntyTypeDesc}: - argsAsgns.add getAst(asgn(arg, paramAst)) - argsPrintOuts.add getAst(print(argStr, arg)) - if exp[i].kind != nnkExprEqExpr: - result[i] = arg - else: - result[i][1] = arg + if exp[0].kind == nnkIdent and + $exp[0] in ["and", "or", "not", "in", "notin", "==", "<=", + ">=", "<", ">", "!=", "is", "isnot"]: + for i in countup(1, exp.len - 1): + if exp[i].kind notin nnkLiterals: + inc counter + var arg = newIdentNode(":p" & $counter) + var argStr = exp[i].toStrLit + var paramAst = exp[i] + if exp[i].kind == nnkIdent: + argsPrintOuts.add getAst(print(argStr, paramAst)) + if exp[i].kind in nnkCallKinds: + var callVar = newIdentNode(":c" & $counter) + argsAsgns.add getAst(asgn(callVar, paramAst)) + result[i] = callVar + argsPrintOuts.add getAst(print(argStr, callVar)) + if exp[i].kind == nnkExprEqExpr: + # ExprEqExpr + # Ident !"v" + # IntLit 2 + result[i] = exp[i][1] + if exp[i].typekind notin {ntyTypeDesc}: + argsAsgns.add getAst(asgn(arg, paramAst)) + argsPrintOuts.add getAst(print(argStr, arg)) + if exp[i].kind != nnkExprEqExpr: + result[i] = arg + else: + result[i][1] = arg case checked.kind of nnkCallKinds: diff --git a/lib/system.nim b/lib/system.nim index 5a84f4a52..b4919db2d 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -669,6 +669,12 @@ proc newSeq*[T](len = 0.Natural): seq[T] = ## #inputStrings[3] = "out of bounds" newSeq(result, len) +proc newSeqOfCap*[T](cap: Natural): seq[T] {. + magic: "NewSeqOfCap", noSideEffect.} = + ## creates a new sequence of type ``seq[T]`` with length 0 and capacity + ## ``cap``. + discard + proc len*[TOpenArray: openArray|varargs](x: TOpenArray): int {. magic: "LengthOpenArray", noSideEffect.} proc len*(x: string): int {.magic: "LengthStr", noSideEffect.} @@ -1523,7 +1529,7 @@ type # these work for most platforms: ## This is the same as the type ``unsigned long long`` in *C*. cstringArray* {.importc: "char**", nodecl.} = ptr - array [0..ArrayDummySize, cstring] + array[0..ArrayDummySize, cstring] ## This is binary compatible to the type ``char**`` in *C*. The array's ## high value is large enough to disable bounds checking in practice. ## Use `cstringArrayToSeq` to convert it into a ``seq[string]``. @@ -3353,7 +3359,11 @@ proc astToStr*[T](x: T): string {.magic: "AstToStr", noSideEffect.} proc instantiationInfo*(index = -1, fullPaths = false): tuple[ filename: string, line: int] {. magic: "InstantiationInfo", noSideEffect.} - ## provides access to the compiler's instantiation stack line information. + ## provides access to the compiler's instantiation stack line information + ## of a template. + ## + ## While similar to the `caller info`:idx: of other languages, it is determined + ## at compile time. ## ## This proc is mostly useful for meta programming (eg. ``assert`` template) ## to retrieve information about the current filename and line number. @@ -3633,6 +3643,27 @@ proc `==` *(x, y: cstring): bool {.magic: "EqCString", noSideEffect, elif x.isNil or y.isNil: result = false else: result = strcmp(x, y) == 0 +template closureScope*(body: untyped): stmt = + ## Useful when creating a closure in a loop to capture local loop variables by + ## their current iteration values. Example: + ## + ## .. code-block:: nim + ## var myClosure : proc() + ## # without closureScope: + ## for i in 0 .. 5: + ## let j = i + ## if j == 3: + ## myClosure = proc() = echo j + ## myClosure() # outputs 5. `j` is changed after closure creation + ## # with closureScope: + ## for i in 0 .. 5: + ## closureScope: # Everything in this scope is locked after closure creation + ## let j = i + ## if j == 3: + ## myClosure = proc() = echo j + ## myClosure() # outputs 3 + (proc() = body)() + {.pop.} #{.push warning[GcMem]: off, warning[Uninit]: off.} when defined(nimconfig): diff --git a/lib/system/assign.nim b/lib/system/assign.nim index 55d7572e2..231a20d86 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -211,14 +211,14 @@ proc genericReset(dest: pointer, mt: PNimType) = zeroMem(dest, mt.size) # set raw bits to zero proc selectBranch(discVal, L: int, - a: ptr array [0..0x7fff, ptr TNimNode]): ptr TNimNode = + a: ptr array[0..0x7fff, ptr TNimNode]): ptr TNimNode = result = a[L] # a[L] contains the ``else`` part (but may be nil) if discVal <% L: var x = a[discVal] if x != nil: result = x proc FieldDiscriminantCheck(oldDiscVal, newDiscVal: int, - a: ptr array [0..0x7fff, ptr TNimNode], + a: ptr array[0..0x7fff, ptr TNimNode], L: int) {.compilerProc.} = var oldBranch = selectBranch(oldDiscVal, L, a) var newBranch = selectBranch(newDiscVal, L, a) diff --git a/lib/system/debugger.nim b/lib/system/debugger.nim index 55f14e982..cc6919d36 100644 --- a/lib/system/debugger.nim +++ b/lib/system/debugger.nim @@ -159,7 +159,7 @@ type {.deprecated: [THash: Hash, TWatchpoint: Watchpoint].} var - watchpoints: array [0..99, Watchpoint] + watchpoints: array[0..99, Watchpoint] watchpointsLen: int proc `!&`(h: Hash, val: int): Hash {.inline.} = diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 6e512c9b1..5445a067c 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -124,7 +124,9 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) = copyMem(dest, src, mt.size) proc genericDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} = + GC_disable() genericDeepCopyAux(dest, src, mt) + GC_enable() proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} = # also invoked for 'string' diff --git a/lib/system/endb.nim b/lib/system/endb.nim index 92aae8c71..d4d10a52c 100644 --- a/lib/system/endb.nim +++ b/lib/system/endb.nim @@ -475,7 +475,7 @@ proc dbgWriteStackTrace(f: PFrame) = it = f i = 0 total = 0 - tempFrames: array [0..127, PFrame] + tempFrames: array[0..127, PFrame] # setup long head: while it != nil and i <= high(tempFrames)-firstCalls: tempFrames[i] = it diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 526e27c83..b89729850 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -90,13 +90,13 @@ when defined(nativeStacktrace) and nativeStackTraceSupported: when not hasThreadSupport: var - tempAddresses: array [0..127, pointer] # should not be alloc'd on stack + tempAddresses: array[0..127, pointer] # should not be alloc'd on stack tempDlInfo: TDl_info proc auxWriteStackTraceWithBacktrace(s: var string) = when hasThreadSupport: var - tempAddresses: array [0..127, pointer] # but better than a threadvar + tempAddresses: array[0..127, pointer] # but better than a threadvar tempDlInfo: TDl_info # This is allowed to be expensive since it only happens during crashes # (but this way you don't need manual stack tracing) @@ -124,12 +124,12 @@ when defined(nativeStacktrace) and nativeStackTraceSupported: when not hasThreadSupport: var - tempFrames: array [0..127, PFrame] # should not be alloc'd on stack + tempFrames: array[0..127, PFrame] # should not be alloc'd on stack proc auxWriteStackTrace(f: PFrame, s: var string) = when hasThreadSupport: var - tempFrames: array [0..127, PFrame] # but better than a threadvar + tempFrames: array[0..127, PFrame] # but better than a threadvar const firstCalls = 32 var @@ -250,7 +250,7 @@ proc raiseExceptionAux(e: ref Exception) = inc L, slen template add(buf, s: expr) = xadd(buf, s, s.len) - var buf: array [0..2000, char] + var buf: array[0..2000, char] var L = 0 add(buf, "Error: unhandled exception: ") if not isNil(e.msg): add(buf, e.msg) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 3e71b4fe0..a36dbf02b 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -452,10 +452,13 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line + when leakDetector: + res.filename = nil + res.line = 0 + when not hasThreadSupport: + if framePtr != nil and framePtr.prev != nil: + res.filename = framePtr.prev.filename + res.line = framePtr.prev.line # refcount is zero, color is black, but mark it to be in the ZCT res.refcount = ZctFlag sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") @@ -503,10 +506,13 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line + when leakDetector: + res.filename = nil + res.line = 0 + when not hasThreadSupport: + if framePtr != nil and framePtr.prev != nil: + res.filename = framePtr.prev.filename + res.line = framePtr.prev.line res.refcount = rcIncrement # refcount is 1 sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") when logGC: writeCell("new cell", res) diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index 4807bb6f8..cdca36fb7 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -239,7 +239,7 @@ else: # We use a jmp_buf buffer that is in the C stack. # Used to traverse the stack and registers assuming # that 'setjmp' will save registers in the C stack. - type PStackSlice = ptr array [0..7, pointer] + type PStackSlice = ptr array[0..7, pointer] var registers {.noinit.}: Registers getRegisters(registers) for i in registers.low .. registers.high: @@ -277,7 +277,7 @@ else: # We use a jmp_buf buffer that is in the C stack. # Used to traverse the stack and registers assuming # that 'setjmp' will save registers in the C stack. - type PStackSlice = ptr array [0..7, pointer] + type PStackSlice = ptr array[0..7, pointer] var registers {.noinit.}: C_JmpBuf if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. var max = cast[ByteAddress](gch.stackBottom) diff --git a/lib/system/hti.nim b/lib/system/hti.nim index bfb13059e..984f888cb 100644 --- a/lib/system/hti.nim +++ b/lib/system/hti.nim @@ -71,7 +71,7 @@ type typ: ptr TNimType name: cstring len: int - sons: ptr array [0..0x7fff, ptr TNimNode] + sons: ptr array[0..0x7fff, ptr TNimNode] TNimTypeFlag = enum ntfNoRefs = 0, # type contains no tyRef, tySequence, tyString diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index ce67373bc..9c8a18bfe 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -65,7 +65,7 @@ proc auxWriteStackTrace(f: PCallFrame): string = it = f i = 0 total = 0 - tempFrames: array [0..63, TempFrame] + tempFrames: array[0..63, TempFrame] while it != nil and i <= high(tempFrames): tempFrames[i].procname = it.procname tempFrames[i].line = it.line diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 7ef26a7c1..186349152 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -354,6 +354,12 @@ elif defined(gogc): cast[PGenericSeq](result).reserved = len cast[PGenericSeq](result).elemSize = typ.base.size + proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = + result = newObj(typ, cap * typ.base.size + GenericSeqSize) + cast[PGenericSeq](result).len = 0 + cast[PGenericSeq](result).reserved = cap + cast[PGenericSeq](result).elemSize = typ.base.size + proc growObj(old: pointer, newsize: int): pointer = # the Go GC doesn't have a realloc var @@ -447,6 +453,7 @@ elif defined(nogc) and defined(useMalloc): result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len + proc newObjNoInit(typ: PNimType, size: int): pointer = result = alloc(size) @@ -506,6 +513,7 @@ elif defined(nogc): result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len + proc growObj(old: pointer, newsize: int): pointer = result = realloc(old, newsize) @@ -545,4 +553,10 @@ else: else: include "system/gc" +when not declared(nimNewSeqOfCap): + proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = + result = newObj(typ, addInt(mulInt(cap, typ.base.size), GenericSeqSize)) + cast[PGenericSeq](result).len = 0 + cast[PGenericSeq](result).reserved = cap + {.pop.} diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index d587d772f..cc96bcba8 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -39,6 +39,9 @@ proc getCurrentDir(): string = builtin proc rawExec(cmd: string): int {.tags: [ExecIOEffect], raises: [OSError].} = builtin +proc warningImpl(arg, orig: string) = discard +proc hintImpl(arg, orig: string) = discard + proc paramStr*(i: int): string = ## Retrieves the ``i``'th command line parameter. builtin @@ -52,6 +55,31 @@ proc switch*(key: string, val="") = ## example ``switch("checks", "on")``. builtin +proc warning*(name: string; val: bool) = + ## Disables or enables a specific warning. + let v = if val: "on" else: "off" + warningImpl(name & "]:" & v, "warning[" & name & "]:" & v) + +proc hint*(name: string; val: bool) = + ## Disables or enables a specific hint. + let v = if val: "on" else: "off" + hintImpl(name & "]:" & v, "hint[" & name & "]:" & v) + +proc patchFile*(package, filename, replacement: string) = + ## Overrides the location of a given file belonging to the + ## passed package. + ## If the ``replacement`` is not an absolute path, the path + ## is interpreted to be local to the Nimscript file that contains + ## the call to ``patchFile``, Nim's ``--path`` is not used at all + ## to resolve the filename! + ## + ## Example: + ## + ## .. code-block:: nim + ## + ## patchFile("stdlib", "asyncdispatch", "patches/replacement") + discard + proc getCommand*(): string = ## Gets the Nim command that the compiler has been invoked with, for example ## "c", "js", "build", "help". diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index de26f52d9..e864253b6 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -150,8 +150,9 @@ elif defined(windows): #VirtualFree(p, size, MEM_DECOMMIT) elif hostOS == "standalone": + const StandaloneHeapSize {.intdefine.}: int = 1024 * PageSize var - theHeap: array[1024*PageSize, float64] # 'float64' for alignment + theHeap: array[StandaloneHeapSize, float64] # 'float64' for alignment bumpPointer = cast[int](addr theHeap) proc osAllocPages(size: int): pointer {.inline.} = diff --git a/lib/system/profiler.nim b/lib/system/profiler.nim index ae8ff4e19..c00a7d53a 100644 --- a/lib/system/profiler.nim +++ b/lib/system/profiler.nim @@ -19,7 +19,7 @@ const MaxTraceLen = 20 # tracking the last 20 calls is enough type - StackTrace* = array [0..MaxTraceLen-1, cstring] + StackTrace* = array[0..MaxTraceLen-1, cstring] ProfilerHook* = proc (st: StackTrace) {.nimcall.} {.deprecated: [TStackTrace: StackTrace, TProfilerHook: ProfilerHook].} diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 4da4781ef..cf7d6d7a9 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -16,7 +16,7 @@ proc reprInt(x: int64): string {.compilerproc.} = return $x proc reprFloat(x: float): string {.compilerproc.} = return $x proc reprPointer(x: pointer): string {.compilerproc.} = - var buf: array [0..59, char] + var buf: array[0..59, char] discard c_sprintf(buf, "%p", x) return $buf @@ -24,7 +24,7 @@ proc `$`(x: uint64): string = if x == 0: result = "0" else: - var buf: array [60, char] + var buf: array[60, char] var i = 0 var n = x while n != 0: @@ -73,23 +73,20 @@ proc reprChar(x: char): string {.compilerRtl.} = add result, "\'" proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = - # we read an 'int' but this may have been too large, so mask the other bits: - let b = (sizeof(int)-typ.size)*8 # bits - let m = 1 shl (b-1) # mask - var o = e and ((1 shl b)-1) # clear upper bits - o = (o xor m) - m # sign extend - # XXX we need a proper narrowing based on signedness here - #e and ((1 shl (typ.size*8)) - 1) + ## Return string representation for enumeration values + var n = typ.node if ntfEnumHole notin typ.flags: - if o <% typ.node.len: - return $typ.node.sons[o].name + let o = e - n.sons[0].offset + if o >= 0 and o <% typ.node.len: + return $n.sons[o].name else: # ugh we need a slow linear search: - var n = typ.node var s = n.sons for i in 0 .. n.len-1: - if s[i].offset == o: return $s[i].name - result = $o & " (invalid data!)" + if s[i].offset == e: + return $s[i].name + + result = $e & " (invalid data!)" type PByteArray = ptr array[0.. 0xffff, int8] diff --git a/lib/system/sets.nim b/lib/system/sets.nim index 66877de30..53d222468 100644 --- a/lib/system/sets.nim +++ b/lib/system/sets.nim @@ -10,7 +10,7 @@ # set handling type - NimSet = array [0..4*2048-1, uint8] + NimSet = array[0..4*2048-1, uint8] {.deprecated: [TNimSet: NimSet].} proc countBits32(n: int32): int {.compilerproc.} = diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index 6622a4f64..3e9657ce0 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -271,7 +271,7 @@ else: importc: "freopen", nodecl.} const - FormatOpen: array [FileMode, string] = ["rb", "wb", "w+b", "r+b", "ab"] + FormatOpen: array[FileMode, string] = ["rb", "wb", "w+b", "r+b", "ab"] #"rt", "wt", "w+t", "r+t", "at" # we always use binary here as for Nim the OS line ending # should not be translated. diff --git a/lib/system/threads.nim b/lib/system/threads.nim index 8505202b5..583e3ae86 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -24,7 +24,7 @@ ## import locks ## ## var -## thr: array [0..4, Thread[tuple[a,b: int]]] +## thr: array[0..4, Thread[tuple[a,b: int]]] ## L: Lock ## ## proc threadFunc(interval: tuple[a,b: int]) {.thread.} = @@ -51,7 +51,7 @@ const when defined(windows): type - SysThread = Handle + SysThread* = Handle WinThreadProc = proc (x: pointer): int32 {.stdcall.} {.deprecated: [TSysThread: SysThread, TWinThreadProc: WinThreadProc].} @@ -123,7 +123,7 @@ else: type Time = int type - SysThread {.importc: "pthread_t", header: "<sys/types.h>", + SysThread* {.importc: "pthread_t", header: "<sys/types.h>", final, pure.} = object Pthread_attr {.importc: "pthread_attr_t", header: "<sys/types.h>", final, pure.} = object @@ -198,7 +198,7 @@ when emulatedThreadVars: # allocations are needed. Currently less than 7K are used on a 64bit machine. # We use ``float`` for proper alignment: type - ThreadLocalStorage = array [0..1_000, float] + ThreadLocalStorage = array[0..1_000, float] PGcThread = ptr GcThread GcThread {.pure, inheritable.} = object @@ -379,6 +379,10 @@ proc running*[TArg](t: Thread[TArg]): bool {.inline.} = ## returns true if `t` is running. result = t.dataFn != nil +proc handle*[TArg](t: Thread[TArg]): SysThread {.inline.} = + ## returns the thread handle of `t`. + result = t.sys + when hostOS == "windows": proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = ## waits for the thread `t` to finish. diff --git a/lib/system/widestrs.nim b/lib/system/widestrs.nim index 6ad0cfd58..578bebe80 100644 --- a/lib/system/widestrs.nim +++ b/lib/system/widestrs.nim @@ -73,11 +73,11 @@ template fastRuneAt(s: cstring, i: int, result: expr, doInc = true) = result = 0xFFFD when doInc: inc(i) -iterator runes(s: cstring): int = +iterator runes(s: cstring, L: int): int = var i = 0 result: int - while s[i] != '\0': + while i < L: fastRuneAt(s, i, result, true) yield result @@ -85,7 +85,7 @@ proc newWideCString*(source: cstring, L: int): WideCString = unsafeNew(result, L * 4 + 2) #result = cast[wideCString](alloc(L * 4 + 2)) var d = 0 - for ch in runes(source): + for ch in runes(source, L): if ch <=% UNI_MAX_BMP: if ch >=% UNI_SUR_HIGH_START and ch <=% UNI_SUR_LOW_END: result[d] = UNI_REPLACEMENT_CHAR diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim new file mode 100644 index 000000000..162ac5e08 --- /dev/null +++ b/lib/upcoming/asyncdispatch.nim @@ -0,0 +1,2154 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +include "system/inclrtl" + +import os, oids, tables, strutils, macros, times, heapqueue + +import nativesockets, net, queues + +export Port, SocketFlag + +#{.injectStmt: newGcInvariant().} + +## AsyncDispatch +## ************* +## +## This module implements asynchronous IO. This includes a dispatcher, +## a ``Future`` type implementation, and an ``async`` macro which allows +## asynchronous code to be written in a synchronous style with the ``await`` +## keyword. +## +## The dispatcher acts as a kind of event loop. You must call ``poll`` on it +## (or a function which does so for you such as ``waitFor`` or ``runForever``) +## in order to poll for any outstanding events. The underlying implementation +## is based on epoll on Linux, IO Completion Ports on Windows and select on +## other operating systems. +## +## The ``poll`` function will not, on its own, return any events. Instead +## an appropriate ``Future`` object will be completed. A ``Future`` is a +## type which holds a value which is not yet available, but which *may* be +## available in the future. You can check whether a future is finished +## by using the ``finished`` function. When a future is finished it means that +## either the value that it holds is now available or it holds an error instead. +## The latter situation occurs when the operation to complete a future fails +## with an exception. You can distinguish between the two situations with the +## ``failed`` function. +## +## Future objects can also store a callback procedure which will be called +## automatically once the future completes. +## +## Futures therefore can be thought of as an implementation of the proactor +## pattern. In this +## pattern you make a request for an action, and once that action is fulfilled +## a future is completed with the result of that action. Requests can be +## made by calling the appropriate functions. For example: calling the ``recv`` +## function will create a request for some data to be read from a socket. The +## future which the ``recv`` function returns will then complete once the +## requested amount of data is read **or** an exception occurs. +## +## Code to read some data from a socket may look something like this: +## +## .. code-block::nim +## var future = socket.recv(100) +## future.callback = +## proc () = +## echo(future.read) +## +## All asynchronous functions returning a ``Future`` will not block. They +## will not however return immediately. An asynchronous function will have +## code which will be executed before an asynchronous request is made, in most +## cases this code sets up the request. +## +## In the above example, the ``recv`` function will return a brand new +## ``Future`` instance once the request for data to be read from the socket +## is made. This ``Future`` instance will complete once the requested amount +## of data is read, in this case it is 100 bytes. The second line sets a +## callback on this future which will be called once the future completes. +## All the callback does is write the data stored in the future to ``stdout``. +## The ``read`` function is used for this and it checks whether the future +## completes with an error for you (if it did it will simply raise the +## error), if there is no error however it returns the value of the future. +## +## Asynchronous procedures +## ----------------------- +## +## Asynchronous procedures remove the pain of working with callbacks. They do +## this by allowing you to write asynchronous code the same way as you would +## write synchronous code. +## +## An asynchronous procedure is marked using the ``{.async.}`` pragma. +## When marking a procedure with the ``{.async.}`` pragma it must have a +## ``Future[T]`` return type or no return type at all. If you do not specify +## a return type then ``Future[void]`` is assumed. +## +## Inside asynchronous procedures ``await`` can be used to call any +## procedures which return a +## ``Future``; this includes asynchronous procedures. When a procedure is +## "awaited", the asynchronous procedure it is awaited in will +## suspend its execution +## until the awaited procedure's Future completes. At which point the +## asynchronous procedure will resume its execution. During the period +## when an asynchronous procedure is suspended other asynchronous procedures +## will be run by the dispatcher. +## +## The ``await`` call may be used in many contexts. It can be used on the right +## hand side of a variable declaration: ``var data = await socket.recv(100)``, +## in which case the variable will be set to the value of the future +## automatically. It can be used to await a ``Future`` object, and it can +## be used to await a procedure returning a ``Future[void]``: +## ``await socket.send("foobar")``. +## +## Discarding futures +## ------------------ +## +## Futures should **never** be discarded. This is because they may contain +## errors. If you do not care for the result of a Future then you should +## use the ``asyncCheck`` procedure instead of the ``discard`` keyword. +## +## Examples +## -------- +## +## For examples take a look at the documentation for the modules implementing +## asynchronous IO. A good place to start is the +## `asyncnet module <asyncnet.html>`_. +## +## Limitations/Bugs +## ---------------- +## +## * The effect system (``raises: []``) does not work with async procedures. +## * Can't await in a ``except`` body +## * Forward declarations for async procs are broken, +## link includes workaround: https://github.com/nim-lang/Nim/issues/3182. +## * FutureVar[T] needs to be completed manually. + +# TODO: Check if yielded future is nil and throw a more meaningful exception + +# -- Futures + +type + FutureBase* = ref object of RootObj ## Untyped future. + cb: proc () {.closure,gcsafe.} + finished: bool + error*: ref Exception ## Stored exception + errorStackTrace*: string + when not defined(release): + stackTrace: string ## For debugging purposes only. + id: int + fromProc: string + + Future*[T] = ref object of FutureBase ## Typed future. + value: T ## Stored value + + FutureVar*[T] = distinct Future[T] + + FutureError* = object of Exception + cause*: FutureBase + +{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} + +when not defined(release): + var currentID = 0 + +proc callSoon*(cbproc: proc ()) {.gcsafe.} + +proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = + ## Creates a new future. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + new(result) + result.finished = false + when not defined(release): + result.stackTrace = getStackTrace() + result.id = currentID + result.fromProc = fromProc + currentID.inc() + +proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] = + ## Create a new ``FutureVar``. This Future type is ideally suited for + ## situations where you want to avoid unnecessary allocations of Futures. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + result = FutureVar[T](newFuture[T](fromProc)) + +proc clean*[T](future: FutureVar[T]) = + ## Resets the ``finished`` status of ``future``. + Future[T](future).finished = false + Future[T](future).error = nil + +proc checkFinished[T](future: Future[T]) = + ## Checks whether `future` is finished. If it is then raises a + ## ``FutureError``. + when not defined(release): + if future.finished: + var msg = "" + msg.add("An attempt was made to complete a Future more than once. ") + msg.add("Details:") + msg.add("\n Future ID: " & $future.id) + msg.add("\n Created in proc: " & future.fromProc) + msg.add("\n Stack trace to moment of creation:") + msg.add("\n" & indent(future.stackTrace.strip(), 4)) + when T is string: + msg.add("\n Contents (string): ") + msg.add("\n" & indent(future.value.repr, 4)) + msg.add("\n Stack trace to moment of secondary completion:") + msg.add("\n" & indent(getStackTrace().strip(), 4)) + var err = newException(FutureError, msg) + err.cause = future + raise err + +proc complete*[T](future: Future[T], val: T) = + ## Completes ``future`` with value ``val``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.value = val + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*(future: Future[void]) = + ## Completes a void ``future``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*[T](future: FutureVar[T]) = + ## Completes a ``FutureVar``. + template fut: expr = Future[T](future) + checkFinished(fut) + assert(fut.error == nil) + fut.finished = true + if fut.cb != nil: + fut.cb() + +proc fail*[T](future: Future[T], error: ref Exception) = + ## Completes ``future`` with ``error``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + future.finished = true + future.error = error + future.errorStackTrace = + if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) + if future.cb != nil: + future.cb() + else: + # This is to prevent exceptions from being silently ignored when a future + # is discarded. + # TODO: This may turn out to be a bad idea. + # Turns out this is a bad idea. + #raise error + discard + +proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + ## + ## **Note**: You most likely want the other ``callback`` setter which + ## passes ``future`` as a param to the callback. + future.cb = cb + if future.finished: + callSoon(future.cb) + +proc `callback=`*[T](future: Future[T], + cb: proc (future: Future[T]) {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + future.callback = proc () = cb(future) + +proc injectStacktrace[T](future: Future[T]) = + # TODO: Come up with something better. + when not defined(release): + var msg = "" + msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") + + if not future.errorStackTrace.isNil and future.errorStackTrace != "": + msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) + else: + msg.add("\n Empty or nil stack trace.") + future.error.msg.add(msg) + +proc read*[T](future: Future[T]): T = + ## Retrieves the value of ``future``. Future must be finished otherwise + ## this function will fail with a ``ValueError`` exception. + ## + ## If the result of the future is an error then that error will be raised. + if future.finished: + if future.error != nil: + injectStacktrace(future) + raise future.error + when T isnot void: + return future.value + else: + # TODO: Make a custom exception type for this? + raise newException(ValueError, "Future still in progress.") + +proc readError*[T](future: Future[T]): ref Exception = + ## Retrieves the exception stored in ``future``. + ## + ## An ``ValueError`` exception will be thrown if no exception exists + ## in the specified Future. + if future.error != nil: return future.error + else: + raise newException(ValueError, "No error in future.") + +proc mget*[T](future: FutureVar[T]): var T = + ## Returns a mutable value stored in ``future``. + ## + ## Unlike ``read``, this function will not raise an exception if the + ## Future has not been finished. + result = Future[T](future).value + +proc finished*[T](future: Future[T]): bool = + ## Determines whether ``future`` has completed. + ## + ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. + future.finished + +proc failed*(future: FutureBase): bool = + ## Determines whether ``future`` completed with an error. + return future.error != nil + +proc asyncCheck*[T](future: Future[T]) = + ## Sets a callback on ``future`` which raises an exception if the future + ## finished with an error. + ## + ## This should be used instead of ``discard`` to discard void futures. + future.callback = + proc () = + if future.failed: + injectStacktrace(future) + raise future.error + +proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once both ``fut1`` and ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`and`") + fut1.callback = + proc () = + if fut2.finished: retFuture.complete() + fut2.callback = + proc () = + if fut1.finished: retFuture.complete() + return retFuture + +proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once either ``fut1`` or ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`or`") + proc cb() = + if not retFuture.finished: retFuture.complete() + fut1.callback = cb + fut2.callback = cb + return retFuture + +proc all*[T](futs: varargs[Future[T]]): auto = + ## Returns a future which will complete once + ## all futures in ``futs`` complete. + ## + ## If the awaited futures are not ``Future[void]``, the returned future + ## will hold the values of all awaited futures in a sequence. + ## + ## If the awaited futures *are* ``Future[void]``, + ## this proc returns ``Future[void]``. + + when T is void: + var + retFuture = newFuture[void]("asyncdispatch.all") + completedFutures = 0 + + let totalFutures = len(futs) + + for fut in futs: + fut.callback = proc(f: Future[T]) = + inc(completedFutures) + + if completedFutures == totalFutures: + retFuture.complete() + + return retFuture + + else: + var + retFuture = newFuture[seq[T]]("asyncdispatch.all") + retValues = newSeq[T](len(futs)) + completedFutures = 0 + + for i, fut in futs: + proc setCallback(i: int) = + fut.callback = proc(f: Future[T]) = + retValues[i] = f.read() + inc(completedFutures) + + if completedFutures == len(retValues): + retFuture.complete(retValues) + + setCallback(i) + + return retFuture + +type + PDispatcherBase = ref object of RootRef + timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]] + callbacks: Queue[proc ()] + +proc processTimers(p: PDispatcherBase) {.inline.} = + while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt: + p.timers.pop().fut.complete() + +proc processPendingCallbacks(p: PDispatcherBase) = + while p.callbacks.len > 0: + var cb = p.callbacks.dequeue() + cb() + +proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = + # If dispatcher has active timers this proc returns the timeout + # of the nearest timer. Returns `timeout` otherwise. + result = timeout + if p.timers.len > 0: + let timerTimeout = p.timers[0].finishAt + let curTime = epochTime() + if timeout == -1 or (curTime + (timeout / 1000)) > timerTimeout: + result = int((timerTimeout - curTime) * 1000) + if result < 0: result = 0 + +when defined(windows) or defined(nimdoc): + import winlean, sets, hashes + type + CompletionKey = ULONG_PTR + + CompletionData* = object + fd*: AsyncFD # TODO: Rename this. + cb*: proc (fd: AsyncFD, bytesTransferred: Dword, + errcode: OSErrorCode) {.closure,gcsafe.} + cell*: ForeignCell # we need this `cell` to protect our `cb` environment, + # when using RegisterWaitForSingleObject, because + # waiting is done in different thread. + + PDispatcher* = ref object of PDispatcherBase + ioPort: Handle + handles: HashSet[AsyncFD] + + CustomOverlapped = object of OVERLAPPED + data*: CompletionData + + PCustomOverlapped* = ref CustomOverlapped + + AsyncFD* = distinct int + + PostCallbackData = object + ioPort: Handle + handleFd: AsyncFD + waitFd: Handle + ovl: PCustomOverlapped + PostCallbackDataPtr = ptr PostCallbackData + + AsyncEventImpl = object + hEvent: Handle + hWaiter: Handle + pcd: PostCallbackDataPtr + AsyncEvent* = ptr AsyncEventImpl + + Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} + {.deprecated: [TCompletionKey: CompletionKey, TAsyncFD: AsyncFD, + TCustomOverlapped: CustomOverlapped, TCompletionData: CompletionData].} + + proc hash(x: AsyncFD): Hash {.borrow.} + proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow.} + + proc newDispatcher*(): PDispatcher = + ## Creates a new Dispatcher instance. + new result + result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) + result.handles = initSet[AsyncFD]() + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + proc getGlobalDispatcher*(): PDispatcher = + ## Retrieves the global thread-local dispatcher. + if gDisp.isNil: gDisp = newDispatcher() + result = gDisp + + proc register*(fd: AsyncFD) = + ## Registers ``fd`` with the dispatcher. + let p = getGlobalDispatcher() + if createIoCompletionPort(fd.Handle, p.ioPort, + cast[CompletionKey](fd), 1) == 0: + raiseOSError(osLastError()) + p.handles.incl(fd) + + proc verifyPresence(fd: AsyncFD) = + ## Ensures that file descriptor has been registered with the dispatcher. + let p = getGlobalDispatcher() + if fd notin p.handles: + raise newException(ValueError, + "Operation performed on a socket which has not been registered with" & + " the dispatcher yet.") + + proc poll*(timeout = 500) = + ## Waits for completion events and processes them. + let p = getGlobalDispatcher() + if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: + raise newException(ValueError, + "No handles or timers registered in dispatcher.") + + let at = p.adjustedTimeout(timeout) + var llTimeout = + if at == -1: winlean.INFINITE + else: at.int32 + + var lpNumberOfBytesTransferred: Dword + var lpCompletionKey: ULONG_PTR + var customOverlapped: PCustomOverlapped + let res = getQueuedCompletionStatus(p.ioPort, + addr lpNumberOfBytesTransferred, addr lpCompletionKey, + cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool + + # http://stackoverflow.com/a/12277264/492186 + # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html + if res: + # This is useful for ensuring the reliability of the overlapped struct. + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, OSErrorCode(-1)) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + + GC_unref(customOverlapped) + else: + let errCode = osLastError() + if customOverlapped != nil: + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, errCode) + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) + else: + if errCode.int32 == WAIT_TIMEOUT: + # Timed out + discard + else: raiseOSError(errCode) + + # Timer processing. + processTimers(p) + # Callback queue processing + processPendingCallbacks(p) + + var acceptEx*: WSAPROC_ACCEPTEX + var connectEx*: WSAPROC_CONNECTEX + var getAcceptExSockAddrs*: WSAPROC_GETACCEPTEXSOCKADDRS + + proc initPointer(s: SocketHandle, fun: var pointer, guid: var GUID): bool = + # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c + var bytesRet: Dword + fun = nil + result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid, + sizeof(GUID).Dword, addr fun, sizeof(pointer).Dword, + addr bytesRet, nil, nil) == 0 + + proc initAll() = + let dummySock = newNativeSocket() + if dummySock == INVALID_SOCKET: + raiseOSError(osLastError()) + var fun: pointer = nil + if not initPointer(dummySock, fun, WSAID_CONNECTEX): + raiseOSError(osLastError()) + connectEx = cast[WSAPROC_CONNECTEX](fun) + if not initPointer(dummySock, fun, WSAID_ACCEPTEX): + raiseOSError(osLastError()) + acceptEx = cast[WSAPROC_ACCEPTEX](fun) + if not initPointer(dummySock, fun, WSAID_GETACCEPTEXSOCKADDRS): + raiseOSError(osLastError()) + getAcceptExSockAddrs = cast[WSAPROC_GETACCEPTEXSOCKADDRS](fun) + close(dummySock) + + proc connect*(socket: AsyncFD, address: string, port: Port, + domain = nativesockets.AF_INET): Future[void] = + ## Connects ``socket`` to server at ``address:port``. + ## + ## Returns a ``Future`` which will complete when the connection succeeds + ## or an error occurs. + verifyPresence(socket) + var retFuture = newFuture[void]("connect") + # Apparently ``ConnectEx`` expects the socket to be initially bound: + var saddr: Sockaddr_in + saddr.sin_family = int16(toInt(domain)) + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(socket.SocketHandle, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) < 0'i32: + raiseOSError(osLastError()) + + var aiList = getAddrInfo(address, port, domain) + var success = false + var lastError: OSErrorCode + var it = aiList + while it != nil: + # "the OVERLAPPED structure must remain valid until the I/O completes" + # http://blogs.msdn.com/b/oldnewthing/archive/2011/02/02/10123392.aspx + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + var ret = connectEx(socket.SocketHandle, it.ai_addr, + sizeof(Sockaddr_in).cint, nil, 0, nil, + cast[POVERLAPPED](ol)) + if ret: + # Request to connect completed immediately. + success = true + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + break + else: + lastError = osLastError() + if lastError.int32 == ERROR_IO_PENDING: + # In this case ``ol`` will be deallocated in ``poll``. + success = true + break + else: + GC_unref(ol) + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + return retFuture + + proc recv*(socket: AsyncFD, size: int, + flags = {SocketFlag.SafeDisconn}): Future[string] = + ## Reads **up to** ``size`` bytes from ``socket``. Returned future will + ## complete once all the data requested is read, a part of the data has been + ## read, or the socket has disconnected in which case the future will + ## complete with a value of ``""``. + ## + ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + + + # Things to note: + # * When WSARecv completes immediately then ``bytesReceived`` is very + # unreliable. + # * Still need to implement message-oriented socket disconnection, + # '\0' in the message currently signifies a socket disconnect. Who + # knows what will happen when someone sends that to our socket. + verifyPresence(socket) + assert SocketFlag.Peek notin flags, "Peek not supported on Windows." + + var retFuture = newFuture[string]("recv") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](alloc0(size)) + dataBuf.len = size.ULONG + + var bytesReceived: Dword + var flagsio = flags.toOSFlags().Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + if bytesCount == 0 and dataBuf.buf[0] == '\0': + retFuture.complete("") + else: + var data = newString(bytesCount) + assert bytesCount <= size + copyMem(addr data[0], addr dataBuf.buf[0], bytesCount) + retFuture.complete($data) + else: + if flags.isDisconnectionError(errcode): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + if dataBuf.buf != nil: + dealloc dataBuf.buf + dataBuf.buf = nil + ) + + let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + addr flagsio, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + if dataBuf.buf != nil: + dealloc dataBuf.buf + dataBuf.buf = nil + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + var data = newString(bytesReceived) + assert bytesReceived <= size + copyMem(addr data[0], addr dataBuf.buf[0], bytesReceived) + retFuture.complete($data) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete("") + return retFuture + + proc recvInto*(socket: AsyncFD, buf: cstring, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must + ## at least be of that size. Returned future will complete once all the + ## data requested is read, a part of the data has been read, or the socket + ## has disconnected in which case the future will complete with a value of + ## ``0``. + ## + ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + + + # Things to note: + # * When WSARecv completes immediately then ``bytesReceived`` is very + # unreliable. + # * Still need to implement message-oriented socket disconnection, + # '\0' in the message currently signifies a socket disconnect. Who + # knows what will happen when someone sends that to our socket. + verifyPresence(socket) + assert SocketFlag.Peek notin flags, "Peek not supported on Windows." + + var retFuture = newFuture[int]("recvInto") + + #buf[] = '\0' + var dataBuf: TWSABuf + dataBuf.buf = buf + dataBuf.len = size.ULONG + + var bytesReceived: Dword + var flagsio = flags.toOSFlags().Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + if bytesCount == 0 and dataBuf.buf[0] == '\0': + retFuture.complete(0) + else: + retFuture.complete(bytesCount) + else: + if flags.isDisconnectionError(errcode): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + if dataBuf.buf != nil: + dataBuf.buf = nil + ) + + let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + addr flagsio, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + if dataBuf.buf != nil: + dataBuf.buf = nil + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) + return retFuture + + proc send*(socket: AsyncFD, data: string, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to ``socket``. The returned future will complete once all + ## data has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("send") + + var dataBuf: TWSABuf + dataBuf.buf = data # since this is not used in a callback, this is fine + dataBuf.len = data.len.ULONG + + var bytesReceived, lowFlags: Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + if flags.isDisconnectionError(errcode): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + lowFlags, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: Socklen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to specified destination ``saddr``, using + ## socket ``socket``. The returned future will complete once all data + ## has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("sendTo") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](data) + dataBuf.len = size.ULONG + var bytesSent = 0.Dword + var lowFlags = 0.Dword + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen: cint = cint(saddrLen) + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, + lowFlags, cast[ptr SockAddr](addr(staddr[0])), + stalen, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``buf``, which must + ## be at least of size ``size``, address of datagram's sender will be + ## stored into ``saddr`` and ``saddrLen``. Returned future will complete + ## once one datagram has been received, and will return size of packet + ## received. + verifyPresence(socket) + var retFuture = newFuture[int]("recvFromInto") + + var dataBuf = TWSABuf(buf: cast[cstring](data), len: size.ULONG) + + var bytesReceived = 0.Dword + var lowFlags = 0.Dword + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount <= size + retFuture.complete(bytesCount) + else: + # datagram sockets don't have disconnection, + # so we can just raise an exception + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, + addr bytesReceived, addr lowFlags, + saddr, cast[ptr cint](saddrLen), + cast[POVERLAPPED](ol), nil) + if res == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) + return retFuture + + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): + Future[tuple[address: string, client: AsyncFD]] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection and the remote address of the client. + ## The future will complete when the connection is successfully accepted. + ## + ## The resulting client socket is automatically registered to the + ## dispatcher. + ## + ## The ``accept`` call may result in an error if the connecting socket + ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## flag is specified then this error will not be raised and instead + ## accept will be called again. + verifyPresence(socket) + var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") + + var clientSock = newNativeSocket() + if clientSock == osInvalidSocket: raiseOSError(osLastError()) + + const lpOutputLen = 1024 + var lpOutputBuf = newString(lpOutputLen) + var dwBytesReceived: Dword + let dwReceiveDataLength = 0.Dword # We don't want any data to be read. + let dwLocalAddressLength = Dword(sizeof (Sockaddr_in) + 16) + let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16) + + template completeAccept(): stmt {.immediate, dirty.} = + var listenSock = socket + let setoptRet = setsockopt(clientSock, SOL_SOCKET, + SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, + sizeof(listenSock).SockLen) + if setoptRet != 0: raiseOSError(osLastError()) + + var localSockaddr, remoteSockaddr: ptr SockAddr + var localLen, remoteLen: int32 + getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, + dwLocalAddressLength, dwRemoteAddressLength, + addr localSockaddr, addr localLen, + addr remoteSockaddr, addr remoteLen) + register(clientSock.AsyncFD) + # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 + retFuture.complete( + (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr), + client: clientSock.AsyncFD) + ) + + template failAccept(errcode): stmt = + if flags.isDisconnectionError(errcode): + var newAcceptFut = acceptAddr(socket, flags) + newAcceptFut.callback = + proc () = + if newAcceptFut.failed: + retFuture.fail(newAcceptFut.readError) + else: + retFuture.complete(newAcceptFut.read) + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + completeAccept() + else: + failAccept(errcode) + ) + + # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx + let ret = acceptEx(socket.SocketHandle, clientSock, addr lpOutputBuf[0], + dwReceiveDataLength, + dwLocalAddressLength, + dwRemoteAddressLength, + addr dwBytesReceived, cast[POVERLAPPED](ol)) + + if not ret: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + failAccept(err) + GC_unref(ol) + else: + completeAccept() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + + return retFuture + + proc newAsyncNativeSocket*(domain, sockType, protocol: cint): AsyncFD = + ## Creates a new socket and registers it with the dispatcher implicitly. + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + register(result) + + proc newAsyncNativeSocket*(domain: Domain = nativesockets.AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP): AsyncFD = + ## Creates a new socket and registers it with the dispatcher implicitly. + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + register(result) + + proc closeSocket*(socket: AsyncFD) = + ## Closes a socket and ensures that it is unregistered. + socket.SocketHandle.close() + getGlobalDispatcher().handles.excl(socket) + + proc unregister*(fd: AsyncFD) = + ## Unregisters ``fd``. + getGlobalDispatcher().handles.excl(fd) + + {.push stackTrace:off.} + proc waitableCallback(param: pointer, + timerOrWaitFired: WINBOOL): void {.stdcall.} = + var p = cast[PostCallbackDataPtr](param) + discard postQueuedCompletionStatus(p.ioPort, timerOrWaitFired.Dword, + ULONG_PTR(p.handleFd), + cast[pointer](p.ovl)) + {.pop.} + + template registerWaitableEvent(mask) = + let p = getGlobalDispatcher() + var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword + var hEvent = wsaCreateEvent() + if hEvent == 0: + raiseOSError(osLastError()) + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + pcd.ioPort = p.ioPort + pcd.handleFd = fd + var ol = PCustomOverlapped() + GC_ref(ol) + + ol.data = CompletionData(fd: fd, cb: + proc(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + # we excluding our `fd` because cb(fd) can register own handler + # for this `fd` + p.handles.excl(fd) + # unregisterWait() is called before callback, because appropriate + # winsockets function can re-enable event. + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms741576(v=vs.85).aspx + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + if cb(fd): + # callback returned `true`, so we free all allocated resources + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + # pcd.ovl will be unrefed in poll(). + else: + # callback returned `false` we need to continue + if p.handles.contains(fd): + # new callback was already registered with `fd`, so we free all + # allocated resources. This happens because in callback `cb` + # addRead/addWrite was called with same `fd`. + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + else: + # we need to include `fd` again + p.handles.incl(fd) + # and register WaitForSingleObject again + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + # pcd.ovl will be unrefed in poll() + discard wsaCloseEvent(hEvent) + deallocShared(cast[pointer](pcd)) + raiseOSError(osLastError()) + else: + # we ref pcd.ovl one more time, because it will be unrefed in + # poll() + GC_ref(pcd.ovl) + ) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + # This is main part of `hacky way` is using WSAEventSelect, so `hEvent` + # will be signaled when appropriate `mask` events will be triggered. + if wsaEventSelect(fd.SocketHandle, hEvent, mask) != 0: + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + p.handles.incl(fd) + + proc addRead*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for read availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addRead` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.recv() + ## or asyncdispatch.accept(), because they are using IOCP, please use + ## nativesockets.recv() and nativesockets.accept() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `read` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) + + proc addWrite*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for write availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addWrite` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.send() + ## or asyncdispatch.connect(), because they are using IOCP, please use + ## nativesockets.send() and nativesockets.connect() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `write` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_WRITE or FD_CONNECT or FD_CLOSE) + + template registerWaitableHandle(p, hEvent, flags, pcd, handleCallback) = + let handleFD = AsyncFD(hEvent) + pcd.ioPort = p.ioPort + pcd.handleFd = handleFD + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: handleFD, cb: handleCallback) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + p.handles.incl(handleFD) + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Registers callback ``cb`` to be called when timer expired. + ## ``timeout`` - timeout value in milliseconds. + ## ``oneshot`` - `true`, to generate only one timeout event, `false`, to + ## generate timeout events periodically. + + doAssert(timeout > 0) + let p = getGlobalDispatcher() + + var hEvent = createEvent(nil, 1, 0, nil) + if hEvent == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + if oneshot: flags = flags or WT_EXECUTEONLYONCE + + proc timercb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + let res = cb(fd) + if res or oneshot: + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + discard closeHandle(hEvent) + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + + registerWaitableHandle(p, hEvent, flags, pcd, timercb) + + proc addProcess*(pid: int, cb: Callback) = + ## Registers callback ``cb`` to be called when process with pid ``pid`` + ## exited. + let p = getGlobalDispatcher() + let procFlags = SYNCHRONIZE + var hProcess = openProcess(procFlags, 0, pid.Dword) + if hProcess == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + + proc proccb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + discard closeHandle(hProcess) + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + discard cb(fd) + + registerWaitableHandle(p, hProcess, flags, pcd, proccb) + + proc newEvent*(): AsyncEvent = + ## Creates new ``AsyncEvent`` object. + var sa = SECURITY_ATTRIBUTES( + nLength: sizeof(SECURITY_ATTRIBUTES).cint, + bInheritHandle: 1 + ) + var event = createEvent(addr(sa), 0'i32, 0'i32, nil) + if event == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + result = cast[AsyncEvent](allocShared0(sizeof(AsyncEventImpl))) + + proc setEvent*(ev: AsyncEvent) = + ## Set event ``ev`` to signaled state. + if setEvent(ev.hEvent) == 0: + raiseOSError(osLastError()) + + proc close*(ev: AsyncEvent) = + ## Closes event ``ev``. + if ev.hWaiter != 0: + let p = getGlobalDispatcher() + if unregisterWait(ev.hWaiter) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + p.handles.excl(AsyncFD(ev.hEvent)) + + if closeHandle(ev.hEvent) == 0: + raiseOSError(osLastError()) + deallocShared(cast[pointer](ev)) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Registers callback ``cb`` to be called when ``ev`` will be signaled + if ev.hWaiter != 0: + raise newException(ValueError, "Event is already registered!") + + let p = getGlobalDispatcher() + let hEvent = ev.hEvent + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + + proc eventcb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if cb(fd): + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + ev.hWaiter = 0 + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + + registerWaitableHandle(p, hEvent, flags, pcd, eventcb) + ev.hWaiter = pcd.waitFd + + initAll() +else: + import ioselectors + when defined(windows): + import winlean + const + EINTR = WSAEINPROGRESS + EINPROGRESS = WSAEINPROGRESS + EWOULDBLOCK = WSAEWOULDBLOCK + EAGAIN = EINPROGRESS + MSG_NOSIGNAL = 0 + else: + from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, + MSG_NOSIGNAL + + const supportedPlatform = defined(linux) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(macosx) + + type + AsyncFD* = distinct cint + Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} + + AsyncData = object + readCB: Callback + writeCB: Callback + + AsyncEvent = SelectEvent + + PDispatcher* = ref object of PDispatcherBase + selector: Selector[AsyncData] + {.deprecated: [TAsyncFD: AsyncFD, TCallback: Callback].} + + proc `==`*(x, y: AsyncFD): bool {.borrow.} + + proc newDispatcher*(): PDispatcher = + new result + result.selector = newSelector[AsyncData]() + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + proc getGlobalDispatcher*(): PDispatcher = + if gDisp.isNil: gDisp = newDispatcher() + result = gDisp + + proc register*(fd: AsyncFD) = + let p = getGlobalDispatcher() + var data = AsyncData() + p.selector.registerHandle(fd.SocketHandle, {}, data) + + proc newAsyncNativeSocket*(domain: cint, sockType: cint, + protocol: cint): AsyncFD = + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + when defined(macosx): + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) + register(result) + + proc newAsyncNativeSocket*(domain: Domain = AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP): AsyncFD = + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + when defined(macosx): + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) + register(result) + + proc closeSocket*(sock: AsyncFD) = + let disp = getGlobalDispatcher() + disp.selector.unregister(sock.SocketHandle) + sock.SocketHandle.close() + + proc unregister*(fd: AsyncFD) = + getGlobalDispatcher().selector.unregister(fd.SocketHandle) + + # proc unregister*(ev: AsyncEvent) = + # getGlobalDispatcher().selector.unregister(SelectEvent(ev)) + + proc addRead*(fd: AsyncFD, cb: Callback) = + let p = getGlobalDispatcher() + withData(p.selector, fd.SocketHandle, adata) do: + adata.readCB = cb + do: + raise newException(ValueError, "File descriptor not registered.") + p.selector.updateHandle(fd.SocketHandle, {Event.Read}) + + proc addWrite*(fd: AsyncFD, cb: Callback) = + let p = getGlobalDispatcher() + withData(p.selector, fd.SocketHandle, adata) do: + adata.writeCB = cb + do: + raise newException(ValueError, "File descriptor not registered.") + p.selector.updateHandle(fd.SocketHandle, {Event.Write}) + + proc poll*(timeout = 500) = + var keys: array[64, ReadyKey[AsyncData]] + + let p = getGlobalDispatcher() + when supportedPlatform: + let customSet = {Event.Timer, Event.Signal, Event.Process, + Event.Vnode, Event.User} + + if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: + raise newException(ValueError, + "No handles or timers registered in dispatcher.") + + if not p.selector.isEmpty(): + var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys) + var i = 0 + while i < count: + var update = false + var fd = keys[i].fd.SocketHandle + let events = keys[i].events + + if Event.Read in events: + let cb = keys[i].data.readCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.readCB == cb: + adata.readCB = nil + update = true + + if Event.Write in events: + let cb = keys[i].data.writeCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.writeCB == cb: + adata.writeCB = nil + update = true + + when supportedPlatform: + if (customSet * events) != {}: + let cb = keys[i].data.readCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.readCB == cb: + adata.readCB = nil + p.selector.unregister(fd) + + if update: + var newEvents: set[Event] = {} + p.selector.withData(fd, adata) do: + if adata.readCB != nil: incl(newEvents, Event.Read) + if adata.writeCB != nil: incl(newEvents, Event.Write) + p.selector.updateHandle(fd, newEvents) + inc(i) + + # Timer processing. + processTimers(p) + # Callback queue processing + processPendingCallbacks(p) + + proc connect*(socket: AsyncFD, address: string, port: Port, + domain = AF_INET): Future[void] = + var retFuture = newFuture[void]("connect") + + proc cb(fd: AsyncFD): bool = + var ret = SocketHandle(fd).getSockOptInt(cint(SOL_SOCKET), cint(SO_ERROR)) + if ret == 0: + # We have connected. + retFuture.complete() + return true + elif ret == EINTR: + # interrupted, keep waiting + return false + else: + retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + return true + + assert getSockDomain(socket.SocketHandle) == domain + var aiList = getAddrInfo(address, port, domain) + var success = false + var lastError: OSErrorCode + var it = aiList + while it != nil: + var ret = connect(socket.SocketHandle, it.ai_addr, it.ai_addrlen.Socklen) + if ret == 0: + # Request to connect completed immediately. + success = true + retFuture.complete() + break + else: + lastError = osLastError() + if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: + success = true + addWrite(socket, cb) + break + else: + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + return retFuture + + proc recv*(socket: AsyncFD, size: int, + flags = {SocketFlag.SafeDisconn}): Future[string] = + var retFuture = newFuture[string]("recv") + + var readBuffer = newString(size) + + proc cb(sock: AsyncFD): bool = + result = true + let res = recv(sock.SocketHandle, addr readBuffer[0], size.cint, + flags.toOSFlags()) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + elif res == 0: + # Disconnected + retFuture.complete("") + else: + readBuffer.setLen(res) + retFuture.complete(readBuffer) + # TODO: The following causes a massive slowdown. + #if not cb(socket): + addRead(socket, cb) + return retFuture + + proc recvInto*(socket: AsyncFD, buf: cstring, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = + var retFuture = newFuture[int]("recvInto") + + proc cb(sock: AsyncFD): bool = + result = true + let res = recv(sock.SocketHandle, buf, size.cint, + flags.toOSFlags()) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete(res) + # TODO: The following causes a massive slowdown. + #if not cb(socket): + addRead(socket, cb) + return retFuture + + proc send*(socket: AsyncFD, data: string, + flags = {SocketFlag.SafeDisconn}): Future[void] = + var retFuture = newFuture[void]("send") + + var written = 0 + + proc cb(sock: AsyncFD): bool = + result = true + let netSize = data.len-written + var d = data.cstring + let res = send(sock.SocketHandle, addr d[written], netSize.cint, + MSG_NOSIGNAL) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + written.inc(res) + if res != netSize: + result = false # We still have data to send. + else: + retFuture.complete() + # TODO: The following causes crashes. + #if not cb(socket): + addWrite(socket, cb) + return retFuture + + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: SockLen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` of size ``size`` in bytes to specified destination + ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``. + ## The returned future will complete once all data has been sent. + var retFuture = newFuture[void]("sendTo") + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen = saddrLen + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + proc cb(sock: AsyncFD): bool = + result = true + let res = sendto(sock.SocketHandle, data, size, MSG_NOSIGNAL, + cast[ptr SockAddr](addr(staddr[0])), stalen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete() + + addWrite(socket, cb) + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``data``, which must + ## be at least of size ``size`` in bytes, address of datagram's sender + ## will be stored into ``saddr`` and ``saddrLen``. Returned future will + ## complete once one datagram has been received, and will return size + ## of packet received. + var retFuture = newFuture[int]("recvFromInto") + proc cb(sock: AsyncFD): bool = + result = true + let res = recvfrom(sock.SocketHandle, data, size.cint, flags.toOSFlags(), + saddr, saddrLen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false + else: + retFuture.complete(res) + addRead(socket, cb) + return retFuture + + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): + Future[tuple[address: string, client: AsyncFD]] = + var retFuture = newFuture[tuple[address: string, + client: AsyncFD]]("acceptAddr") + proc cb(sock: AsyncFD): bool = + result = true + var sockAddress: Sockaddr_storage + var addrLen = sizeof(sockAddress).Socklen + var client = accept(sock.SocketHandle, + cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + if client == osInvalidSocket: + let lastError = osLastError() + assert lastError.int32 notin {EWOULDBLOCK, EAGAIN} + if lastError.int32 == EINTR: + return false + else: + if flags.isDisconnectionError(lastError): + return false + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + register(client.AsyncFD) + retFuture.complete((getAddrString(cast[ptr SockAddr](addr sockAddress)), + client.AsyncFD)) + addRead(socket, cb) + return retFuture + + when supportedPlatform: + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Start watching for timeout expiration, and then call the + ## callback ``cb``. + ## ``timeout`` - time in milliseconds, + ## ``oneshot`` - if ``true`` only one event will be dispatched, + ## if ``false`` continuous events every ``timeout`` milliseconds. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerTimer(timeout, oneshot, data) + + proc addSignal*(signal: int, cb: Callback) = + ## Start watching signal ``signal``, and when signal appears, call the + ## callback ``cb``. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerSignal(signal, data) + + proc addProcess*(pid: int, cb: Callback) = + ## Start watching for process exit with pid ``pid``, and then call + ## the callback ``cb``. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerProcess(pid, data) + + proc newAsyncEvent*(): AsyncEvent = + ## Creates new ``AsyncEvent``. + result = AsyncEvent(ioselectors.newSelectEvent()) + + proc setEvent*(ev: AsyncEvent) = + ## Sets new ``AsyncEvent`` to signaled state. + setEvent(SelectEvent(ev)) + + proc close*(ev: AsyncEvent) = + ## Closes ``AsyncEvent`` + close(SelectEvent(ev)) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Start watching for event ``ev``, and call callback ``cb``, when + ## ev will be set to signaled state. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerEvent(SelectEvent(ev), data) + +proc sleepAsync*(ms: int): Future[void] = + ## Suspends the execution of the current async procedure for the next + ## ``ms`` milliseconds. + var retFuture = newFuture[void]("sleepAsync") + let p = getGlobalDispatcher() + p.timers.push((epochTime() + (ms / 1000), retFuture)) + return retFuture + +proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] = + ## Returns a future which will complete once ``fut`` completes or after + ## ``timeout`` milliseconds has elapsed. + ## + ## If ``fut`` completes first the returned future will hold true, + ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned + ## future will hold false. + + var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") + var timeoutFuture = sleepAsync(timeout) + fut.callback = + proc () = + if not retFuture.finished: retFuture.complete(true) + timeoutFuture.callback = + proc () = + if not retFuture.finished: retFuture.complete(false) + return retFuture + +proc accept*(socket: AsyncFD, + flags = {SocketFlag.SafeDisconn}): Future[AsyncFD] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection. + ## The future will complete when the connection is successfully accepted. + var retFut = newFuture[AsyncFD]("accept") + var fut = acceptAddr(socket, flags) + fut.callback = + proc (future: Future[tuple[address: string, client: AsyncFD]]) = + assert future.finished + if future.failed: + retFut.fail(future.error) + else: + retFut.complete(future.read.client) + return retFut + +# -- Await Macro + +proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = + # Skips a nest of StmtList's. + result = node + if node[0].kind == nnkStmtList: + result = skipUntilStmtList(node[0]) + +proc skipStmtList(node: NimNode): NimNode {.compileTime.} = + result = node + if node[0].kind == nnkStmtList: + result = node[0] + +template createCb(retFutureSym, iteratorNameSym, + name: expr): stmt {.immediate.} = + var nameIterVar = iteratorNameSym + #{.push stackTrace: off.} + proc cb {.closure,gcsafe.} = + try: + if not nameIterVar.finished: + var next = nameIterVar() + if next == nil: + assert retFutureSym.finished, "Async procedure's (" & + name & ") return Future was not finished." + else: + next.callback = cb + except: + if retFutureSym.finished: + # Take a look at tasyncexceptions for the bug which this fixes. + # That test explains it better than I can here. + raise + else: + retFutureSym.fail(getCurrentException()) + cb() + #{.pop.} +proc generateExceptionCheck(futSym, + tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = + if tryStmt.kind == nnkNilLit: + result = rootReceiver + else: + var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[] + let errorNode = newDotExpr(futSym, newIdentNode("error")) + for i in 1 .. <tryStmt.len: + let exceptBranch = tryStmt[i] + if exceptBranch[0].kind == nnkStmtList: + exceptionChecks.add((newIdentNode("true"), exceptBranch[0])) + else: + var exceptIdentCount = 0 + var ifCond: NimNode + for i in 0 .. <exceptBranch.len: + let child = exceptBranch[i] + if child.kind == nnkIdent: + let cond = infix(errorNode, "of", child) + if exceptIdentCount == 0: + ifCond = cond + else: + ifCond = infix(ifCond, "or", cond) + else: + break + exceptIdentCount.inc + + expectKind(exceptBranch[exceptIdentCount], nnkStmtList) + exceptionChecks.add((ifCond, exceptBranch[exceptIdentCount])) + # -> -> else: raise futSym.error + exceptionChecks.add((newIdentNode("true"), + newNimNode(nnkRaiseStmt).add(errorNode))) + # Read the future if there is no error. + # -> else: futSym.read + let elseNode = newNimNode(nnkElse, fromNode) + elseNode.add newNimNode(nnkStmtList, fromNode) + elseNode[0].add rootReceiver + + let ifBody = newStmtList() + ifBody.add newCall(newIdentNode("setCurrentException"), errorNode) + ifBody.add newIfStmt(exceptionChecks) + ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit()) + + result = newIfStmt( + (newDotExpr(futSym, newIdentNode("failed")), ifBody) + ) + result.add elseNode + +template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, + rootReceiver: expr, fromNode: NimNode) = + ## Params: + ## futureVarNode: The NimNode which is a symbol identifying the Future[T] + ## variable to yield. + ## fromNode: Used for better debug information (to give context). + ## valueReceiver: The node which defines an expression that retrieves the + ## future's value. + ## + ## rootReceiver: ??? TODO + # -> yield future<x> + result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) + # -> future<x>.read + valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) + result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, + fromNode) + +template createVar(result: var NimNode, futSymName: string, + asyncProc: NimNode, + valueReceiver, rootReceiver: expr, + fromNode: NimNode) = + result = newNimNode(nnkStmtList, fromNode) + var futSym = genSym(nskVar, "future") + result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y + useVar(result, futSym, valueReceiver, rootReceiver, fromNode) + +proc processBody(node, retFutureSym: NimNode, + subTypeIsVoid: bool, + tryStmt: NimNode): NimNode {.compileTime.} = + #echo(node.treeRepr) + result = node + case node.kind + of nnkReturnStmt: + result = newNimNode(nnkStmtList, node) + if node[0].kind == nnkEmpty: + if not subTypeIsVoid: + result.add newCall(newIdentNode("complete"), retFutureSym, + newIdentNode("result")) + else: + result.add newCall(newIdentNode("complete"), retFutureSym) + else: + result.add newCall(newIdentNode("complete"), retFutureSym, + node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt)) + + result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) + return # Don't process the children of this return stmt + of nnkCommand, nnkCall: + if node[0].kind == nnkIdent and node[0].ident == !"await": + case node[1].kind + of nnkIdent, nnkInfix, nnkDotExpr: + # await x + # await x or y + result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x + of nnkCall, nnkCommand: + # await foo(p, x) + # await foo p, x + var futureValue: NimNode + result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, + futureValue, node) + else: + error("Invalid node kind in 'await', got: " & $node[1].kind) + elif node.len > 1 and node[1].kind == nnkCommand and + node[1][0].kind == nnkIdent and node[1][0].ident == !"await": + # foo await x + var newCommand = node + result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], + newCommand, node) + + of nnkVarSection, nnkLetSection: + case node[0][2].kind + of nnkCommand: + if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": + # var x = await y + var newVarSection = node # TODO: Should this use copyNimNode? + result.createVar("future" & $node[0][0].ident, node[0][2][1], + newVarSection[0][2], newVarSection, node) + else: discard + of nnkAsgn: + case node[1].kind + of nnkCommand: + if node[1][0].ident == !"await": + # x = await y + var newAsgn = node + result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node) + else: discard + of nnkDiscardStmt: + # discard await x + if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and + node[0][0].ident == !"await": + var newDiscard = node + result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], + newDiscard[0], newDiscard, node) + of nnkTryStmt: + # try: await x; except: ... + result = newNimNode(nnkStmtList, node) + template wrapInTry(n, tryBody: expr) = + var temp = n + n[0] = tryBody + tryBody = temp + + # Transform ``except`` body. + # TODO: Could we perform some ``await`` transformation here to get it + # working in ``except``? + tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, nil) + + proc processForTry(n: NimNode, i: var int, + res: NimNode): bool {.compileTime.} = + ## Transforms the body of the tryStmt. Does not transform the + ## body in ``except``. + ## Returns true if the tryStmt node was transformed into an ifStmt. + result = false + var skipped = n.skipStmtList() + while i < skipped.len: + var processed = processBody(skipped[i], retFutureSym, + subTypeIsVoid, n) + + # Check if we transformed the node into an exception check. + # This suggests skipped[i] contains ``await``. + if processed.kind != skipped[i].kind or processed.len != skipped[i].len: + processed = processed.skipUntilStmtList() + expectKind(processed, nnkStmtList) + expectKind(processed[2][1], nnkElse) + i.inc + + if not processForTry(n, i, processed[2][1][0]): + # We need to wrap the nnkElse nodes back into a tryStmt. + # As they are executed if an exception does not happen + # inside the awaited future. + # The following code will wrap the nodes inside the + # original tryStmt. + wrapInTry(n, processed[2][1][0]) + + res.add processed + result = true + else: + res.add skipped[i] + i.inc + var i = 0 + if not processForTry(node, i, result): + # If the tryStmt hasn't been transformed we can just put the body + # back into it. + wrapInTry(node, result) + return + else: discard + + for i in 0 .. <result.len: + result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, nil) + +proc getName(node: NimNode): string {.compileTime.} = + case node.kind + of nnkPostfix: + return $node[1].ident + of nnkIdent: + return $node.ident + of nnkEmpty: + return "anonymous" + else: + error("Unknown name.") + +proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = + ## This macro transforms a single procedure into a closure iterator. + ## The ``async`` macro supports a stmtList holding multiple async procedures. + if prc.kind notin {nnkProcDef, nnkLambda}: + error("Cannot transform this node kind into an async proc." & + " Proc definition or lambda node expected.") + + hint("Processing " & prc[0].getName & " as an async proc.") + + let returnType = prc[3][0] + var baseType: NimNode + # Verify that the return type is a Future[T] + if returnType.kind == nnkBracketExpr: + let fut = repr(returnType[0]) + if fut != "Future": + error("Expected return type of 'Future' got '" & fut & "'") + baseType = returnType[1] + elif returnType.kind in nnkCallKinds and $returnType[0] == "[]": + let fut = repr(returnType[1]) + if fut != "Future": + error("Expected return type of 'Future' got '" & fut & "'") + baseType = returnType[2] + elif returnType.kind == nnkEmpty: + baseType = returnType + else: + error("Expected return type of 'Future' got '" & repr(returnType) & "'") + + let subtypeIsVoid = returnType.kind == nnkEmpty or + (baseType.kind == nnkIdent and returnType[1].ident == !"void") + + var outerProcBody = newNimNode(nnkStmtList, prc[6]) + + # -> var retFuture = newFuture[T]() + var retFutureSym = genSym(nskVar, "retFuture") + var subRetType = + if returnType.kind == nnkEmpty: newIdentNode("void") + else: baseType + outerProcBody.add( + newVarStmt(retFutureSym, + newCall( + newNimNode(nnkBracketExpr, prc[6]).add( + newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. + subRetType), + newLit(prc[0].getName)))) # Get type from return type of this proc + + # -> iterator nameIter(): FutureBase {.closure.} = + # -> {.push warning[resultshadowed]: off.} + # -> var result: T + # -> {.pop.} + # -> <proc_body> + # -> complete(retFuture, result) + var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") + var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) + if not subtypeIsVoid: + procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), + newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( + newIdentNode("warning"), newIdentNode("resultshadowed")), + newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} + + procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( + newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T + + procBody.insert(2, newNimNode(nnkPragma).add( + newIdentNode("pop"))) # -> {.pop.}) + + procBody.add( + newCall(newIdentNode("complete"), + retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) + else: + # -> complete(retFuture) + procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + + var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], + procBody, nnkIteratorDef) + closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) + outerProcBody.add(closureIterator) + + # -> createCb(retFuture) + #var cbName = newIdentNode("cb") + var procCb = newCall(bindSym"createCb", retFutureSym, iteratorNameSym, + newStrLitNode(prc[0].getName)) + outerProcBody.add procCb + + # -> return retFuture + outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) + + result = prc + + # Remove the 'async' pragma. + for i in 0 .. <result[4].len: + if result[4][i].kind == nnkIdent and result[4][i].ident == !"async": + result[4].del(i) + result[4] = newEmptyNode() + if subtypeIsVoid: + # Add discardable pragma. + if returnType.kind == nnkEmpty: + # Add Future[void] + result[3][0] = parseExpr("Future[void]") + + result[6] = outerProcBody + + #echo(treeRepr(result)) + #if prc[0].getName == "testInfix": + # echo(toStrLit(result)) + +macro async*(prc: stmt): stmt {.immediate.} = + ## Macro which processes async procedures into the appropriate + ## iterators and yield statements. + if prc.kind == nnkStmtList: + for oneProc in prc: + result = newStmtList() + result.add asyncSingleProc(oneProc) + else: + result = asyncSingleProc(prc) + +proc recvLine*(socket: AsyncFD): Future[string] {.async.} = + ## Reads a line of data from ``socket``. Returned future will complete once + ## a full line is read or an error occurs. + ## + ## If a full line is read ``\r\L`` is not + ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## will be set to it. + ## + ## If the socket is disconnected, ``line`` will be set to ``""``. + ## + ## If the socket is disconnected in the middle of a line (before ``\r\L`` + ## is read) then line will be set to ``""``. + ## The partial line **will be lost**. + ## + ## **Warning**: This assumes that lines are delimited by ``\r\L``. + ## + ## **Note**: This procedure is mostly used for testing. You likely want to + ## use ``asyncnet.recvLine`` instead. + + template addNLIfEmpty(): stmt = + if result.len == 0: + result.add("\c\L") + + result = "" + var c = "" + while true: + c = await recv(socket, 1) + if c.len == 0: + return "" + if c == "\r": + c = await recv(socket, 1) + assert c == "\l" + addNLIfEmpty() + return + elif c == "\L": + addNLIfEmpty() + return + add(result, c) + +proc callSoon*(cbproc: proc ()) = + ## Schedule `cbproc` to be called as soon as possible. + ## The callback is called when control returns to the event loop. + getGlobalDispatcher().callbacks.enqueue(cbproc) + +proc runForever*() = + ## Begins a never ending global dispatcher poll loop. + while true: + poll() + +proc waitFor*[T](fut: Future[T]): T = + ## **Blocks** the current thread until the specified future completes. + while not fut.finished: + poll() + + fut.read diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 2989e5ac9..b766ead9c 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -455,7 +455,7 @@ type sin_zero*: array[0..7, char] In6_addr* {.importc: "IN6_ADDR", header: "winsock2.h".} = object - bytes*: array[0..15, char] + bytes* {.importc: "u.Byte".}: array[0..15, char] Sockaddr_in6* {.importc: "SOCKADDR_IN6", header: "ws2tcpip.h".} = object @@ -750,7 +750,7 @@ type D1*: int32 D2*: int16 D3*: int16 - D4*: array [0..7, int8] + D4*: array[0..7, int8] {.deprecated: [TOVERLAPPED: OVERLAPPED, TGUID: GUID].} const @@ -825,11 +825,23 @@ proc WSARecv*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. stdcall, importc: "WSARecv", dynlib: "Ws2_32.dll".} +proc WSARecvFrom*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, + bytesReceived: PDWORD, flags: PDWORD, name: ptr SockAddr, + namelen: ptr cint, lpOverlapped: POVERLAPPED, + completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. + stdcall, importc: "WSARecvFrom", dynlib: "Ws2_32.dll".} + proc WSASend*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, bytesSent: PDWORD, flags: DWORD, lpOverlapped: POVERLAPPED, completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. stdcall, importc: "WSASend", dynlib: "Ws2_32.dll".} +proc WSASendTo*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, + bytesSent: PDWORD, flags: DWORD, name: ptr SockAddr, + namelen: cint, lpOverlapped: POVERLAPPED, + completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. + stdcall, importc: "WSASendTo", dynlib: "Ws2_32.dll".} + proc get_osfhandle*(fd:FileHandle): Handle {. importc: "_get_osfhandle", header:"<io.h>".} @@ -994,7 +1006,7 @@ const FD_ALL_EVENTS* = 0x000003FF'i32 proc wsaEventSelect*(s: SocketHandle, hEventObject: Handle, - lNetworkEvents: clong): cint + lNetworkEvents: clong): cint {.stdcall, importc: "WSAEventSelect", dynlib: "ws2_32.dll".} proc wsaCreateEvent*(): Handle @@ -1004,4 +1016,23 @@ proc wsaCloseEvent*(hEvent: Handle): bool {.stdcall, importc: "WSACloseEvent", dynlib: "ws2_32.dll".} proc wsaResetEvent*(hEvent: Handle): bool - {.stdcall, importc: "WSAResetEvent", dynlib: "ws2_32.dll".} \ No newline at end of file + {.stdcall, importc: "WSAResetEvent", dynlib: "ws2_32.dll".} + +type + KEY_EVENT_RECORD* {.final, pure.} = object + eventType*: int16 + bKeyDown*: WINBOOL + wRepeatCount*: int16 + wVirtualKeyCode*: int16 + wVirtualScanCode*: int16 + uChar*: int16 + dwControlKeyState*: DWORD + +when defined(useWinAnsi): + proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, + lpNumberOfEventsRead: ptr cint): cint + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputA".} +else: + proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, + lpNumberOfEventsRead: ptr cint): cint + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} \ No newline at end of file diff --git a/tests/assert/tunittests.nim b/tests/assert/tunittests.nim index cbbfe63c6..de917511c 100644 --- a/tests/assert/tunittests.nim +++ b/tests/assert/tunittests.nim @@ -1 +1,4 @@ +discard """ +output: "" +""" import "../template/utemplates", "../closure/uclosures" diff --git a/tests/async/tasync_forward.nim b/tests/async/tasync_forward.nim new file mode 100644 index 000000000..ffb7acafd --- /dev/null +++ b/tests/async/tasync_forward.nim @@ -0,0 +1,9 @@ + +import asyncdispatch + +# bug #1970 + +proc foo {.async.} + +proc foo {.async.} = + discard diff --git a/tests/async/tgeneric_async.nim b/tests/async/tgeneric_async.nim new file mode 100644 index 000000000..af6370181 --- /dev/null +++ b/tests/async/tgeneric_async.nim @@ -0,0 +1,9 @@ + +import asyncdispatch + +when true: + # bug #2377 + proc test[T](v: T) {.async.} = + echo $v + + asyncCheck test[int](1) diff --git a/tests/async/tioselectors.nim b/tests/async/tioselectors.nim new file mode 100644 index 000000000..ed2fea84f --- /dev/null +++ b/tests/async/tioselectors.nim @@ -0,0 +1,408 @@ +discard """ + file: "tioselectors.nim" + output: "All tests passed!" +""" +import ioselectors + +const hasThreadSupport = compileOption("threads") + +template processTest(t, x: untyped) = + #stdout.write(t) + #stdout.flushFile() + if not x: echo(t & " FAILED\r\n") + +when not defined(windows): + import os, posix, osproc, nativesockets, times + + const supportedPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(linux) + + proc socket_notification_test(): bool = + proc create_test_socket(): SocketHandle = + var sock = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP) + var x: int = fcntl(sock, F_GETFL, 0) + if x == -1: raiseOSError(osLastError()) + else: + var mode = x or O_NONBLOCK + if fcntl(sock, F_SETFL, mode) == -1: + raiseOSError(osLastError()) + result = sock + + var client_message = "SERVER HELLO =>" + var server_message = "CLIENT HELLO" + var buffer : array[128, char] + + var selector = newSelector[int]() + var client_socket = create_test_socket() + var server_socket = create_test_socket() + + registerHandle(selector, server_socket, {Event.Read}, 0) + registerHandle(selector, client_socket, {Event.Write}, 0) + + var option : int32 = 1 + if setsockopt(server_socket, cint(SOL_SOCKET), cint(SO_REUSEADDR), + addr(option), sizeof(option).SockLen) < 0: + raiseOSError(osLastError()) + + var aiList = getAddrInfo("0.0.0.0", Port(13337)) + if bindAddr(server_socket, aiList.ai_addr, + aiList.ai_addrlen.Socklen) < 0'i32: + dealloc(aiList) + raiseOSError(osLastError()) + discard server_socket.listen() + dealloc(aiList) + + aiList = getAddrInfo("127.0.0.1", Port(13337)) + discard posix.connect(client_socket, aiList.ai_addr, + aiList.ai_addrlen.Socklen) + dealloc(aiList) + discard selector.select(100) + var rc1 = selector.select(100) + assert(len(rc1) == 2) + + var sockAddress: SockAddr + var addrLen = sizeof(sockAddress).Socklen + var server2_socket = accept(server_socket, + cast[ptr SockAddr](addr(sockAddress)), + addr(addrLen)) + assert(server2_socket != osInvalidSocket) + selector.registerHandle(server2_socket, {Event.Read}, 0) + + if posix.send(client_socket, addr(client_message[0]), + len(client_message), 0) == -1: + raiseOSError(osLastError()) + + selector.updateHandle(client_socket, {Event.Read}) + + var rc2 = selector.select(100) + assert(len(rc2) == 1) + + var read_count = posix.recv(server2_socket, addr (buffer[0]), 128, 0) + if read_count == -1: + raiseOSError(osLastError()) + + assert(read_count == len(client_message)) + var test1 = true + for i in 0..<read_count: + if client_message[i] != buffer[i]: + test1 = false + break + assert(test1) + + selector.updateHandle(server2_socket, {Event.Write}) + var rc3 = selector.select(0) + assert(len(rc3) == 1) + if posix.send(server2_socket, addr(server_message[0]), + len(server_message), 0) == -1: + raiseOSError(osLastError()) + selector.updateHandle(server2_socket, {Event.Read}) + + var rc4 = selector.select(100) + assert(len(rc4) == 1) + read_count = posix.recv(client_socket, addr(buffer[0]), 128, 0) + if read_count == -1: + raiseOSError(osLastError()) + + assert(read_count == len(server_message)) + var test2 = true + for i in 0..<read_count: + if server_message[i] != buffer[i]: + test2 = false + break + assert(test2) + + selector.unregister(server_socket) + selector.unregister(server2_socket) + selector.unregister(client_socket) + discard posix.close(server_socket) + discard posix.close(server2_socket) + discard posix.close(client_socket) + assert(selector.isEmpty()) + close(selector) + result = true + + proc event_notification_test(): bool = + var selector = newSelector[int]() + var event = newSelectEvent() + selector.registerEvent(event, 1) + selector.flush() + event.setEvent() + var rc1 = selector.select(0) + event.setEvent() + var rc2 = selector.select(0) + var rc3 = selector.select(0) + assert(len(rc1) == 1 and len(rc2) == 1 and len(rc3) == 0) + var ev1 = rc1[0].data + var ev2 = rc2[0].data + assert(ev1 == 1 and ev2 == 1) + selector.unregister(event) + event.close() + assert(selector.isEmpty()) + selector.close() + result = true + + when supportedPlatform: + proc timer_notification_test(): bool = + var selector = newSelector[int]() + var timer = selector.registerTimer(100, false, 0) + var rc1 = selector.select(140) + var rc2 = selector.select(140) + assert(len(rc1) == 1 and len(rc2) == 1) + selector.unregister(timer) + selector.flush() + selector.registerTimer(100, true, 0) + var rc3 = selector.select(120) + var rc4 = selector.select(120) + assert(len(rc3) == 1 and len(rc4) == 0) + assert(selector.isEmpty()) + selector.close() + result = true + + proc process_notification_test(): bool = + var selector = newSelector[int]() + var process2 = startProcess("/bin/sleep", "", ["2"], nil, + {poStdErrToStdOut, poUsePath}) + discard startProcess("/bin/sleep", "", ["1"], nil, + {poStdErrToStdOut, poUsePath}) + + selector.registerProcess(process2.processID, 0) + var rc1 = selector.select(1500) + var rc2 = selector.select(1500) + var r = len(rc1) + len(rc2) + assert(r == 1) + result = true + + proc signal_notification_test(): bool = + var sigset1n, sigset1o, sigset2n, sigset2o: Sigset + var pid = posix.getpid() + + discard sigemptyset(sigset1n) + discard sigemptyset(sigset1o) + discard sigemptyset(sigset2n) + discard sigemptyset(sigset2o) + + when hasThreadSupport: + if pthread_sigmask(SIG_BLOCK, sigset1n, sigset1o) == -1: + raiseOSError(osLastError()) + else: + if sigprocmask(SIG_BLOCK, sigset1n, sigset1o) == -1: + raiseOSError(osLastError()) + + var selector = newSelector[int]() + var s1 = selector.registerSignal(SIGUSR1, 1) + var s2 = selector.registerSignal(SIGUSR2, 2) + var s3 = selector.registerSignal(SIGTERM, 3) + selector.flush() + + discard posix.kill(pid, SIGUSR1) + discard posix.kill(pid, SIGUSR2) + discard posix.kill(pid, SIGTERM) + var rc = selector.select(0) + selector.unregister(s1) + selector.unregister(s2) + selector.unregister(s3) + + when hasThreadSupport: + if pthread_sigmask(SIG_BLOCK, sigset2n, sigset2o) == -1: + raiseOSError(osLastError()) + else: + if sigprocmask(SIG_BLOCK, sigset2n, sigset2o) == -1: + raiseOSError(osLastError()) + + assert(len(rc) == 3) + assert(rc[0].data + rc[1].data + rc[2].data == 6) # 1 + 2 + 3 + assert(equalMem(addr sigset1o, addr sigset2o, sizeof(Sigset))) + assert(selector.isEmpty()) + result = true + + when hasThreadSupport: + + var counter = 0 + + proc event_wait_thread(event: SelectEvent) {.thread.} = + var selector = newSelector[int]() + selector.registerEvent(event, 1) + selector.flush() + var rc = selector.select(1000) + if len(rc) == 1: + inc(counter) + selector.unregister(event) + assert(selector.isEmpty()) + + proc mt_event_test(): bool = + var + thr: array [0..7, Thread[SelectEvent]] + var selector = newSelector[int]() + var sock = newNativeSocket() + var event = newSelectEvent() + for i in 0..high(thr): + createThread(thr[i], event_wait_thread, event) + selector.registerHandle(sock, {Event.Read}, 1) + discard selector.select(500) + selector.unregister(sock) + event.setEvent() + joinThreads(thr) + assert(counter == 1) + result = true + + processTest("Socket notification test...", socket_notification_test()) + processTest("User event notification test...", event_notification_test()) + when hasThreadSupport: + processTest("Multithreaded user event notification test...", + mt_event_test()) + when supportedPlatform: + processTest("Timer notification test...", timer_notification_test()) + processTest("Process notification test...", process_notification_test()) + processTest("Signal notification test...", signal_notification_test()) + echo("All tests passed!") +else: + import nativesockets, winlean, os, osproc + + proc socket_notification_test(): bool = + proc create_test_socket(): SocketHandle = + var sock = newNativeSocket() + setBlocking(sock, false) + result = sock + + var client_message = "SERVER HELLO =>" + var server_message = "CLIENT HELLO" + var buffer : array[128, char] + + var selector = newSelector[int]() + var client_socket = create_test_socket() + var server_socket = create_test_socket() + + selector.registerHandle(server_socket, {Event.Read}, 0) + selector.registerHandle(client_socket, {Event.Write}, 0) + + var option : int32 = 1 + if setsockopt(server_socket, cint(SOL_SOCKET), cint(SO_REUSEADDR), + addr(option), sizeof(option).SockLen) < 0: + raiseOSError(osLastError()) + + var aiList = getAddrInfo("0.0.0.0", Port(13337)) + if bindAddr(server_socket, aiList.ai_addr, + aiList.ai_addrlen.Socklen) < 0'i32: + dealloc(aiList) + raiseOSError(osLastError()) + discard server_socket.listen() + dealloc(aiList) + + aiList = getAddrInfo("127.0.0.1", Port(13337)) + discard connect(client_socket, aiList.ai_addr, + aiList.ai_addrlen.Socklen) + dealloc(aiList) + # for some reason Windows select doesn't return both + # descriptors from first call, so we need to make 2 calls + discard selector.select(100) + var rcm = selector.select(100) + assert(len(rcm) == 2) + + var sockAddress = SockAddr() + var addrLen = sizeof(sockAddress).Socklen + var server2_socket = accept(server_socket, + cast[ptr SockAddr](addr(sockAddress)), + addr(addrLen)) + assert(server2_socket != osInvalidSocket) + selector.registerHandle(server2_socket, {Event.Read}, 0) + + if send(client_socket, cast[pointer](addr(client_message[0])), + cint(len(client_message)), 0) == -1: + raiseOSError(osLastError()) + + selector.updateHandle(client_socket, {Event.Read}) + + var rc2 = selector.select(100) + assert(len(rc2) == 1) + + var read_count = recv(server2_socket, addr (buffer[0]), 128, 0) + if read_count == -1: + raiseOSError(osLastError()) + + assert(read_count == len(client_message)) + var test1 = true + for i in 0..<read_count: + if client_message[i] != buffer[i]: + test1 = false + break + assert(test1) + + if send(server2_socket, cast[pointer](addr(server_message[0])), + cint(len(server_message)), 0) == -1: + raiseOSError(osLastError()) + + var rc3 = selector.select(0) + assert(len(rc3) == 1) + read_count = recv(client_socket, addr(buffer[0]), 128, 0) + if read_count == -1: + raiseOSError(osLastError()) + + assert(read_count == len(server_message)) + var test2 = true + for i in 0..<read_count: + if server_message[i] != buffer[i]: + test2 = false + break + assert(test2) + + selector.unregister(server_socket) + selector.unregister(server2_socket) + selector.unregister(client_socket) + close(server_socket) + close(server2_socket) + close(client_socket) + assert(selector.isEmpty()) + close(selector) + result = true + + proc event_notification_test(): bool = + var selector = newSelector[int]() + var event = newSelectEvent() + selector.registerEvent(event, 1) + selector.flush() + event.setEvent() + var rc1 = selector.select(0) + event.setEvent() + var rc2 = selector.select(0) + var rc3 = selector.select(0) + assert(len(rc1) == 1 and len(rc2) == 1 and len(rc3) == 0) + var ev1 = rc1[0].data + var ev2 = rc2[0].data + assert(ev1 == 1 and ev2 == 1) + selector.unregister(event) + event.close() + assert(selector.isEmpty()) + selector.close() + result = true + + when hasThreadSupport: + var counter = 0 + + proc event_wait_thread(event: SelectEvent) {.thread.} = + var selector = newSelector[int]() + selector.registerEvent(event, 1) + selector.flush() + var rc = selector.select(500) + if len(rc) == 1: + inc(counter) + selector.unregister(event) + assert(selector.isEmpty()) + + proc mt_event_test(): bool = + var thr: array [0..7, Thread[SelectEvent]] + var event = newSelectEvent() + for i in 0..high(thr): + createThread(thr[i], event_wait_thread, event) + event.setEvent() + joinThreads(thr) + assert(counter == 1) + result = true + + processTest("Socket notification test...", socket_notification_test()) + processTest("User event notification test...", event_notification_test()) + when hasThreadSupport: + processTest("Multithreaded user event notification test...", + mt_event_test()) + echo("All tests passed!") diff --git a/tests/async/tioselectors.nim.cfg b/tests/async/tioselectors.nim.cfg new file mode 100644 index 000000000..b1b896858 --- /dev/null +++ b/tests/async/tioselectors.nim.cfg @@ -0,0 +1 @@ +threads:on -d:threadsafe diff --git a/tests/async/tnewasyncudp.nim b/tests/async/tnewasyncudp.nim new file mode 100644 index 000000000..7025fa20d --- /dev/null +++ b/tests/async/tnewasyncudp.nim @@ -0,0 +1,102 @@ +discard """ + file: "tnewasyncudp.nim" + output: "5000" +""" +import asyncdispatch, nativesockets, net, strutils, os + +when defined(windows): + import winlean +else: + import posix + +var msgCount = 0 +var recvCount = 0 + +const + messagesToSend = 100 + swarmSize = 50 + serverPort = 10333 + +var + sendports = 0 + recvports = 0 + +proc saveSendingPort(port: int) = + sendports = sendports + port + +proc saveReceivedPort(port: int) = + recvports = recvports + port + +proc prepareAddress(intaddr: uint32, intport: uint16): ptr Sockaddr_in = + result = cast[ptr Sockaddr_in](alloc0(sizeof(Sockaddr_in))) + when defined(windows): + result.sin_family = toInt(nativesockets.AF_INET).int16 + else: + result.sin_family = toInt(nativesockets.AF_INET) + result.sin_port = htons(intport) + result.sin_addr.s_addr = htonl(intaddr) + +proc launchSwarm(name: ptr SockAddr) {.async.} = + var i = 0 + var k = 0 + while i < swarmSize: + var peeraddr = prepareAddress(INADDR_ANY, 0) + var sock = newAsyncNativeSocket(nativesockets.AF_INET, + nativesockets.SOCK_DGRAM, + Protocol.IPPROTO_UDP) + if bindAddr(sock.SocketHandle, cast[ptr SockAddr](peeraddr), + sizeof(Sockaddr_in).Socklen) < 0'i32: + raiseOSError(osLastError()) + let sockport = getSockName(sock.SocketHandle).int + k = 0 + while k < messagesToSend: + var message = "Message " & $(i * messagesToSend + k) + await sendTo(sock, addr message[0], len(message), + name, sizeof(Sockaddr_in).SockLen) + saveSendingPort(sockport) + inc(k) + closeSocket(sock) + inc(i) + +proc readMessages(server: AsyncFD) {.async.} = + var buffer: array[16384, char] + var slen = sizeof(Sockaddr_in).SockLen + var saddr = Sockaddr_in() + var maxResponses = (swarmSize * messagesToSend) + + var i = 0 + while i < maxResponses: + zeroMem(addr(buffer[0]), 16384) + zeroMem(cast[pointer](addr(saddr)), sizeof(Sockaddr_in)) + var size = await recvFromInto(server, cast[cstring](addr buffer[0]), + 16384, cast[ptr SockAddr](addr(saddr)), + addr(slen)) + size = 0 + var grammString = $buffer + if grammString.startswith("Message ") and + saddr.sin_addr.s_addr == 0x100007F: + inc(msgCount) + saveReceivedPort(ntohs(saddr.sin_port).int) + inc(recvCount) + inc(i) + +proc createServer() {.async.} = + var name = prepareAddress(INADDR_ANY, serverPort) + var server = newAsyncNativeSocket(nativesockets.AF_INET, + nativesockets.SOCK_DGRAM, + Protocol.IPPROTO_UDP) + if bindAddr(server.SocketHandle, cast[ptr SockAddr](name), + sizeof(Sockaddr_in).Socklen) < 0'i32: + raiseOSError(osLastError()) + asyncCheck readMessages(server) + +var name = prepareAddress(0x7F000001, serverPort) # 127.0.0.1 +asyncCheck createServer() +asyncCheck launchSwarm(cast[ptr SockAddr](name)) +while true: + poll() + if recvCount == swarmSize * messagesToSend: + break +assert msgCount == swarmSize * messagesToSend +assert sendports == recvports +echo msgCount diff --git a/tests/async/treturn_await.nim b/tests/async/treturn_await.nim new file mode 100644 index 000000000..8d266d665 --- /dev/null +++ b/tests/async/treturn_await.nim @@ -0,0 +1,23 @@ + +# bug #4371 + +import strutils, asyncdispatch, asynchttpserver + +type + List[A] = ref object + value: A + next: List[A] + StrPair* = tuple[k, v: string] + Context* = object + position*: int + accept*: bool + headers*: List[StrPair] + Handler* = proc(req: ref Request, ctx: Context): Future[Context] + +proc logging*(handler: Handler): auto = + proc h(req: ref Request, ctx: Context): Future[Context] {.async.} = + let ret = handler(req, ctx) + debugEcho "$3 $1 $2".format(req.reqMethod, req.url.path, req.hostname) + return await ret + + return h diff --git a/tests/ccgbugs/tuplecast.nim b/tests/ccgbugs/tuplecast.nim new file mode 100644 index 000000000..d60e8c490 --- /dev/null +++ b/tests/ccgbugs/tuplecast.nim @@ -0,0 +1,8 @@ + +# bug #4345 + +# only needs to compile +proc f(): tuple[a, b: uint8] = (1'u8, 2'u8) + +let a, b = f() +let c = cast[int](b) diff --git a/tests/closure/uclosures.nim b/tests/closure/uclosures.nim index 817bfec6b..f259cfeb9 100644 --- a/tests/closure/uclosures.nim +++ b/tests/closure/uclosures.nim @@ -1,12 +1,23 @@ +# This test is included from within tunittests import unittest -test "loop variables are captured by copy": +test "loop variables are captured by ref": var funcs: seq[proc (): int {.closure.}] = @[] for i in 0..10: let ii = i funcs.add do -> int: return ii * ii + check funcs[0]() == 100 + check funcs[3]() == 100 + +test "loop variables in closureScope are captured by copy": + var funcs: seq[proc (): int {.closure.}] = @[] + + for i in 0..10: + closureScope: + let ii = i + funcs.add do -> int: return ii * ii + check funcs[0]() == 0 check funcs[3]() == 9 - diff --git a/tests/converter/tconverter_with_varargs.nim b/tests/converter/tconverter_with_varargs.nim new file mode 100644 index 000000000..6d7e31e85 --- /dev/null +++ b/tests/converter/tconverter_with_varargs.nim @@ -0,0 +1,18 @@ + +# bug #888 + +type + PyRef = object + PPyRef* = ref PyRef + +converter to_py*(i: int) : PPyRef = nil + +when false: + proc to_tuple*(vals: openarray[PPyRef]): PPyRef = + discard + +proc abc(args: varargs[PPyRef]) = + #let args_tup = to_tuple(args) + discard + +abc(1, 2) diff --git a/tests/generics/t88.nim b/tests/generics/t88.nim new file mode 100644 index 000000000..93d93f063 --- /dev/null +++ b/tests/generics/t88.nim @@ -0,0 +1,25 @@ +# Issue 88 + +type + BaseClass[V] = object of RootObj + b: V + +proc new[V](t: typedesc[BaseClass], v: V): BaseClass[V] = + BaseClass[V](b: v) + +proc baseMethod[V](v: BaseClass[V]): V = v.b +proc overridedMethod[V](v: BaseClass[V]): V = v.baseMethod + +type + ChildClass[V] = object of BaseClass[V] + c: V + +proc new[V](t: typedesc[ChildClass], v1, v2: V): ChildClass[V] = + ChildClass[V](b: v1, c: v2) + +proc overridedMethod[V](v: ChildClass[V]): V = v.c + +let c = ChildClass[string].new("Base", "Child") + +assert c.baseMethod == "Base" +assert c.overridedMethod == "Child" diff --git a/tests/generics/tforward_generic.nim b/tests/generics/tforward_generic.nim new file mode 100644 index 000000000..169279cb3 --- /dev/null +++ b/tests/generics/tforward_generic.nim @@ -0,0 +1,28 @@ +discard """ + output: '''b() +720 120.0''' +""" + +# bug #3055 +proc b(t: int | string) +proc a(t: int) = b(t) +proc b(t: int | string) = echo "b()" +a(1) + +# test recursive generics still work: +proc fac[T](x: T): T = + if x == 0: return 1 + else: return fac(x-1)*x + +echo fac(6), " ", fac(5.0) + +when false: + # This still doesn't work... + # test recursive generic with forwarding: + proc fac2[T](x: T): T + + echo fac2(6), " ", fac2(5.0) + + proc fac2[T](x: T): T = + if x == 0: return 1 + else: return fac2(x-1)*x diff --git a/tests/js/tstring_assignment.nim b/tests/js/tstring_assignment.nim new file mode 100644 index 000000000..bdd93e6b5 --- /dev/null +++ b/tests/js/tstring_assignment.nim @@ -0,0 +1,11 @@ +discard """ + output: '''true''' +""" + +# bug #4471 +when true: + let s1 = "123" + var s2 = s1 + s2.setLen(0) + # fails - s1.len == 0 + echo s1.len == 3 diff --git a/tests/macros/tcomplexecho.nim b/tests/macros/tcomplexecho.nim new file mode 100644 index 000000000..f7f933c1c --- /dev/null +++ b/tests/macros/tcomplexecho.nim @@ -0,0 +1,42 @@ +discard """ + output: '''3 +OK +56 +123 +56 +61''' +""" + +import macros + +# Bug from the forum +macro addEcho1(s: untyped): stmt = + s.body.add(newCall("echo", newStrLitNode("OK"))) + result = s + +proc f1() {.addEcho1.} = + let i = 1+2 + echo i + +f1() + +# bug #537 +proc test(): seq[NimNode] {.compiletime.} = + result = @[] + result.add parseExpr("echo 56") + result.add parseExpr("echo 123") + result.add parseExpr("echo 56") + +proc foo(): seq[NimNode] {.compiletime.} = + result = @[] + result.add test() + result.add parseExpr("echo(5+56)") + +macro bar(): stmt = + result = newNimNode(nnkStmtList) + let x = foo() + for xx in x: + result.add xx + echo treeRepr(result) + +bar() diff --git a/tests/macros/tvarargsuntyped.nim b/tests/macros/tvarargsuntyped.nim new file mode 100644 index 000000000..b7d2bc001 --- /dev/null +++ b/tests/macros/tvarargsuntyped.nim @@ -0,0 +1,79 @@ +discard """ + output: '''Let's go! +(left: 2, r: 7, x: 8, height: 4, s: test, width: 3, y: 9, top: 1, g: 7, b: 8) +(left: 2, r: 7, x: 8, height: 4, s: text, width: 3, y: 9, top: 1, g: 7, b: 8) +(left: 2, r: 7, x: 8, height: 4, s: text, width: 3, y: 9, top: 4, g: 7, b: 8) +(left: 2, r: 7, x: 8, height: 4, s: test, width: 3, y: 9, top: 1, g: 7, b: 8)''' +""" + +import macros + +proc internalBar(top, left, width, height: cint, s: string, x, y: int, r,g,b: int) = + echo locals() + +# we need these dummy constructors due to the wrong implementation +# of 'varargs[untyped]' in the compiler: + +proc point(x, y: int): int = discard +proc color(r, g, b: int): int = discard +proc rect(a, b, c, d: int): int = discard + +template declareUnpackingMacro(nimname,extname) = + macro nimname(n: varargs[untyped]): untyped = + var s: string = astToStr(extname) & "(" + var first = true + echo repr n + for x in n.children: + var unpack = false + if x.kind in nnkCallKinds: + case $x[0] + of "point": + expectLen(x, 3) + unpack = true + of "rect": + expectLen(x, 5) + unpack = true + of "color": + expectLen(x, 4) + unpack = true + else: discard + if unpack: + for i in 1..<x.len: + if first: + first = false + else: + add(s, ", ") + add(s, repr(x[i])) + else: + if first: + first = false + else: + add(s, ", ") + add(s, repr(x)) + + add(s, ")") + echo s + result = parseStmt(s) + +declareUnpackingMacro(bar,internalBar) + +type MyInt = distinct int + +proc myInt(i: int): MyInt = cast[MyInt](i) + +converter toCInt(mi: MyInt): cint = cast[cint](mi) + +echo "Let's go!" + +bar(rect(1, 2, 3, 4), "test", point(8, 9), color(7,7,8)) + +bar(1,2,3,4,"text",8,9,7,7,8) + +bar(myInt(4),2,3,4,"text",8,9,7,7,8) + +let top: cint = 1 +let left: cint = 2 +let width: cint = 3 +let height: cint = 4 + +bar(rect(top, left, width, height), "test", point(8, 9), color(7,7,8)) diff --git a/tests/metatype/tautoproc.nim b/tests/metatype/tautoproc.nim index ef5377096..2c8f6a3f7 100644 --- a/tests/metatype/tautoproc.nim +++ b/tests/metatype/tautoproc.nim @@ -1,5 +1,5 @@ discard """ - output: "empty" + output: "void" """ # bug #898 diff --git a/tests/metatype/tmodulo.nim b/tests/metatype/tmodulo.nim new file mode 100644 index 000000000..08bcc7935 --- /dev/null +++ b/tests/metatype/tmodulo.nim @@ -0,0 +1,20 @@ +discard """ + output: '''1 mod 7''' +""" + +# bug #3706 + +type Modulo[M: static[int]] = distinct int + +proc modulo(a: int, M: static[int]): Modulo[M] = Modulo[M](a %% M) + +proc `+`[M: static[int]](a, b: Modulo[M]): Modulo[M] = (a.int + b.int).modulo(M) + +proc `$`*[M: static[int]](a: Modulo[M]): string = $(a.int) & " mod " & $(M) + +when isMainModule: + let + a = 3.modulo(7) + b = 5.modulo(7) + echo a + b + diff --git a/tests/metatype/tvoid_must_not_match.nim b/tests/metatype/tvoid_must_not_match.nim new file mode 100644 index 000000000..240c3f751 --- /dev/null +++ b/tests/metatype/tvoid_must_not_match.nim @@ -0,0 +1,21 @@ +discard """ + errormsg: "type mismatch: got (Future[system.int], void)" + line: 20 +""" + +type + Future[T] = object + value: T + +proc complete[T](x: T) = + echo "completed" + let y = x + + +proc complete*[T](future: var Future[T], val: T) = + future.value = val + +var a: Future[int] + +complete(a): + echo "yielding void" diff --git a/tests/method/tgeneric_methods.nim b/tests/method/tgeneric_methods.nim new file mode 100644 index 000000000..76a68fbb0 --- /dev/null +++ b/tests/method/tgeneric_methods.nim @@ -0,0 +1,24 @@ +discard """ + output: "wow2" +""" +type + First[T] = ref object of RootObj + value: T + + Second[T] = ref object of First[T] + value2: T + +method wow[T](y: int; x: First[T]) {.base.} = + echo "wow1" + +method wow[T](y: int; x: Second[T]) = + echo "wow2" + +var + x: Second[int] +new(x) + +proc takeFirst(x: First[int]) = + wow(2, x) + +takeFirst(x) diff --git a/tests/method/tmultim6.nim b/tests/method/tmultim6.nim index 97ed2845c..2c622b832 100644 --- a/tests/method/tmultim6.nim +++ b/tests/method/tmultim6.nim @@ -4,27 +4,27 @@ discard """ # Test multi methods type - TThing = object {.inheritable.} - TUnit[T] = object of TThing + Thing = object {.inheritable.} + Unit[T] = object of Thing x: T - TParticle = object of TThing + Particle = object of Thing a, b: int -method collide(a, b: TThing) {.base, inline.} = +method collide(a, b: Thing) {.base, inline.} = quit "to override!" -method collide[T](a: TThing, b: TUnit[T]) {.inline.} = +method collide[T](a: Thing, b: Unit[T]) {.inline.} = write stdout, "collide: thing, unit | " -method collide[T](a: TUnit[T], b: TThing) {.inline.} = +method collide[T](a: Unit[T], b: Thing) {.inline.} = write stdout, "collide: unit, thing | " -proc test(a, b: TThing) {.inline.} = +proc test(a, b: Thing) {.inline.} = collide(a, b) var - a: TThing - b, c: TUnit[string] -collide(b, TThing(c)) + a: Thing + b, c: Unit[string] +collide(b, Thing(c)) test(b, c) collide(a, b) diff --git a/tests/modules/texplicit_system_import.nim b/tests/modules/texplicit_system_import.nim new file mode 100644 index 000000000..bc4d018bf --- /dev/null +++ b/tests/modules/texplicit_system_import.nim @@ -0,0 +1,9 @@ +##. +import system except `+` +discard """ + errormsg: "undeclared identifier: '+'" + line: 9 +""" +# Testament requires that the initial """ occurs before the 40th byte +# in the file. No kidding... +echo 4+5 diff --git a/tests/newconfig/mymath.nim b/tests/newconfig/mymath.nim new file mode 100644 index 000000000..5668b448b --- /dev/null +++ b/tests/newconfig/mymath.nim @@ -0,0 +1,4 @@ + + +proc ln*(x: float): float = + return 0.5 diff --git a/tests/newconfig/tfoo.nim b/tests/newconfig/tfoo.nim index d593d4a75..2e10167b1 100644 --- a/tests/newconfig/tfoo.nim +++ b/tests/newconfig/tfoo.nim @@ -1,10 +1,12 @@ discard """ cmd: "nim default $file" - output: '''hello world!''' + output: '''hello world! 0.5''' msg: '''[NimScript] exec: gcc -v''' """ when not defined(definedefine): {.fatal: "wrong nim script configuration".} -echo "hello world!" +import math + +echo "hello world! ", ln 2.0 diff --git a/tests/newconfig/tfoo.nims b/tests/newconfig/tfoo.nims index 519a868d5..057c0ed92 100644 --- a/tests/newconfig/tfoo.nims +++ b/tests/newconfig/tfoo.nims @@ -8,6 +8,11 @@ import ospaths --forceBuild +warning("uninit", off) +hint("processing", off) +#--verbosity:2 +patchFile("stdlib", "math", "mymath") + task listDirs, "lists every subdirectory": for x in listDirs("."): echo "DIR ", x diff --git a/tests/osproc/passenv.nim b/tests/osproc/passenv.nim new file mode 100644 index 000000000..815f7536f --- /dev/null +++ b/tests/osproc/passenv.nim @@ -0,0 +1,32 @@ +discard """ + file: "passenv.nim" + output: "123" + targets: "c c++ objc" +""" + +import osproc, os, strtabs + +# Checks that the environment is passed correctly in startProcess +# To do that launches a copy of itself with a new environment. + +if paramCount() == 0: + # Parent process + + let env = newStringTable() + env["A"] = "1" + env["B"] = "2" + env["C"] = "3" + + let p = startProcess( + getAppFilename(), + args = @["child"], + env = env, + options = {poStdErrToStdOut, poUsePath, poParentStreams} + ) + + discard p.waitForExit + +else: + # Child process + # should output "123" + echo getEnv("A") & getEnv("B") & getEnv("C") diff --git a/tests/overload/tissue4475.nim b/tests/overload/tissue4475.nim new file mode 100644 index 000000000..34618cac5 --- /dev/null +++ b/tests/overload/tissue4475.nim @@ -0,0 +1,6 @@ +# Bug: https://github.com/nim-lang/Nim/issues/4475 +# Fix: https://github.com/nim-lang/Nim/pull/4477 + +proc test(x: varargs[string], y: int) = discard + +test(y = 1) diff --git a/tests/stdlib/thtmlparser2814.nim b/tests/stdlib/thtmlparser2814.nim new file mode 100644 index 000000000..968d390f1 --- /dev/null +++ b/tests/stdlib/thtmlparser2814.nim @@ -0,0 +1,44 @@ +discard """ + output: true +""" +import htmlparser +import xmltree +import strutils +from streams import newStringStream + + +## builds the two cases below and test that +## ``//[dd,li]`` has "<p>that</p>" as children +## +## <dl> +## <dt>this</dt> +## <dd> +## <p>that</p> +## </dd> +## </dl> + +## +## <ul> +## <li> +## <p>that</p> +## </li> +## </ul> + + +for ltype in [["dl","dd"], ["ul","li"]]: + let desc_item = if ltype[0]=="dl": "<dt>this</dt>" else: "" + let item = "$1<$2><p>that</p></$2>" % [desc_item, ltype[1]] + let list = """ <$1> + $2 +</$1> """ % [ltype[0], item] + + var errors : seq[string] = @[] + + let parseH = parseHtml(newStringStream(list),"statichtml", errors =errors) + + if $parseH.findAll(ltype[1])[0].child("p") != "<p>that</p>": + echo "case " & ltype[0] & " failed !" + quit(2) + + +echo "true" diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index cae388792..1ddaacfcb 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -1,12 +1,88 @@ -# test some things of the os module +discard """ + output: '''true +true +true +true +true +true +true +true +true +All: +__really_obscure_dir_name/are.x +__really_obscure_dir_name/created +__really_obscure_dir_name/dirs +__really_obscure_dir_name/files.q +__really_obscure_dir_name/some +__really_obscure_dir_name/test +__really_obscure_dir_name/testing.r +__really_obscure_dir_name/these.txt +Files: +__really_obscure_dir_name/are.x +__really_obscure_dir_name/files.q +__really_obscure_dir_name/testing.r +__really_obscure_dir_name/these.txt +Dirs: +__really_obscure_dir_name/created +__really_obscure_dir_name/dirs +__really_obscure_dir_name/some +__really_obscure_dir_name/test +false +false +false +false +false +false +false +false +false +''' +""" +# test os path creation, iteration, and deletion import os -proc walkDirTree(root: string) = - for k, f in walkDir(root): - case k - of pcFile, pcLinkToFile: echo(f) - of pcDir: walkDirTree(f) - of pcLinkToDir: discard +let files = @["these.txt", "are.x", "testing.r", "files.q"] +let dirs = @["some", "created", "test", "dirs"] -walkDirTree(".") +let dname = "__really_obscure_dir_name" + +createDir(dname) +echo dirExists(dname) + +# Test creating files and dirs +for dir in dirs: + createDir(dname/dir) + echo dirExists(dname/dir) + +for file in files: + let fh = open(dname/file, fmReadWrite) + fh.close() + echo fileExists(dname/file) + +echo "All:" + +for path in walkPattern(dname/"*"): + echo path + +echo "Files:" + +for path in walkFiles(dname/"*"): + echo path + +echo "Dirs:" + +for path in walkDirs(dname/"*"): + echo path + +# Test removal of files dirs +for dir in dirs: + removeDir(dname/dir) + echo dirExists(dname/dir) + +for file in files: + removeFile(dname/file) + echo fileExists(dname/file) + +removeDir(dname) +echo dirExists(dname) diff --git a/tests/stdlib/tsplit.nim b/tests/stdlib/tsplit.nim index 5a1cd2f5f..44da58aca 100644 --- a/tests/stdlib/tsplit.nim +++ b/tests/stdlib/tsplit.nim @@ -9,7 +9,7 @@ for w in split("|abc|xy|z", {'|'}): s.add("#") s.add(w) -if s == "#abc#xy#z": +if s == "##abc#xy#z": echo "true" else: echo "false" diff --git a/tests/stdlib/tunittest.nim b/tests/stdlib/tunittest.nim index 73113ac68..e87cd3508 100644 --- a/tests/stdlib/tunittest.nim +++ b/tests/stdlib/tunittest.nim @@ -83,3 +83,9 @@ suite "suite with both": test "unittest with both 2": check c == 2 + +suite "bug #4494": + test "Uniqueness check": + var tags = @[1, 2, 3, 4, 5] + check: + allIt(0..3, tags[it] != tags[it + 1]) diff --git a/tests/tuples/tconver_tuple.nim b/tests/tuples/tconver_tuple.nim new file mode 100644 index 000000000..306da77fe --- /dev/null +++ b/tests/tuples/tconver_tuple.nim @@ -0,0 +1,23 @@ +# Bug 4479 + +type + MyTuple = tuple + num: int + strings: seq[string] + ints: seq[int] + +var foo = MyTuple(( + num: 7, + strings: @[], + ints: @[], +)) + +var bar = ( + num: 7, + strings: @[], + ints: @[], +).MyTuple + +var fooUnnamed = MyTuple((7, @[], @[])) +var n = 7 +var fooSym = MyTuple((num: n, strings: @[], ints: @[])) diff --git a/tests/tuples/twrong_generic_caching.nim b/tests/tuples/twrong_generic_caching.nim new file mode 100644 index 000000000..32ef344d2 --- /dev/null +++ b/tests/tuples/twrong_generic_caching.nim @@ -0,0 +1,4 @@ + +import parsecfg + +import asynchttpserver diff --git a/tests/typerel/tclosure_nil_as_default.nim b/tests/typerel/tclosure_nil_as_default.nim new file mode 100644 index 000000000..fe9f42b14 --- /dev/null +++ b/tests/typerel/tclosure_nil_as_default.nim @@ -0,0 +1,11 @@ + +# bug #4328 +type + foo[T] = object + z: T + +proc test[T](x: foo[T], p: proc(a: T) = nil) = + discard + +var d: foo[int] +d.test() # <- param omitted diff --git a/tests/typerel/temptynode.nim b/tests/typerel/temptynode.nim index 91e45f3ca..b32b16121 100644 --- a/tests/typerel/temptynode.nim +++ b/tests/typerel/temptynode.nim @@ -1,6 +1,6 @@ discard """ line: 16 - errormsg: "type mismatch: got (empty)" + errormsg: "type mismatch: got (void)" """ # bug #950 diff --git a/tests/typerel/tgeneric_subtype_regression.nim b/tests/typerel/tgeneric_subtype_regression.nim new file mode 100644 index 000000000..e279c0ad4 --- /dev/null +++ b/tests/typerel/tgeneric_subtype_regression.nim @@ -0,0 +1,19 @@ +discard """ + errormsg: "type mismatch: got (FooRef[system.string])" + line: 15 +""" + +# bug #4478 + +type + Foo[T] = object + FooRef[T] = ref Foo[T] + +proc takeFoo[T](foo: Foo[T]): int = discard + +proc g(x: FooRef[string]) = + echo x.takeFoo() != 8 + +var x: FooRef[string] + +g(x) diff --git a/tests/types/tassignemptytuple.nim b/tests/types/tassignemptytuple.nim new file mode 100644 index 000000000..bdfc653a5 --- /dev/null +++ b/tests/types/tassignemptytuple.nim @@ -0,0 +1,11 @@ +discard """ + file: "tassignemptytuple.nim" + line: 11 + errormsg: "cannot infer the type of the tuple" +""" + +var + foo: seq[int] + bar: tuple[a: seq[int], b: set[char]] + +(foo, bar) = (@[], (@[], {})) diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim new file mode 100644 index 000000000..b7112b099 --- /dev/null +++ b/tests/vm/tvmmisc.nim @@ -0,0 +1,16 @@ + + +# bug #4462 +import macros + +proc foo(t: typedesc) {.compileTime.} = + echo getType(t).treeRepr + +static: + foo(int) + +# #4412 +proc default[T](t: typedesc[T]): T {.inline.} = discard + +static: + var x = default(type(0)) diff --git a/todo.txt b/todo.txt index d77001115..235d79b78 100644 --- a/todo.txt +++ b/todo.txt @@ -1,26 +1,24 @@ +version 1.0 battle plan +======================= -nim c --gc:v2 -r -d:useSysAssert -d:useGcAssert -d:smokeCycles -d:useRealtimeGc tests/gc/gctest - -- https://github.com/nim-lang/Nim/issues/3898 - -essential for 1.0 -================= - -- introduce newSeqOfCap(10) -- annotation support for getType() -- overloading of `()` needs to be in .experimental -- find a solution for the x.f[T](y) gotcha -- ``concept`` needs to be refined, a nice name for the feature is not enough. -- make '--implicitStatic:on' the default; then we can also clean up the - 'static[T]' mess in the compiler! - Deprecate ``immediate`` for templates and macros -- document NimMain and check whether it works for threading +- fix "high priority" bugs +- try to fix as many compiler crashes as reasonable Not critical for 1.0 ==================== +- get GC:v2 stable: nim c --gc:v2 -r -d:useSysAssert -d:useGcAssert -d:smokeCycles -d:useRealtimeGc tests/gc/gctest + +- find a solution for the x.f[T](y) gotcha +- implement ``.delegate`` for .experimental + - Destructors need to be refined. +- annotation support for getType() +- ``concept`` needs to be refined, a nice name for the feature is not enough. +- make '--implicitStatic:on' the default; then we can also clean up the + 'static[T]' mess in the compiler! - ``not`` or ``~`` for the effects system - document and stress test ``.partial`` object declarations - add "all threads are blocked" detection to 'spawn' diff --git a/tools/niminst/buildsh.tmpl b/tools/niminst/buildsh.tmpl index 2c66ea493..13dfe5226 100644 --- a/tools/niminst/buildsh.tmpl +++ b/tools/niminst/buildsh.tmpl @@ -105,7 +105,11 @@ case $ucpu in *amd*64* | *x86-64* | *x86_64* ) mycpu="amd64" ;; *sparc*|*sun* ) - mycpu="sparc" ;; + mycpu="sparc" + if [ "$(isainfo -b)" = "64" ]; then + mycpu="sparc64" + fi + ;; *ppc64* ) if [ "$myos" = "linux" ] ; then COMP_FLAGS="$COMP_FLAGS -m64" diff --git a/tools/niminst/install.tmpl b/tools/niminst/install.tmpl index f9266b867..3f17840a8 100644 --- a/tools/niminst/install.tmpl +++ b/tools/niminst/install.tmpl @@ -3,7 +3,35 @@ # result = "#! /bin/sh\n# Generated by niminst\n" # var proj = c.name.toLower -set -e +## Current directory you start script from +BASE_DIR=$(pwd) + +## The following one-liner takes directory path which contains install script. +## `command -v -- "$0"` takes path if script sourced from interactive shell +## `dirname` returns relative directory path to install script +## `cd -P` dive into directory to use `pwd` +## `pwd -P` prints full path to install script directory path +## -P option allows to use symlinks in path +## Good explanation can be found here: +## http://stackoverflow.com/questions/29832037/how-to-get-script-directory-in-posix-sh +NIM_DIR=$(cd -P -- "$(dirname -- "$(command -v -- "$0")")" && pwd -P) + +go_back() { + cd $BASE_DIR +} + +## Go to base dir on exit +trap go_back EXIT + +install_error() { + echo "Nim installation failed!" + exit 1 +} + +## Exit if any command failed +trap install_error ERR ## `set -e` alternative + +cd $NIM_DIR if [ $# -eq 1 ] ; then # if c.cat[fcUnixBin].len > 0: diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index 79e552bde..4c8dfcddf 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -18,7 +18,7 @@ import const maxOS = 20 # max number of OSes - maxCPU = 10 # max number of CPUs + maxCPU = 20 # max number of CPUs buildShFile = "build.sh" buildBatFile32 = "build.bat" buildBatFile64 = "build64.bat" @@ -253,7 +253,7 @@ proc addFiles(s: var seq[string], patterns: seq[string]) = walkDirRecursively(s, p, p, false) else: var i = 0 - for f in walkFiles(p): + for f in walkPattern(p): if existsDir(f): walkDirRecursively(s, f, p, false) elif not ignoreFile(f, p, false): diff --git a/tools/nimweb.nim b/tools/nimweb.nim index 7cb4b5ab4..4cf7020c2 100644 --- a/tools/nimweb.nim +++ b/tools/nimweb.nim @@ -29,7 +29,7 @@ type TRssItem = object year, month, day, title, url, content: string TAction = enum - actAll, actOnlyWebsite, actPdf + actAll, actOnlyWebsite, actPdf, actJson2 Sponsor = object logo: string @@ -157,6 +157,7 @@ proc parseCmdLine(c: var TConfigData) = c.vars[substr(val, 0, idx-1)] = substr(val, idx+1) of "website": action = actOnlyWebsite of "pdf": action = actPdf + of "json2": action = actJson2 of "googleanalytics": c.gaId = val c.nimArgs.add("--doc.googleAnalytics:" & val & " ") @@ -379,7 +380,7 @@ proc genNewsLink(title: string): string = result = title result.insert("Z") for i in 1..len(result)-1: - let letter = result[i].toLower() + let letter = result[i].toLowerAscii() if letter in validAnchorCharacters: result[i] = letter else: @@ -487,7 +488,6 @@ proc buildNews(c: var TConfigData, newsDir: string, outputDir: string) = echo("Skipping file in news directory: ", path) proc buildWebsite(c: var TConfigData) = - if c.ticker.len > 0: try: c.ticker = readFile("web" / c.ticker) @@ -512,6 +512,19 @@ proc main(c: var TConfigData) = buildDocSamples(c, "doc") buildDoc(c, "doc") +proc json2(c: var TConfigData) = + const destPath = "web/json2" + var commands = newSeq[string](c.srcdoc2.len) + var i = 0 + for d in items(c.srcdoc2): + createDir(destPath / splitFile(d).dir) + commands[i] = findNim() & " jsondoc2 $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % + [c.nimArgs, c.gitRepo, c.gitCommit, d.pathPart, + destPath / changeFileExt(d, "json"), d] + i.inc + + mexec(commands, c.numProcessors) + var c: TConfigData initConfigData(c) parseCmdLine(c) @@ -520,3 +533,4 @@ case action of actOnlyWebsite: buildWebsite(c) of actPdf: buildPdfDoc(c, "doc") of actAll: main(c) +of actJson2: json2(c) diff --git a/tools/website.tmpl b/tools/website.tmpl index 6bdbd9b31..cf2c72a60 100644 --- a/tools/website.tmpl +++ b/tools/website.tmpl @@ -224,6 +224,7 @@ runForever() <p>This page lists the companies and individuals that are, very kindly, contributing a monthly amount to help sustain Nim's development. For more details take a look at the <a href="https://salt.bountysource.com/teams/nim">Bountysource campaign</a>.</p> +<p class="lastUpdate">Last updated: ${getTime().getGMTime().format("dd/MM/yyyy")}</p> <dl> #for sponsor in sponsors: <dt class="level-${sponsor.level}"> diff --git a/web/assets/bountysource/xored.png b/web/assets/bountysource/xored.png new file mode 100644 index 000000000..7a07cdd31 --- /dev/null +++ b/web/assets/bountysource/xored.png Binary files differdiff --git a/web/assets/style.css b/web/assets/style.css index d63177469..af32fad07 100644 --- a/web/assets/style.css +++ b/web/assets/style.css @@ -606,6 +606,11 @@ dt.level-1 { color: #2f2f2f !important; } +p.lastUpdate { + font-size: 12pt; + color: #6d6d6d; +} + /* News articles */ .metadata { @@ -614,3 +619,4 @@ dt.level-1 { margin-top: -16pt; color: #4f4f4f; } + diff --git a/web/news.rst b/web/news.rst index cd605b760..26f3adfdf 100644 --- a/web/news.rst +++ b/web/news.rst @@ -2,6 +2,9 @@ News ==== +`2016-06-23 Launching the 2016 Nim community survey <news/2016_06_23_launching_the_2016_nim_community_survey.html>`_ +=================================== + `2016-06-11 Version 0.14.2 released <news/2016_06_11_version_0_14_2_released.html>`_ =================================== diff --git a/web/news/2016_06_23_launching_the_2016_nim_community_survey.rst b/web/news/2016_06_23_launching_the_2016_nim_community_survey.rst new file mode 100644 index 000000000..0b87577aa --- /dev/null +++ b/web/news/2016_06_23_launching_the_2016_nim_community_survey.rst @@ -0,0 +1,29 @@ +Launching the 2016 Nim Community Survey +======================================= + +.. container:: metadata + + Posted by Dominik Picheta on 23/06/2016 + +We are proud to announce the official +`2016 Nim Community Survey <http://goo.gl/forms/XJ3TPsaiIQe5HlTB2>`_! No matter +whether you use Nim today, have used Nim previously, or never used Nim before; +we want to know your opinions. +Your feedback will help the Nim project understand its strengths and +weaknesses, and to determine development priorities for the future. + +It shouldn't take you much longer than 5 to 10 minutes to complete this survey. +Submissions will be accepted until around the 23rd of July, depending on the +response rates. If you have any questions or feedback, please don't hesitate +to get in touch with us via email at survey@nim-lang.org or on the +`Nim Forum <http://forum.nim-lang.org>`_. + +We would appreciate your help in spreading the word about this survey. Share +the above link on your social network feeds, with your colleagues and in +other communities. + +Thank you to everyone that helped develop and test the survey! Once the +submission period ends, the results will be shown here and publicised via +Twitter, the Nim Forum and IRC. + +Thanks for your time! diff --git a/web/news/version_0_15_released.rst b/web/news/version_0_15_released.rst index 76d78ce71..e16bdc9ae 100644 --- a/web/news/version_0_15_released.rst +++ b/web/news/version_0_15_released.rst @@ -13,6 +13,25 @@ Changes affecting backwards compatibility - De-deprecated ``re.nim`` because we have too much code using it and it got the basic API right. +- ``split`` with ``set[char]`` as a delimiter in ``strutils.nim`` + no longer strips and splits characters out of the target string + by the entire set of characters. Instead, it now behaves in a + similar fashion to ``split`` with ``string`` and ``char`` + delimiters. Use ``splitWhitespace`` to get the old behaviour. +- The command invocation syntax will soon apply to open brackets + and curlies too. This means that code like ``a [i]`` will be + interpreted as ``a([i])`` and not as ``a[i]`` anymore. Likewise + ``f (a, b)`` means that the tuple ``(a, b)`` is passed to ``f``. + The compiler produces a warning for ``a [i]``:: + + Warning: a [b] will be parsed as command syntax; spacing is deprecated + + See `https://github.com/nim-lang/Nim/issues/3898`_ for the relevant + discussion. +- Overloading the special operators ``.``, ``.()``, ``.=``, ``()`` now + should be enabled via ``{.experimental.}``. + + Library Additions ----------------- @@ -23,18 +42,44 @@ Library Additions - Added ``center`` and ``rsplit`` to ``strutils.nim`` to provide similar Python functionality for Nim's strings. -- Added ``isTitle``, ``title``, and ``swapCase`` to ``unicode.nim`` to - provide unicode aware string case manipulation. +- Added ``isTitle``, ``title``, ``swapCase``, ``isUpper``, ``toUpper``, + ``isLower``, ``toLower``, ``isAlpha``, ``isSpace``, and ``capitalize`` + to ``unicode.nim`` to provide unicode aware case manipulation and case + testing. - Added a new module ``lib/pure/strmisc.nim`` to hold uncommon string operations. Currently contains ``partition``, ``rpartition`` and ``expandTabs``. +- Split out ``walkFiles`` in ``os.nim`` to three separate procs in order + to make a clear distinction of functionality. ``walkPattern`` iterates + over both files and directories, while ``walkFiles`` now only iterates + over files and ``walkDirs`` only iterates over directories. + Compiler Additions ------------------ +- The ``-d/--define`` flag can now optionally take a value to be used + by code at compile time. + +Nimscript Additions +------------------- + +- Finally it's possible to dis/enable specific hints and warnings in + Nimscript via the procs ``warning`` and ``hint``. +- Nimscript exports a proc named ``patchFile`` which can be used to + patch modules or include files for different Nimble packages, including + the ``stdlib`` package. + + Language Additions ------------------ +- Added ``{.intdefine.}`` and ``{.strdefine.}`` macros to make use of + (optional) compile time defines. +- If the first statement is an ``import system`` statement then ``system`` + is not imported implicitly anymore. This allows for code like + ``import system except echo`` or ``from system import nil``. + Bugfixes -------- diff --git a/web/question.rst b/web/question.rst index b3a357bb8..9de178c90 100644 --- a/web/question.rst +++ b/web/question.rst @@ -126,6 +126,7 @@ General FAQ -------------------------- - Nim IDE: https://github.com/nim-lang/Aporia + - Visual Studio Code: https://marketplace.visualstudio.com/items?itemName=kosz78.nim - Emacs: https://github.com/nim-lang/nim-mode - Vim: https://github.com/zah/nimrod.vim/ - Scite: Included @@ -136,7 +137,7 @@ General FAQ - LiClipse: http://www.liclipse.com/ (Eclipse based plugin) - Howl: Included - Notepad++: Available via `plugin <https://github.com/jangko/nppnim/releases>`_ - + .. container:: standout diff --git a/web/sponsors.csv b/web/sponsors.csv index 046469d82..9c28f43ff 100644 --- a/web/sponsors.csv +++ b/web/sponsors.csv @@ -1,22 +1,31 @@ logo,name,url,this_month,all_time,since,level -assets/bountysource/secondspectrum.png,Second Spectrum,http://www.secondspectrum.com/,250,250,"May 5, 2016",250 +assets/bountysource/secondspectrum.png,Second Spectrum,http://www.secondspectrum.com/,250,500,"May 5, 2016",250 +assets/bountysource/xored.png,Xored Software,http://www.xored.com/,250,250,"Jun 20, 2016",250 ,flyx,http://flyx.org,35,70,"Apr 7, 2016",25 -,Adrian Veith,https://github.com/AdrianV,25,50,"Apr 20, 2016",25 +,Adrian Veith,,25,50,"Apr 20, 2016",25 ,Federico Ceratto,http://firelet.net,25,50,"Apr 7, 2016",25 ,Lloyd Moore,https://github.com/iolloyd,25,50,"Apr 29, 2016",25 ,Ruslan Mustakov,https://github.com/endragor,25,50,"Apr 7, 2016",25 ,Yuriy Glukhov,https://github.com/yglukhov/,25,50,"Apr 6, 2016",25 +,Euan T.,http://euantorano.co.uk,25,25,"Jun 7, 2016",25 ,TedSinger,https://github.com/TedSinger,15,30,"Apr 9, 2016",10 ,Pradeep Gowda,https://www.btbytes.com/,10,20,"Apr 6, 2016",10 ,Jonathan Arnett,http://j3rn.com,10,10,"May 20, 2016",10 +,Matthew Baulch,,10,10,"Jun 7, 2016",10 ,niebaopeng,https://github.com/niebaopeng,10,10,"Apr 15, 2016",10 +,Oskari Timperi,,10,10,"Jun 8, 2016",10 +,RationalG,https://github.com/RationalG,10,10,"Jun 17, 2016",10 +,Sergey Avseyev,http://avsej.net,10,10,"Jun 10, 2016",10 +,Zach Aysan,http://venn.lc,10,10,"Jun 7, 2016",10 +,swalf,https://joindiaspora.com/people/889b5d8ff6ed0b98,5,30,"May 9, 2016",5 ,Handojo Goenadi,,5,10,"Apr 19, 2016",5 ,John Novak,http://www.johnnovak.net/,5,10,"Apr 29, 2016",5 +,Konstantin Molchanov,http://sloth-ci.com,5,10,"May 13, 2016",5 ,Matthew Newton,,5,10,"Apr 20, 2016",5 -,McSpiros,https://github.com/SpirosMakris,5,10,"Apr 6, 2016",5 ,Mirek Rusin,http://quartzcomposer.com,5,10,"Apr 9, 2016",5 -,pyloor,https://github.com/pyloor,5,10,"May 16, 2016",5 +,pyloor ,,5,10,"May 16, 2016",5 ,Sokolov Yura,https://github.com/funny-falcon,5,10,"Apr 7, 2016",5 +,calind,,5,5,"Jun 7, 2016",5 ,Ivan Florentin,https://github.com/ivanflorentin,5,5,"May 2, 2016",5 ,Michael D. Sklaroff,,1,2,"Apr 27, 2016",1 ,Svend Knudsen,,1,2,"Apr 11, 2016",1 diff --git a/web/ticker.html b/web/ticker.html index 9e88456a9..e1232d107 100644 --- a/web/ticker.html +++ b/web/ticker.html @@ -1,3 +1,8 @@ +<a class="news" href="$1news/2016_06_23_launching_the_2016_nim_community_survey.html"> + <h4>June 23, 2016</h4> + <p>Launching the 2016 Nim community survey!</p> +</a> + <a class="news" href="$1news/2016_06_11_version_0_14_2_released.html"> <h4>June 11, 2016</h4> <p>Nim version 0.14.2 has been released!</p> @@ -18,9 +23,4 @@ <p>Nim version 0.13.0 has been released!</p> </a> -<a class="news" href="$1news/2016_01_18_andreas_rumpfs_talk_at_oscon_amsterdam.html"> - <h4>January 18, 2016</h4> - <p>Andreas Rumpf's talk at OSCON Amsterdam</p> -</a> - <a href="$1news.html" class="blue">See All News...</a> |
