diff options
434 files changed, 22373 insertions, 6324 deletions
diff --git a/.gitignore b/.gitignore index 1971a23cb..e4cec0ef6 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ nimcache/ *.dylib *.zip *.iss +*.log mapping.txt tags @@ -42,3 +43,7 @@ xcuserdata/ /testresults.json testament.db /csources + +# Private directories and files (IDEs) +.*/ +~* diff --git a/.travis.yml b/.travis.yml index f68375e93..0ebeeb995 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ addons: - libgc-dev before_script: - set -e - - wget http://flatassembler.net/fasm-1.71.39.tgz + - wget http://nim-lang.org/download/fasm-1.71.39.tgz - tar xvf fasm-1.71.39.tgz - git clone --depth 1 https://github.com/nim-lang/csources.git - cd csources @@ -23,6 +23,7 @@ script: - ./koch boot - ./koch boot -d:release - nim e install_nimble.nims + - nim e tests/test_nimscript.nims - nimble update - nimble install zip - nimble install opengl diff --git a/bootstrap.sh b/bootstrap.sh index 7f19c2440..dd551b52d 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -18,5 +18,3 @@ set +x echo echo 'Install Nim using "./install.sh <dir>" or "sudo ./install.sh <dir>".' - -exit 0 diff --git a/compiler.nimble b/compiler.nimble index a98981a97..c63fe49bf 100644 --- a/compiler.nimble +++ b/compiler.nimble @@ -1,6 +1,6 @@ [Package] name = "compiler" -version = "0.13.0" +version = "0.14.3" author = "Andreas Rumpf" description = "Compiler package providing the compiler sources as a library." license = "MIT" @@ -8,4 +8,4 @@ license = "MIT" InstallDirs = "doc, compiler" [Deps] -Requires: "nim >= 0.12.0" +Requires: "nim >= 0.13.0" diff --git a/compiler/ast.nim b/compiler/ast.nim index fecbefd7e..277a21ba5 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -271,13 +271,14 @@ type TSymFlags* = set[TSymFlag] const - sfFakeConst* = sfDeadCodeElim # const cannot be put into a data section sfDispatcher* = sfDeadCodeElim # copied method symbol is the dispatcher sfNoInit* = sfMainModule # don't generate code to init the variable sfImmediate* = sfDeadCodeElim # macro or template is immediately expanded # without considering any possible overloads + sfAllUntyped* = sfVolatile # macro or template is immediately expanded \ + # in a generic context sfDirty* = sfPure # template is not hygienic (old styled template) @@ -395,6 +396,9 @@ type # sons[1]: field type # .n: nkDotExpr storing the field name + tyVoid #\ + # now different from tyEmpty, hurray! + static: # remind us when TTypeKind stops to fit in a single 64-bit word assert TTypeKind.high.ord <= 63 @@ -528,8 +532,6 @@ const tfOldSchoolExprStmt* = tfVarargs # for now used to distinguish \ # 'varargs[expr]' from 'varargs[untyped]'. Eventually 'expr' will be # deprecated and this mess can be cleaned up. - tfVoid* = tfVarargs # for historical reasons we conflated 'void' with - # 'empty' ('@[]' has the type 'seq[empty]'). tfReturnsNew* = tfInheritable skError* = skUnknown @@ -544,7 +546,7 @@ type mEcho, mShallowCopy, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst, mExpandToAst, mQuoteAst, mUnaryLt, mInc, mDec, mOrd, - mNew, mNewFinalize, mNewSeq, + mNew, mNewFinalize, mNewSeq, mNewSeqOfCap, mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq, mXLenStr, mXLenSeq, mIncl, mExcl, mCard, mChr, @@ -563,8 +565,8 @@ type mEqEnum, mLeEnum, mLtEnum, mEqCh, mLeCh, mLtCh, mEqB, mLeB, mLtB, - mEqRef, mEqUntracedRef, mLePtr, mLtPtr, mEqCString, - mXor, mEqProc, + mEqRef, mEqUntracedRef, mLePtr, mLtPtr, + mXor, mEqCString, mEqProc, mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, mAbsF64, @@ -590,6 +592,7 @@ type mNewString, mNewStringOfCap, mParseBiggestFloat, mReset, mArray, mOpenArray, mRange, mSet, mSeq, mVarargs, + mRef, mPtr, mVar, mDistinct, mVoid, mTuple, mOrdinal, mInt, mInt8, mInt16, mInt32, mInt64, mUInt, mUInt8, mUInt16, mUInt32, mUInt64, @@ -609,7 +612,7 @@ type mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, mNGenSym, - mNimvm + mNimvm, mIntDefine, mStrDefine # things that we can evaluate safely at compile time, even if not asked for it: const @@ -769,7 +772,7 @@ type procInstCache*: seq[PInstantiation] gcUnsafetyReason*: PSym # for better error messages wrt gcsafe #scope*: PScope # the scope where the proc was defined - of skModule: + of skModule, skPackage: # modules keep track of the generic symbols they use from other modules. # this is because in incremental compilation, when a module is about to # be replaced with a newer version, we must decrement the usage count @@ -848,6 +851,7 @@ type # see instantiateDestructor in semdestruct.nim deepCopy*: PSym # overriden 'deepCopy' operation assignment*: PSym # overriden '=' operator + methods*: seq[(int,PSym)] # attached methods size*: BiggestInt # the size of the type in bytes # -1 means that the size is unkwown align*: int16 # the type's alignment requirements @@ -938,7 +942,7 @@ const genericParamsPos* = 2 paramsPos* = 3 pragmasPos* = 4 - optimizedCodePos* = 5 # will be used for exception tracking + miscPos* = 5 # used for undocumented and hacky stuff bodyPos* = 6 # position of body; use rodread.getBody() instead! resultPos* = 7 dispatcherPos* = 8 # caution: if method has no 'result' it can be position 7! @@ -961,6 +965,8 @@ const skMethod, skConverter} var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things +var + gMainPackageId*: int proc isCallExpr*(n: PNode): bool = result = n.kind in nkCallKinds @@ -984,12 +990,12 @@ proc add*(father, son: PNode) = proc `[]`*(n: PNode, i: int): PNode {.inline.} = result = n.sons[i] -template `-|`*(b, s: expr): expr = +template `-|`*(b, s: untyped): untyped = (if b >= 0: b else: s.len + b) # son access operators with support for negative indices -template `{}`*(n: PNode, i: int): expr = n[i -| n] -template `{}=`*(n: PNode, i: int, s: PNode): stmt = +template `{}`*(n: PNode, i: int): untyped = n[i -| n] +template `{}=`*(n: PNode, i: int, s: PNode) = n.sons[i -| n] = s when defined(useNodeIds): @@ -1584,7 +1590,7 @@ proc isAtom*(n: PNode): bool {.inline.} = proc isEmptyType*(t: PType): bool {.inline.} = ## 'void' and 'stmt' types are often equivalent to 'nil' these days: - result = t == nil or t.kind in {tyEmpty, tyStmt} + result = t == nil or t.kind in {tyVoid, tyStmt} proc makeStmtList*(n: PNode): PNode = if n.kind == nkStmtList: diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index bd17f85e4..dffb8a9a5 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -124,7 +124,7 @@ proc openArrayLoc(p: BProc, n: PNode): Rope = result = "(*$1)->data, (*$1)->$2" % [a.rdLoc, lenField(p)] of tyArray, tyArrayConstr: result = "$1, $2" % [rdLoc(a), rope(lengthOrd(lastSon(a.t)))] - else: + else: internalError("openArrayLoc: " & typeToString(a.t)) else: internalError("openArrayLoc: " & typeToString(a.t)) @@ -260,7 +260,11 @@ proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope = else: result = genArgNoParam(p, ri.sons[i]) #, typ.n.sons[i].sym) else: - result = genArgNoParam(p, ri.sons[i]) + if tfVarargs notin typ.flags: + localError(ri.info, "wrong argument count") + result = nil + else: + result = genArgNoParam(p, ri.sons[i]) discard """ Dot call syntax in C++ diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 4fcbeeec2..6c96f209e 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -45,7 +45,7 @@ proc genHexLiteral(v: PNode): Rope = proc getStrLit(m: BModule, s: string): Rope = discard cgsym(m, "TGenericSeq") - result = "TMP" & rope(backendId()) + result = getTempName(m) addf(m.s[cfsData], "STRING_LITERAL($1, $2, $3);$n", [result, makeCString(s), rope(len(s))]) @@ -67,11 +67,11 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope = of nkNilLit: let t = skipTypes(ty, abstractVarRange) if t.kind == tyProc and t.callConv == ccClosure: - var id = nodeTableTestOrSet(p.module.dataCache, n, gBackendId) - result = "TMP" & rope(id) - if id == gBackendId: + let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) + result = p.module.tmpBase & rope(id) + if id == p.module.labels: # not found in cache: - inc(gBackendId) + inc(p.module.labels) addf(p.module.s[cfsData], "static NIM_CONST $1 $2 = {NIM_NIL,NIM_NIL};$n", [getTypeDesc(p.module, t), result]) @@ -81,13 +81,14 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope = if n.strVal.isNil: result = ropecg(p.module, "((#NimStringDesc*) NIM_NIL)", []) elif skipTypes(ty, abstractVarRange).kind == tyString: - var id = nodeTableTestOrSet(p.module.dataCache, n, gBackendId) - if id == gBackendId: + let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) + if id == p.module.labels: # string literal not found in the cache: result = ropecg(p.module, "((#NimStringDesc*) &$1)", [getStrLit(p.module, n.strVal)]) else: - result = ropecg(p.module, "((#NimStringDesc*) &TMP$1)", [rope(id)]) + result = ropecg(p.module, "((#NimStringDesc*) &$1$2)", + [p.module.tmpBase, rope(id)]) else: result = makeCString(n.strVal) of nkFloatLit..nkFloat64Lit: @@ -134,11 +135,11 @@ proc genSetNode(p: BProc, n: PNode): Rope = var size = int(getSize(n.typ)) toBitSet(n, cs) if size > 8: - var id = nodeTableTestOrSet(p.module.dataCache, n, gBackendId) - result = "TMP" & rope(id) - if id == gBackendId: + let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) + result = p.module.tmpBase & rope(id) + if id == p.module.labels: # not found in cache: - inc(gBackendId) + inc(p.module.labels) addf(p.module.s[cfsData], "static NIM_CONST $1 $2 = $3;$n", [getTypeDesc(p.module, n.typ), result, genRawSetData(cs, size)]) else: @@ -490,7 +491,7 @@ proc binaryArithOverflowRaw(p: BProc, t: PType, a, b: TLoc; var size = getSize(t) let storage = if size < platform.intSize: rope("NI") else: getTypeDesc(p.module, t) - result = getTempName() + result = getTempName(p.module) linefmt(p, cpsLocals, "$1 $2;$n", storage, result) lineCg(p, cpsStmts, frmt, result, rdCharLoc(a), rdCharLoc(b)) if size < platform.intSize or t.kind in {tyRange, tyEnum}: @@ -591,7 +592,6 @@ proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) = "($1 == $2)", # EqPtr "($1 <= $2)", # LePtr "($1 < $2)", # LtPtr - "($1 == $2)", # EqCString "($1 != $2)"] # Xor var a, b: TLoc @@ -612,7 +612,7 @@ proc genEqProc(p: BProc, e: PNode, d: var TLoc) = assert(e.sons[2].typ != nil) initLocExpr(p, e.sons[1], a) initLocExpr(p, e.sons[2], b) - if a.t.callConv == ccClosure: + if a.t.skipTypes(abstractInst).callConv == ccClosure: putIntoDest(p, d, e.typ, "($1.ClPrc == $2.ClPrc && $1.ClEnv == $2.ClEnv)" % [rdLoc(a), rdLoc(b)]) else: @@ -670,6 +670,8 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) = #if e[0].kind != nkBracketExpr: # message(e.info, warnUser, "CAME HERE " & renderTree(e)) expr(p, e.sons[0], d) + if e.sons[0].typ.skipTypes(abstractInst).kind == tyRef: + d.s = OnHeap else: var a: TLoc let typ = skipTypes(e.sons[0].typ, abstractInst) @@ -802,9 +804,9 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym; v.r.add(d.loc.r) genInExprAux(p, it, u, v, test) let id = nodeTableTestOrSet(p.module.dataCache, - newStrNode(nkStrLit, field.name.s), gBackendId) - let strLit = if id == gBackendId: getStrLit(p.module, field.name.s) - else: "TMP" & rope(id) + newStrNode(nkStrLit, field.name.s), p.module.labels) + let strLit = if id == p.module.labels: getStrLit(p.module, field.name.s) + else: p.module.tmpBase & rope(id) if op.magic == mNot: linefmt(p, cpsStmts, "if ($1) #raiseFieldError(((#NimStringDesc*) &$2));$n", @@ -1141,7 +1143,35 @@ proc genNewSeq(p: BProc, e: PNode) = genNewSeqAux(p, a, b.rdLoc) gcUsage(e) +proc genNewSeqOfCap(p: BProc; e: PNode; d: var TLoc) = + let seqtype = skipTypes(e.typ, abstractVarRange) + var a: TLoc + initLocExpr(p, e.sons[1], a) + putIntoDest(p, d, e.typ, ropecg(p.module, + "($1)#nimNewSeqOfCap($2, $3)", [ + getTypeDesc(p.module, seqtype), + genTypeInfo(p.module, seqtype), a.rdLoc])) + gcUsage(e) + +proc genConstExpr(p: BProc, n: PNode): Rope +proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool = + if d.k == locNone and n.len > ord(n.kind == nkObjConstr) and n.isDeepConstExpr: + var t = getUniqueType(n.typ) + discard getTypeDesc(p.module, t) # so that any fields are initialized + let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) + fillLoc(d, locData, t, p.module.tmpBase & rope(id), OnStatic) + if id == p.module.labels: + # expression not found in the cache: + inc(p.module.labels) + addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n", + [getTypeDesc(p.module, t), d.r, genConstExpr(p, n)]) + result = true + else: + result = false + proc genObjConstr(p: BProc, e: PNode, d: var TLoc) = + if handleConstExpr(p, e, d): return + #echo rendertree e, " ", e.isDeepConstExpr var tmp: TLoc var t = e.typ.skipTypes(abstractInst) getTemp(p, t, tmp) @@ -1235,7 +1265,7 @@ proc genOfHelper(p: BProc; dest: PType; a: Rope): Rope = # unfortunately 'genTypeInfo' sets tfObjHasKids as a side effect, so we # have to call it here first: let ti = genTypeInfo(p.module, dest) - if tfFinal in dest.flags or (p.module.objHasKidsValid and + if tfFinal in dest.flags or (objHasKidsValid in p.module.flags and tfObjHasKids notin dest.flags): result = "$1.m_type == $2" % [a, ti] else: @@ -1293,7 +1323,7 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) = putIntoDest(p, d, e.typ, ropecg(p.module, "#reprChar($1)", [rdLoc(a)]), a.s) of tyEnum, tyOrdinal: putIntoDest(p, d, e.typ, - ropecg(p.module, "#reprEnum($1, $2)", [ + ropecg(p.module, "#reprEnum((NI)$1, $2)", [ rdLoc(a), genTypeInfo(p.module, t)]), a.s) of tyString: putIntoDest(p, d, e.typ, ropecg(p.module, "#reprStr($1)", [rdLoc(a)]), a.s) @@ -1320,7 +1350,7 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) = putIntoDest(p, d, e.typ, ropecg(p.module, "#reprAny($1, $2)", [ rdLoc(a), genTypeInfo(p.module, t)]), a.s) - of tyEmpty: + of tyEmpty, tyVoid: localError(e.info, "'repr' doesn't support 'void' type") else: putIntoDest(p, d, e.typ, ropecg(p.module, "#reprAny($1, $2)", @@ -1558,11 +1588,12 @@ proc genSomeCast(p: BProc, e: PNode, d: var TLoc) = [getTypeDesc(p.module, e.typ), rdCharLoc(a)], a.s) proc genCast(p: BProc, e: PNode, d: var TLoc) = - const floatTypes = {tyFloat..tyFloat128} + const ValueTypes = {tyFloat..tyFloat128, tyTuple, tyObject, + tyArray, tyArrayConstr} let destt = skipTypes(e.typ, abstractRange) srct = skipTypes(e.sons[1].typ, abstractRange) - if destt.kind in floatTypes or srct.kind in floatTypes: + if destt.kind in ValueTypes or srct.kind in ValueTypes: # 'cast' and some float type involved? --> use a union. inc(p.labels) var lbl = p.labels.rope @@ -1668,10 +1699,10 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = if optOverflowCheck notin p.options: unaryExpr(p, e, d, "($1 - 1)") else: unaryExpr(p, e, d, "#subInt($1, 1)") of mInc, mDec: - const opr: array [mInc..mDec, string] = ["$1 += $2;$n", "$1 -= $2;$n"] - const fun64: array [mInc..mDec, string] = ["$# = #addInt64($#, $#);$n", + const opr: array[mInc..mDec, string] = ["$1 += $2;$n", "$1 -= $2;$n"] + const fun64: array[mInc..mDec, string] = ["$# = #addInt64($#, $#);$n", "$# = #subInt64($#, $#);$n"] - const fun: array [mInc..mDec, string] = ["$# = #addInt($#, $#);$n", + const fun: array[mInc..mDec, string] = ["$# = #addInt($#, $#);$n", "$# = #subInt($#, $#);$n"] let underlying = skipTypes(e.sons[1].typ, {tyGenericInst, tyVar, tyRange}) if optOverflowCheck notin p.options or underlying.kind in {tyUInt..tyUInt64}: @@ -1712,6 +1743,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mNew: genNew(p, e) of mNewFinalize: genNewFinalize(p, e) of mNewSeq: genNewSeq(p, e) + of mNewSeqOfCap: genNewSeqOfCap(p, e, d) of mSizeOf: let t = e.sons[1].typ.skipTypes({tyTypeDesc}) putIntoDest(p, d, e.typ, "((NI)sizeof($1))" % [getTypeDesc(p.module, t)]) @@ -1754,25 +1786,9 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = initLocExpr(p, x, a) initLocExpr(p, e.sons[2], b) genDeepCopy(p, a, b) - of mDotDot: genCall(p, e, d) + of mDotDot, mEqCString: genCall(p, e, d) else: internalError(e.info, "genMagicExpr: " & $op) -proc genConstExpr(p: BProc, n: PNode): Rope -proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool = - if nfAllConst in n.flags and d.k == locNone and n.len > 0 and n.isDeepConstExpr: - var t = getUniqueType(n.typ) - discard getTypeDesc(p.module, t) # so that any fields are initialized - var id = nodeTableTestOrSet(p.module.dataCache, n, gBackendId) - fillLoc(d, locData, t, "TMP" & rope(id), OnStatic) - if id == gBackendId: - # expression not found in the cache: - inc(gBackendId) - addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n", - [getTypeDesc(p.module, t), d.r, genConstExpr(p, n)]) - result = true - else: - result = false - proc genSetConstr(p: BProc, e: PNode, d: var TLoc) = # example: { a..b, c, d, e, f..g } # we have to emit an expression of the form: @@ -1856,10 +1872,16 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) = initLocExpr(p, n.sons[1], b) if n.sons[0].skipConv.kind == nkClosure: internalError(n.info, "closure to closure created") - getTemp(p, n.typ, tmp) - linefmt(p, cpsStmts, "$1.ClPrc = $2; $1.ClEnv = $3;$n", - tmp.rdLoc, a.rdLoc, b.rdLoc) - putLocIntoDest(p, d, tmp) + # tasyncawait.nim breaks with this optimization: + when false: + if d.k != locNone: + linefmt(p, cpsStmts, "$1.ClPrc = $2; $1.ClEnv = $3;$n", + d.rdLoc, a.rdLoc, b.rdLoc) + else: + getTemp(p, n.typ, tmp) + linefmt(p, cpsStmts, "$1.ClPrc = $2; $1.ClEnv = $3;$n", + tmp.rdLoc, a.rdLoc, b.rdLoc) + putLocIntoDest(p, d, tmp) proc genArrayConstr(p: BProc, n: PNode, d: var TLoc) = var arr: TLoc @@ -1947,12 +1969,12 @@ proc downConv(p: BProc, n: PNode, d: var TLoc) = proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) = var t = getUniqueType(n.typ) discard getTypeDesc(p.module, t) # so that any fields are initialized - var id = nodeTableTestOrSet(p.module.dataCache, n, gBackendId) - var tmp = "TMP" & rope(id) + let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels) + let tmp = p.module.tmpBase & rope(id) - if id == gBackendId: + if id == p.module.labels: # expression not found in the cache: - inc(gBackendId) + inc(p.module.labels) addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n", [getTypeDesc(p.module, t), tmp, genConstExpr(p, n)]) @@ -1960,6 +1982,10 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) = fillLoc(d, locData, t, tmp, OnStatic) else: putDataIntoDest(p, d, t, tmp) + # This fixes bug #4551, but we really need better dataflow + # analysis to make this 100% safe. + if t.kind notin {tySequence, tyString}: + d.s = OnStatic proc expr(p: BProc, n: PNode, d: var TLoc) = case n.kind @@ -1985,10 +2011,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = internalError(n.info, "expr: proc not init " & sym.name.s) putLocIntoDest(p, d, sym.loc) of skConst: - if sfFakeConst in sym.flags: - if sfGlobal in sym.flags: genVarPrototype(p.module, sym) - putLocIntoDest(p, d, sym.loc) - elif isSimpleConst(sym.typ): + if isSimpleConst(sym.typ): putIntoDest(p, d, n.typ, genLiteral(p, sym.ast, sym.typ), OnStatic) else: genComplexConst(p, sym, d) @@ -2160,7 +2183,8 @@ proc genConstSimpleList(p: BProc, n: PNode): Rope = result = rope("{") for i in countup(ord(n.kind == nkObjConstr), length - 2): addf(result, "$1,$n", [genNamedConstExpr(p, n.sons[i])]) - if length > 0: add(result, genNamedConstExpr(p, n.sons[length - 1])) + if length > ord(n.kind == nkObjConstr): + add(result, genNamedConstExpr(p, n.sons[length - 1])) addf(result, "}$n", []) proc genConstSeq(p: BProc, n: PNode, t: PType): Rope = @@ -2174,8 +2198,7 @@ proc genConstSeq(p: BProc, n: PNode, t: PType): Rope = data.add("}") data.add("}") - inc(gBackendId) - result = "CNSTSEQ" & gBackendId.rope + result = getTempName(p.module) appcg(p.module, cfsData, "NIM_CONST struct {$n" & diff --git a/compiler/ccgmerge.nim b/compiler/ccgmerge.nim index 2a37257b6..92b6aa9dc 100644 --- a/compiler/ccgmerge.nim +++ b/compiler/ccgmerge.nim @@ -107,13 +107,13 @@ proc genMergeInfo*(m: BModule): Rope = writeIntSet(m.typeInfoMarker, s) s.add("labels:") encodeVInt(m.labels, s) - s.add(" hasframe:") - encodeVInt(ord(m.frameDeclared), s) + s.add(" flags:") + encodeVInt(cast[int](m.flags), s) s.add(tnl) s.add("*/") result = s.rope -template `^`(pos: int): expr = L.buf[pos] +template `^`(pos: int): untyped = L.buf[pos] proc skipWhite(L: var TBaseLexer) = var pos = L.bufpos @@ -222,13 +222,14 @@ proc processMergeInfo(L: var TBaseLexer, m: BModule) = of "declared": readIntSet(L, m.declaredThings) of "typeInfo": readIntSet(L, m.typeInfoMarker) of "labels": m.labels = decodeVInt(L.buf, L.bufpos) - of "hasframe": m.frameDeclared = decodeVInt(L.buf, L.bufpos) != 0 + of "flags": + m.flags = cast[set[CodegenFlag]](decodeVInt(L.buf, L.bufpos) != 0) else: internalError("ccgmerge: unknown key: " & k) when not defined(nimhygiene): {.pragma: inject.} -template withCFile(cfilename: string, body: stmt) {.immediate.} = +template withCFile(cfilename: string, body: untyped) = var s = llStreamOpen(cfilename, fmRead) if s == nil: return var L {.inject.}: TBaseLexer diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index f2ceadcce..a5ce147c3 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -16,7 +16,7 @@ const # above X strings a hash-switch for strings is generated proc registerGcRoot(p: BProc, v: PSym) = - if gSelectedGC in {gcMarkAndSweep, gcGenerational, gcV2} and + if gSelectedGC in {gcMarkAndSweep, gcGenerational, gcV2, gcRefc} and containsGarbageCollectedRef(v.loc.t): # we register a specialized marked proc here; this has the advantage # that it works out of the box for thread local storage then :-) @@ -136,7 +136,7 @@ proc exprBlock(p: BProc, n: PNode, d: var TLoc) = expr(p, n, d) endBlock(p) -template preserveBreakIdx(body: stmt): stmt {.immediate.} = +template preserveBreakIdx(body: untyped): untyped = var oldBreakIdx = p.breakIdx body p.breakIdx = oldBreakIdx @@ -269,8 +269,6 @@ proc genConstStmt(p: BProc, t: PNode) = if it.kind != nkConstDef: internalError(t.info, "genConstStmt") var c = it.sons[0].sym if c.typ.containsCompileTimeOnly: continue - if sfFakeConst in c.flags: - genSingleVar(p, it) elif c.typ.kind in ConstantDataTypes and lfNoDecl notin c.loc.flags and c.ast.len != 0: if not emitLazily(c): requestConstImpl(p, c) @@ -570,7 +568,7 @@ proc genRaiseStmt(p: BProc, t: PNode) = else: genLineDir(p, t) # reraise the last exception: - if p.module.compileToCpp: + if p.module.compileToCpp and optNoCppExceptions notin gGlobalOptions: line(p, cpsStmts, ~"throw;$n") else: linefmt(p, cpsStmts, "#reraiseException();$n") @@ -793,7 +791,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = if not isEmptyType(t.typ) and d.k == locNone: getTemp(p, t.typ, d) genLineDir(p, t) - let exc = getTempName() + let exc = getTempName(p.module) if getCompilerProc("Exception") != nil: discard cgsym(p.module, "Exception") else: @@ -886,7 +884,7 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) = getTemp(p, t.typ, d) discard lists.includeStr(p.module.headerFiles, "<setjmp.h>") genLineDir(p, t) - var safePoint = getTempName() + var safePoint = getTempName(p.module) if getCompilerProc("Exception") != nil: discard cgsym(p.module, "Exception") else: @@ -961,6 +959,8 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope = var a: TLoc initLocExpr(p, t.sons[i], a) res.add($rdLoc(a)) + elif sym.kind == skType: + res.add($getTypeDesc(p.module, sym.typ)) else: var r = sym.loc.r if r == nil: diff --git a/compiler/ccgthreadvars.nim b/compiler/ccgthreadvars.nim index ab771d240..81af89249 100644 --- a/compiler/ccgthreadvars.nim +++ b/compiler/ccgthreadvars.nim @@ -18,7 +18,7 @@ proc emulatedThreadVars(): bool = proc accessThreadLocalVar(p: BProc, s: PSym) = if emulatedThreadVars() and not p.threadVarAccessed: p.threadVarAccessed = true - p.module.usesThreadVars = true + incl p.module.flags, usesThreadVars addf(p.procSec(cpsLocals), "\tNimThreadVars* NimTV;$n", []) add(p.procSec(cpsInit), ropecg(p.module, "\tNimTV = (NimThreadVars*) #GetThreadLocalVars();$n")) @@ -51,7 +51,7 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) = addf(m.s[cfsVars], " $1;$n", [s.loc.r]) proc generateThreadLocalStorage(m: BModule) = - if nimtv != nil and (m.usesThreadVars or sfMainModule in m.module.flags): + if nimtv != nil and (usesThreadVars in m.flags or sfMainModule in m.module.flags): for t in items(nimtvDeps): discard getTypeDesc(m, t) addf(m.s[cfsSeqTypes], "typedef struct {$1} NimThreadVars;$n", [nimtv]) diff --git a/compiler/ccgtrav.nim b/compiler/ccgtrav.nim index 0da6396ea..c4d47f148 100644 --- a/compiler/ccgtrav.nim +++ b/compiler/ccgtrav.nim @@ -100,7 +100,7 @@ proc genTraverseProcSeq(c: var TTraversalClosure, accessor: Rope, typ: PType) = proc genTraverseProc(m: BModule, typ: PType, reason: TTypeInfoReason): Rope = var c: TTraversalClosure var p = newProc(nil, m) - result = getGlobalTempName() + result = getTempName(m) case reason of tiNew: c.visitorFrmt = "#nimGCvisit((void*)$1, op);$n" @@ -135,7 +135,7 @@ proc genTraverseProcForGlobal(m: BModule, s: PSym): Rope = var c: TTraversalClosure var p = newProc(nil, m) var sLoc = s.loc.r - result = getGlobalTempName() + result = getTempName(m) if sfThread in s.flags and emulatedThreadVars(): accessThreadLocalVar(p, s) @@ -143,7 +143,7 @@ proc genTraverseProcForGlobal(m: BModule, s: PSym): Rope = c.visitorFrmt = "#nimGCvisit((void*)$1, 0);$n" c.p = p - let header = "N_NIMCALL(void, $1)()" % [result] + let header = "N_NIMCALL(void, $1)(void)" % [result] genTraverseProc(c, sLoc, s.loc.t) let generatedProc = "$1 {$n$2$3$4}$n" % diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 39f16ff0d..3d1c2affc 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1,7 +1,7 @@ # # # The Nim Compiler -# (c) Copyright 2013 Andreas Rumpf +# (c) Copyright 2016 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -11,6 +11,8 @@ # ------------------------- Name Mangling -------------------------------- +import debuginfo + proc isKeyword(w: PIdent): bool = # Nim and C++ share some keywords # it's more efficient to test the whole Nim keywords range @@ -23,70 +25,70 @@ proc isKeyword(w: PIdent): bool = proc mangleField(name: PIdent): string = result = mangle(name.s) if isKeyword(name): - result[0] = result[0].toUpper # Mangling makes everything lowercase, - # but some identifiers are C keywords + result[0] = result[0].toUpperAscii + # Mangling makes everything lowercase, + # but some identifiers are C keywords + +proc hashOwner(s: PSym): FilenameHash = + var m = s + while m.kind != skModule: m = m.owner + let p = m.owner + assert p.kind == skPackage + result = gDebugInfo.register(p.name.s, m.name.s) proc mangleName(s: PSym): Rope = result = s.loc.r if result == nil: - when oKeepVariableNames: - let keepOrigName = s.kind in skLocalVars - {skForVar} and - {sfFromGeneric, sfGlobal, sfShadowed, sfGenSym} * s.flags == {} and - not isKeyword(s.name) - # XXX: This is still very experimental - # - # Even with all these inefficient checks, the bootstrap - # time is actually improved. This is probably because so many - # rope concatenations are now eliminated. - # - # Future notes: - # sfFromGeneric seems to be needed in order to avoid multiple - # definitions of certain variables generated in transf with - # names such as: - # `r`, `res` - # I need to study where these come from. - # - # about sfShadowed: - # consider the following Nim code: - # var x = 10 - # block: - # var x = something(x) - # The generated C code will be: - # NI x; - # x = 10; - # { - # NI x; - # x = something(x); // Oops, x is already shadowed here - # } - # Right now, we work-around by not keeping the original name - # of the shadowed variable, but we can do better - we can - # create an alternative reference to it in the outer scope and - # use that in the inner scope. - # - # about isCKeyword: - # Nim variable names can be C keywords. - # We need to avoid such names in the generated code. - # XXX: Study whether mangleName is called just once per variable. - # Otherwise, there might be better place to do this. - # - # about sfGlobal: - # This seems to be harder - a top level extern variable from - # another modules can have the same name as a local one. - # Maybe we should just implement sfShadowed for them too. - # - # about skForVar: - # These are not properly scoped now - we need to add blocks - # around for loops in transf - if keepOrigName: - result = s.name.s.mangle.rope - else: - add(result, rope(mangle(s.name.s))) - add(result, ~"_") - add(result, rope(s.id)) + let keepOrigName = s.kind in skLocalVars - {skForVar} and + {sfFromGeneric, sfGlobal, sfShadowed, sfGenSym} * s.flags == {} and + not isKeyword(s.name) + # Even with all these inefficient checks, the bootstrap + # time is actually improved. This is probably because so many + # rope concatenations are now eliminated. + # + # sfFromGeneric is needed in order to avoid multiple + # definitions of certain variables generated in transf with + # names such as: + # `r`, `res` + # I need to study where these come from. + # + # about sfShadowed: + # consider the following Nim code: + # var x = 10 + # block: + # var x = something(x) + # The generated C code will be: + # NI x; + # x = 10; + # { + # NI x; + # x = something(x); // Oops, x is already shadowed here + # } + # Right now, we work-around by not keeping the original name + # of the shadowed variable, but we can do better - we can + # create an alternative reference to it in the outer scope and + # use that in the inner scope. + # + # about isCKeyword: + # Nim variable names can be C keywords. + # We need to avoid such names in the generated code. + # + # about sfGlobal: + # This seems to be harder - a top level extern variable from + # another modules can have the same name as a local one. + # Maybe we should just implement sfShadowed for them too. + # + # about skForVar: + # These are not properly scoped now - we need to add blocks + # around for loops in transf + result = s.name.s.mangle.rope + if keepOrigName: + result.add "0" else: - add(result, rope(mangle(s.name.s))) add(result, ~"_") add(result, rope(s.id)) + add(result, ~"_") + add(result, rope(hashOwner(s).BiggestInt)) s.loc.r = result proc typeName(typ: PType): Rope = @@ -137,6 +139,10 @@ proc mapType(typ: PType): TCTypeKind = var base = skipTypes(typ.lastSon, typedescInst) case base.kind of tyOpenArray, tyArrayConstr, tyArray, tyVarargs: result = ctPtrToArray + #of tySet: + # if mapSetType(base) == ctArray: result = ctPtrToArray + # else: result = ctPtr + # XXX for some reason this breaks the pegs module else: result = ctPtr of tyPointer: result = ctPtr of tySequence: result = ctNimSeq @@ -145,11 +151,15 @@ proc mapType(typ: PType): TCTypeKind = of tyCString: result = ctCString of tyInt..tyUInt64: result = TCTypeKind(ord(typ.kind) - ord(tyInt) + ord(ctInt)) + of tyStatic: + if typ.n != nil: result = mapType(lastSon typ) + else: internalError("mapType") else: internalError("mapType") proc mapReturnType(typ: PType): TCTypeKind = - if skipTypes(typ, typedescInst).kind == tyArray: result = ctPtr - else: result = mapType(typ) + #if skipTypes(typ, typedescInst).kind == tyArray: result = ctPtr + #else: + result = mapType(typ) proc isImportedType(t: PType): bool = result = t.sym != nil and sfImportc in t.sym.flags @@ -196,11 +206,9 @@ proc cacheGetType(tab: TIdTable, key: PType): Rope = # linear search is not necessary anymore: result = Rope(idTableGet(tab, key)) -proc getTempName(): Rope = - result = rfmt(nil, "TMP$1", rope(backendId())) - -proc getGlobalTempName(): Rope = - result = rfmt(nil, "TMP$1", rope(backendId())) +proc getTempName(m: BModule): Rope = + result = m.tmpBase & rope(m.labels) + inc m.labels proc ccgIntroducedPtr(s: PSym): bool = var pt = skipTypes(s.typ, typedescInst) @@ -223,7 +231,7 @@ proc ccgIntroducedPtr(s: PSym): bool = proc fillResult(param: PSym) = fillLoc(param.loc, locParam, param.typ, ~"Result", OnStack) - if (mapReturnType(param.typ) != ctArray) and isInvalidReturnType(param.typ): + if mapReturnType(param.typ) != ctArray and isInvalidReturnType(param.typ): incl(param.loc.flags, lfIndirect) param.loc.s = OnUnknown @@ -238,22 +246,10 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = NumericalTypeToStr: array[tyInt..tyUInt64, string] = [ "NI", "NI8", "NI16", "NI32", "NI64", "NF", "NF32", "NF64", "NF128", - "NU", "NU8", "NU16", "NU32", "NU64",] + "NU", "NU8", "NU16", "NU32", "NU64"] case typ.kind of tyPointer: result = typeNameOrLiteral(typ, "void*") - of tyEnum: - if firstOrd(typ) < 0: - result = typeNameOrLiteral(typ, "NI32") - else: - case int(getSize(typ)) - of 1: result = typeNameOrLiteral(typ, "NU8") - of 2: result = typeNameOrLiteral(typ, "NU16") - of 4: result = typeNameOrLiteral(typ, "NI32") - of 8: result = typeNameOrLiteral(typ, "NI64") - else: - internalError(typ.sym.info, "getSimpleTypeDesc: " & $(getSize(typ))) - result = nil of tyString: discard cgsym(m, "NimStringDesc") result = typeNameOrLiteral(typ, "NimStringDesc*") @@ -264,6 +260,11 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = of tyInt..tyUInt64: result = typeNameOrLiteral(typ, NumericalTypeToStr[typ.kind]) of tyDistinct, tyRange, tyOrdinal: result = getSimpleTypeDesc(m, typ.sons[0]) + of tyStatic: + if typ.n != nil: result = getSimpleTypeDesc(m, lastSon typ) + else: internalError("tyStatic for getSimpleTypeDesc") + of tyGenericInst: + result = getSimpleTypeDesc(m, lastSon typ) else: result = nil proc pushType(m: BModule, typ: PType) = @@ -317,7 +318,7 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet): Rope = result = getTypeDescAux(m, t, check) proc paramStorageLoc(param: PSym): TStorageLoc = - if param.typ.skipTypes({tyVar, tyTypeDesc}).kind notin {tyArray, tyOpenArray}: + if param.typ.skipTypes({tyVar, tyTypeDesc}).kind notin {tyArray, tyOpenArray, tyVarargs, tyArrayConstr}: result = OnStack else: result = OnUnknown @@ -326,7 +327,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, check: var IntSet, declareEnvironment=true; weakDep=false) = params = nil - if (t.sons[0] == nil) or isInvalidReturnType(t.sons[0]): + if t.sons[0] == nil or isInvalidReturnType(t.sons[0]): rettype = ~"void" else: rettype = getTypeDescAux(m, t.sons[0], check) @@ -359,10 +360,10 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, addf(params, ", NI $1Len$2", [param.loc.r, j.rope]) inc(j) arr = arr.sons[0] - if (t.sons[0] != nil) and isInvalidReturnType(t.sons[0]): + if t.sons[0] != nil and isInvalidReturnType(t.sons[0]): var arr = t.sons[0] if params != nil: add(params, ", ") - if (mapReturnType(t.sons[0]) != ctArray): + if mapReturnType(t.sons[0]) != ctArray: add(params, getTypeDescWeak(m, arr, check)) add(params, "*") else: @@ -424,7 +425,7 @@ proc genRecordFieldsAux(m: BModule, n: PNode, addf(result, "union{$n$1} $2;$n", [unionBody, uname]) of nkSym: field = n.sym - if field.typ.kind == tyEmpty: return + if field.typ.kind == tyVoid: return #assert(field.ast == nil) sname = mangleRecFieldName(field, rectype) if accessExpr != nil: ae = "$1.$2" % [accessExpr, sname] @@ -483,7 +484,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope, else: addf(result, " {$n", [name]) - var desc = getRecordFields(m, typ, check) + let desc = getRecordFields(m, typ, check) if desc == nil and not hasField: addf(result, "char dummy;$n", []) else: @@ -536,8 +537,8 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = result = getTypePre(m, t) if result != nil: return if containsOrIncl(check, t.id): - if isImportedCppType(typ) or isImportedCppType(t): return - internalError("cannot generate C type for: " & typeToString(typ)) + if not (isImportedCppType(typ) or isImportedCppType(t)): + internalError("cannot generate C type for: " & typeToString(typ)) # XXX: this BUG is hard to fix -> we need to introduce helper structs, # but determining when this needs to be done is hard. We should split # C type generation into an analysis and a code generation phase somehow. @@ -576,8 +577,37 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = result = getTypeDescAux(m, et, check) & star idTablePut(m.typeCache, t, result) of tyOpenArray, tyVarargs: - result = getTypeDescAux(m, t.sons[0], check) & "*" + result = getTypeDescWeak(m, t.sons[0], check) & "*" idTablePut(m.typeCache, t, result) + of tyRange, tyEnum: + let t = if t.kind == tyRange: t.lastSon else: t + result = cacheGetType(m.typeCache, t) + if result == nil: + result = getTypeName(t) + if not (isImportedCppType(t) or + (sfImportc in t.sym.flags and t.sym.magic == mNone)): + idTablePut(m.typeCache, t, result) + var size: int + if firstOrd(t) < 0: + addf(m.s[cfsTypes], "typedef NI32 $1;$n", [result]) + size = 4 + else: + size = int(getSize(t)) + case size + of 1: addf(m.s[cfsTypes], "typedef NU8 $1;$n", [result]) + of 2: addf(m.s[cfsTypes], "typedef NU16 $1;$n", [result]) + of 4: addf(m.s[cfsTypes], "typedef NI32 $1;$n", [result]) + of 8: addf(m.s[cfsTypes], "typedef NI64 $1;$n", [result]) + else: internalError(t.sym.info, "getTypeDescAux: enum") + let owner = hashOwner(t.sym) + if not gDebugInfo.hasEnum(t.sym.name.s, t.sym.info.line, owner): + var vals: seq[(string, int)] = @[] + for i in countup(0, t.n.len - 1): + assert(t.n.sons[i].kind == nkSym) + let field = t.n.sons[i].sym + vals.add((field.name.s, field.position.int)) + gDebugInfo.registerEnum(EnumDesc(size: size, owner: owner, id: t.sym.id, + name: t.sym.name.s, values: vals)) of tyProc: result = getTypeName(t) idTablePut(m.typeCache, t, result) @@ -642,7 +672,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = chunkStart = i let typeInSlot = resolveStarsInCppType(typ, idx + 1, stars) - if typeInSlot == nil or typeInSlot.kind == tyEmpty: + if typeInSlot == nil or typeInSlot.kind == tyVoid: result.add(~"void") else: result.add getTypeDescAux(m, typeInSlot, check) @@ -654,7 +684,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = else: result = cppName & "<" for i in 1 .. typ.len-2: - if i > 1: result.add(", ") + if i > 1: result.add(" COMMA ") result.add(getTypeDescAux(m, typ.sons[i], check)) result.add("> ") # always call for sideeffects: @@ -673,16 +703,13 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = else: getTupleDesc(m, t, result, check) if not isImportedType(t): add(m.s[cfsTypes], recdesc) of tySet: - case int(getSize(t)) - of 1: result = rope("NU8") - of 2: result = rope("NU16") - of 4: result = rope("NU32") - of 8: result = rope("NU64") - else: - result = getTypeName(t) - idTablePut(m.typeCache, t, result) - if not isImportedType(t): - addf(m.s[cfsTypes], "typedef NU8 $1[$2];$n", + result = getTypeName(t.lastSon) & "Set" + idTablePut(m.typeCache, t, result) + if not isImportedType(t): + let s = int(getSize(t)) + case s + of 1, 2, 4, 8: addf(m.s[cfsTypes], "typedef NU$2 $1;$n", [result, rope(s*8)]) + else: addf(m.s[cfsTypes], "typedef NU8 $1[$2];$n", [result, rope(getSize(t))]) of tyGenericInst, tyDistinct, tyOrdinal, tyConst, tyMutable, tyIter, tyTypeDesc: @@ -704,7 +731,7 @@ type proc getClosureType(m: BModule, t: PType, kind: TClosureTypeKind): Rope = assert t.kind == tyProc var check = initIntSet() - result = getTempName() + result = getTempName(m) var rettype, desc: Rope genProcParams(m, t, rettype, desc, check, declareEnvironment=kind != clHalf) if not isImportedType(t): @@ -739,7 +766,7 @@ proc genProcHeader(m: BModule, prc: PSym): Rope = genCLineDir(result, prc.info) # using static is needed for inline procs if lfExportLib in prc.loc.flags: - if m.isHeaderFile: + if isHeaderFile in m.flags: result.add "N_LIB_IMPORT " else: result.add "N_LIB_EXPORT " @@ -819,7 +846,7 @@ proc genObjectFields(m: BModule, typ: PType, n: PNode, expr: Rope) = if L == 1: genObjectFields(m, typ, n.sons[0], expr) elif L > 0: - var tmp = getTempName() + var tmp = getTempName(m) addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(L)]) for i in countup(0, L-1): var tmp2 = getNimNode(m) @@ -891,7 +918,7 @@ proc genTupleInfo(m: BModule, typ: PType, name: Rope) = var expr = getNimNode(m) var length = sonsLen(typ) if length > 0: - var tmp = getTempName() + var tmp = getTempName(m) addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(length)]) for i in countup(0, length - 1): var a = typ.sons[i] @@ -915,7 +942,7 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope) = # anyway. We generate a cstring array and a loop over it. Exceptional # positions will be reset after the loop. genTypeInfoAux(m, typ, typ, name) - var nodePtrs = getTempName() + var nodePtrs = getTempName(m) var length = sonsLen(typ.n) addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [nodePtrs, rope(length)]) @@ -935,8 +962,8 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope) = if field.position != i or tfEnumHasHoles in typ.flags: addf(specialCases, "$1.offset = $2;$n", [elemNode, rope(field.position)]) hasHoles = true - var enumArray = getTempName() - var counter = getTempName() + var enumArray = getTempName(m) + var counter = getTempName(m) addf(m.s[cfsTypeInit1], "NI $1;$n", [counter]) addf(m.s[cfsTypeInit1], "static char* NIM_CONST $1[$2] = {$n$3};$n", [enumArray, rope(length), enumNames]) @@ -1004,9 +1031,12 @@ proc genTypeInfo(m: BModule, t: PType): Rope = [result, rope(typeToString(t))]) return "(&".rope & result & ")".rope case t.kind - of tyEmpty: result = rope"0" + of tyEmpty, tyVoid: result = rope"0" of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar: genTypeInfoAuxBase(m, t, t, result, rope"0") + of tyStatic: + if t.n != nil: result = genTypeInfo(m, lastSon t) + else: internalError("genTypeInfo(" & $t.kind & ')') of tyProc: if t.callConv != ccClosure: genTypeInfoAuxBase(m, t, t, result, rope"0") diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim index 6dfd7b52c..ecd98a2bf 100644 --- a/compiler/ccgutils.nim +++ b/compiler/ccgutils.nim @@ -93,7 +93,7 @@ proc getUniqueType*(key: PType): PType = # produced instead of ``NI``. result = key of tyEmpty, tyNil, tyExpr, tyStmt, tyPointer, tyString, - tyCString, tyNone, tyBigNum: + tyCString, tyNone, tyBigNum, tyVoid: result = gCanonicalTypes[k] if result == nil: gCanonicalTypes[k] = key @@ -188,7 +188,7 @@ proc mangle*(name: string): string = let c = name[i] case c of 'A'..'Z': - add(result, c.toLower) + add(result, c.toLowerAscii) of '_': discard of 'a'..'z', '0'..'9': diff --git a/compiler/cgen.nim b/compiler/cgen.nim index db376821c..9851ab0e2 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -64,8 +64,8 @@ proc isSimpleConst(typ: PType): bool = (t.kind == tyProc and t.callConv == ccClosure) proc useStringh(m: BModule) = - if not m.includesStringh: - m.includesStringh = true + if includesStringh notin m.flags: + incl m.flags, includesStringh discard lists.includeStr(m.headerFiles, "<string.h>") proc useHeader(m: BModule, sym: PSym) = @@ -129,7 +129,7 @@ proc ropecg(m: BModule, frmt: FormatStr, args: varargs[Rope]): Rope = if i - 1 >= start: add(result, substr(frmt, start, i - 1)) -template rfmt(m: BModule, fmt: string, args: varargs[Rope]): expr = +template rfmt(m: BModule, fmt: string, args: varargs[Rope]): untyped = ropecg(m, fmt, args) proc appcg(m: BModule, c: var Rope, frmt: FormatStr, @@ -215,7 +215,7 @@ proc accessThreadLocalVar(p: BProc, s: PSym) proc emulatedThreadVars(): bool {.inline.} proc genProc(m: BModule, prc: PSym) -template compileToCpp(m: BModule): expr = +template compileToCpp(m: BModule): untyped = gCmd == cmdCompileToCpp or sfCompileToCpp in m.module.flags include "ccgtypes.nim" @@ -301,7 +301,8 @@ proc resetLoc(p: BProc, loc: var TLoc) = proc constructLoc(p: BProc, loc: TLoc, isTemp = false) = let typ = skipTypes(loc.t, abstractRange) if not isComplexValueType(typ): - linefmt(p, cpsStmts, "$1 = 0;$n", rdLoc(loc)) + linefmt(p, cpsStmts, "$1 = ($2)0;$n", rdLoc(loc), + getTypeDesc(p.module, typ)) else: if not isTemp or containsGarbageCollectedRef(loc.t): # don't use memset for temporary values for performance if we can @@ -327,10 +328,12 @@ proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) = proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) = inc(p.labels) result.r = "LOC" & rope(p.labels) - linefmt(p, cpsLocals, "$1 $2;$n", getTypeDesc(p.module, t), result.r) + addf(p.blocks[0].sections[cpsLocals], + "$1 $2;$n", [getTypeDesc(p.module, t), result.r]) result.k = locTemp #result.a = - 1 - result.t = getUniqueType(t) + result.t = t + #result.t = getUniqueType(t) result.s = OnStack result.flags = {} constructLoc(p, result, not needsInit) @@ -498,7 +501,7 @@ proc loadDynamicLib(m: BModule, lib: PLib) = assert(lib != nil) if not lib.generated: lib.generated = true - var tmp = getGlobalTempName() + var tmp = getTempName(m) assert(lib.name == nil) lib.name = tmp # BUGFIX: cgsym has awful side-effects addf(m.s[cfsVars], "static void* $1;$n", [tmp]) @@ -666,7 +669,7 @@ proc genProcAux(m: BModule, prc: PSym) = fillResult(res) assignParam(p, res) if skipTypes(res.typ, abstractInst).kind == tyArray: - incl(res.loc.flags, lfIndirect) + #incl(res.loc.flags, lfIndirect) res.loc.s = OnUnknown for i in countup(1, sonsLen(prc.typ.n) - 1): @@ -1009,11 +1012,11 @@ proc genInitCode(m: BModule) = add(prc, m.postInitProc.s(cpsLocals)) add(prc, genSectionEnd(cpsLocals)) - if optStackTrace in m.initProc.options and not m.frameDeclared: + if optStackTrace in m.initProc.options and frameDeclared notin m.flags: # BUT: the generated init code might depend on a current frame, so # declare it nevertheless: - m.frameDeclared = true - if not m.preventStackTrace: + incl m.flags, frameDeclared + if preventStackTrace notin m.flags: var procname = makeCString(m.module.name.s) add(prc, initFrame(m.initProc, procname, m.module.info.quotedFilename)) else: @@ -1030,7 +1033,7 @@ proc genInitCode(m: BModule) = add(prc, m.initProc.s(cpsStmts)) add(prc, m.postInitProc.s(cpsStmts)) add(prc, genSectionEnd(cpsStmts)) - if optStackTrace in m.initProc.options and not m.preventStackTrace: + if optStackTrace in m.initProc.options and preventStackTrace notin m.flags: add(prc, deinitFrame(m.initProc)) add(prc, deinitGCFrame(m.initProc)) addf(prc, "}$N$N", []) @@ -1059,9 +1062,8 @@ proc genModule(m: BModule, cfile: string): Rope = result = getFileHeader(cfile) result.add(genMergeInfo(m)) - generateHeaders(m) - generateThreadLocalStorage(m) + generateHeaders(m) for i in countup(cfsHeaders, cfsProcs): add(result, genSectionStart(i)) add(result, m.s[i]) @@ -1083,6 +1085,7 @@ proc initProcOptions(m: BModule): TOptions = proc rawNewModule(module: PSym, filename: string): BModule = new(result) + result.tmpBase = rope("T" & $hashOwner(module) & "_") initLinkedList(result.headerFiles) result.declaredThings = initIntSet() result.declaredProtos = initIntSet() @@ -1099,12 +1102,12 @@ proc rawNewModule(module: PSym, filename: string): BModule = initNodeTable(result.dataCache) result.typeStack = @[] result.forwardedProcs = @[] - result.typeNodesName = getTempName() - result.nimTypesName = getTempName() + result.typeNodesName = getTempName(result) + result.nimTypesName = getTempName(result) # no line tracing for the init sections of the system module so that we # don't generate a TFrame which can confuse the stack botton initialization: if sfSystemModule in module.flags: - result.preventStackTrace = true + incl result.flags, preventStackTrace excl(result.preInitProc.options, optStackTrace) excl(result.postInitProc.options, optStackTrace) @@ -1125,11 +1128,13 @@ proc resetModule*(m: BModule) = initNodeTable(m.dataCache) m.typeStack = @[] m.forwardedProcs = @[] - m.typeNodesName = getTempName() - m.nimTypesName = getTempName() - m.preventStackTrace = sfSystemModule in m.module.flags + m.typeNodesName = getTempName(m) + m.nimTypesName = getTempName(m) + if sfSystemModule in m.module.flags: + incl m.flags, preventStackTrace + else: + excl m.flags, preventStackTrace nullify m.s - m.usesThreadVars = false m.typeNodes = 0 m.nimTypes = 0 nullify m.extensionLoaders @@ -1174,7 +1179,7 @@ proc myOpen(module: PSym): PPassContext = let f = if headerFile.len > 0: headerFile else: gProjectFull generatedHeader = rawNewModule(module, changeFileExt(completeCFilePath(f), hExt)) - generatedHeader.isHeaderFile = true + incl generatedHeader.flags, isHeaderFile proc writeHeader(m: BModule) = var result = getCopyright(m.filename) @@ -1306,7 +1311,7 @@ proc myClose(b: PPassContext, n: PNode): PNode = registerModuleToMain(m.module) if sfMainModule in m.module.flags: - m.objHasKidsValid = true + incl m.flags, objHasKidsValid var disp = generateMethodDispatchers() for i in 0..sonsLen(disp)-1: genProcAux(m, disp.sons[i].sym) genMainProc(m) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 187186373..a94950029 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -15,7 +15,7 @@ import from msgs import TLineInfo type - TLabel* = Rope # for the C generator a label is just a rope + TLabel* = Rope # for the C generator a label is just a rope TCFileSection* = enum # the sections a generated C file consists of cfsMergeInfo, # section containing merge information cfsHeaders, # section for C include file headers @@ -89,28 +89,32 @@ type # requires 'T x = T()' to become 'T x; x = T()' # (yes, C++ is weird like that) gcFrameId*: Natural # for the GC stack marking - gcFrameType*: Rope # the struct {} we put the GC markers into + gcFrameType*: Rope # the struct {} we put the GC markers into TTypeSeq* = seq[PType] + + Codegenflag* = enum + preventStackTrace, # true if stack traces need to be prevented + usesThreadVars, # true if the module uses a thread var + frameDeclared, # hack for ROD support so that we don't declare + # a frame var twice in an init proc + isHeaderFile, # C source file is the header file + includesStringh, # C source file already includes ``<string.h>`` + objHasKidsValid # whether we can rely on tfObjHasKids TCGen = object of TPassContext # represents a C source file + s*: TCFileSections # sections of the C file + flags*: set[Codegenflag] module*: PSym filename*: string - s*: TCFileSections # sections of the C file - preventStackTrace*: bool # true if stack traces need to be prevented - usesThreadVars*: bool # true if the module uses a thread var - frameDeclared*: bool # hack for ROD support so that we don't declare - # a frame var twice in an init proc - isHeaderFile*: bool # C source file is the header file - includesStringh*: bool # C source file already includes ``<string.h>`` - objHasKidsValid*: bool # whether we can rely on tfObjHasKids cfilename*: string # filename of the module (including path, # without extension) + tmpBase*: Rope # base for temp identifier generation typeCache*: TIdTable # cache the generated types forwTypeCache*: TIdTable # cache for forward declarations of types - declaredThings*: IntSet # things we have declared in this .c file - declaredProtos*: IntSet # prototypes we have declared in this .c file + declaredThings*: IntSet # things we have declared in this .c file + declaredProtos*: IntSet # prototypes we have declared in this .c file headerFiles*: TLinkedList # needed headers to include - typeInfoMarker*: IntSet # needed for generating type information + typeInfoMarker*: IntSet # needed for generating type information initProc*: BProc # code for init procedure postInitProc*: BProc # code to be executed after the init proc preInitProc*: BProc # code executed before the init proc diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index 312afec1a..624c5b183 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -63,19 +63,25 @@ proc sameMethodBucket(a, b: PSym): MethodResult = while true: aa = skipTypes(aa, {tyGenericInst}) bb = skipTypes(bb, {tyGenericInst}) - if (aa.kind == bb.kind) and (aa.kind in {tyVar, tyPtr, tyRef}): + if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef}: aa = aa.lastSon bb = bb.lastSon else: break if sameType(aa, bb): - if aa.kind == tyObject and result != Invalid: result = Yes + if aa.kind == tyObject and result != Invalid: + result = Yes elif aa.kind == tyObject and bb.kind == tyObject: let diff = inheritanceDiff(bb, aa) if diff < 0: - if result != Invalid: result = Yes + if result != Invalid: + result = Yes + else: + return No elif diff != high(int): result = Invalid + else: + return No else: return No @@ -181,7 +187,7 @@ proc cmpSignatures(a, b: PSym, relevantCols: IntSet): int = var aa = skipTypes(a.typ.sons[col], skipPtrs) var bb = skipTypes(b.typ.sons[col], skipPtrs) var d = inheritanceDiff(aa, bb) - if (d != high(int)): + if (d != high(int)) and d != 0: return d proc sortBucket(a: var TSymSeq, relevantCols: IntSet) = diff --git a/compiler/commands.nim b/compiler/commands.nim index 86bc1c205..de1197292 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -12,7 +12,7 @@ # We do this here before the 'import' statement so 'defined' does not get # confused with 'TGCMode.gcGenerational' etc. -template bootSwitch(name, expr, userString: expr): expr = +template bootSwitch(name, expr, userString) = # Helper to build boot constants, for debugging you can 'echo' the else part. const name = if expr: " " & userString else: "" @@ -158,7 +158,7 @@ var enableNotes: TNoteKinds disableNotes: TNoteKinds -proc processSpecificNote(arg: string, state: TSpecialWord, pass: TCmdLinePass, +proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, info: TLineInfo; orig: string) = var id = "" # arg = "X]:on|off" var i = 0 @@ -181,10 +181,13 @@ proc processSpecificNote(arg: string, state: TSpecialWord, pass: TCmdLinePass, case whichKeyword(substr(arg, i)) of wOn: incl(gNotes, n) + incl(gMainPackageNotes, n) incl(enableNotes, n) of wOff: excl(gNotes, n) + excl(gMainPackageNotes, n) incl(disableNotes, n) + excl(ForeignPackageNotes, n) else: localError(info, errOnOrOffExpectedButXFound, arg) proc processCompile(filename: string) = @@ -213,6 +216,16 @@ proc testCompileOptionArg*(switch, arg: string, info: TLineInfo): bool = of "size": result = contains(gOptions, optOptimizeSize) of "none": result = gOptions * {optOptimizeSpeed, optOptimizeSize} == {} else: localError(info, errNoneSpeedOrSizeExpectedButXFound, arg) + of "verbosity": result = $gVerbosity == arg + of "app": + case arg.normalize + of "gui": result = contains(gGlobalOptions, optGenGuiApp) + of "console": result = not contains(gGlobalOptions, optGenGuiApp) + of "lib": result = contains(gGlobalOptions, optGenDynLib) and + not contains(gGlobalOptions, optGenGuiApp) + of "staticlib": result = contains(gGlobalOptions, optGenStaticLib) and + not contains(gGlobalOptions, optGenGuiApp) + else: localError(info, errGuiConsoleOrLibExpectedButXFound, arg) else: invalidCmdLineOption(passCmd1, switch, info) proc testCompileOption*(switch: string, info: TLineInfo): bool = @@ -253,20 +266,18 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool = of "experimental": result = gExperimentalMode else: invalidCmdLineOption(passCmd1, switch, info) -proc processPath(path: string, notRelativeToProj = false, - cfginfo = unknownLineInfo()): string = +proc processPath(path: string, info: TLineInfo, + notRelativeToProj = false): string = let p = if notRelativeToProj or os.isAbsolute(path) or '$' in path or path[0] == '.': path else: options.gProjectPath / path - result = unixToNativePath(p % ["nimrod", getPrefixDir(), - "nim", getPrefixDir(), - "lib", libpath, - "home", removeTrailingDirSep(os.getHomeDir()), - "config", cfginfo.toFullPath().splitFile().dir, - "projectname", options.gProjectName, - "projectpath", options.gProjectPath]) + try: + result = pathSubs(p, info.toFullPath().splitFile().dir) + except ValueError: + localError(info, "invalid path: " & p) + result = p proc trackDirty(arg: string, info: TLineInfo) = var a = arg.split(',') @@ -307,19 +318,22 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = case switch.normalize of "path", "p": expectArg(switch, arg, pass, info) - addPath(processPath(arg, cfginfo=info), info) + addPath(processPath(arg, info), info) of "nimblepath", "babelpath": # keep the old name for compat if pass in {passCmd2, passPP} and not options.gNoNimblePath: expectArg(switch, arg, pass, info) - let path = processPath(arg, notRelativeToProj=true) + let path = processPath(arg, info, notRelativeToProj=true) nimblePath(path, info) of "nonimblepath", "nobabelpath": expectNoArg(switch, arg, pass, info) options.gNoNimblePath = true + options.lazyPaths.head = nil + options.lazyPaths.tail = nil + options.lazyPaths.counter = 0 of "excludepath": expectArg(switch, arg, pass, info) - let path = processPath(arg) + let path = processPath(arg, info) lists.excludePath(options.searchPaths, path) lists.excludePath(options.lazyPaths, path) if (len(path) > 0) and (path[len(path) - 1] == DirSep): @@ -328,7 +342,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = lists.excludePath(options.lazyPaths, strippedPath) of "nimcache": expectArg(switch, arg, pass, info) - options.nimcacheDir = processPath(arg, true) + options.nimcacheDir = processPath(arg, info, true) of "out", "o": expectArg(switch, arg, pass, info) options.outFile = arg @@ -339,7 +353,11 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = discard "allow for backwards compatibility, but don't do anything" of "define", "d": expectArg(switch, arg, pass, info) - defineSymbol(arg) + if {':', '='} in arg: + splitSwitch(arg, key, val, pass, info) + defineSymbol(key, val) + else: + defineSymbol(arg) of "undef", "u": expectArg(switch, arg, pass, info) undefSymbol(arg) @@ -407,6 +425,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = if processOnOffSwitchOrList({optHints}, arg, pass, info): listHints() of "threadanalysis": processOnOffSwitchG({optThreadAnalysis}, arg, pass, info) of "stacktrace": processOnOffSwitch({optStackTrace}, arg, pass, info) + of "excessivestacktrace": processOnOffSwitchG({optExcessiveStackTrace}, arg, pass, info) of "linetrace": processOnOffSwitch({optLineTrace}, arg, pass, info) of "debugger": case arg.normalize @@ -493,13 +512,13 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = if pass in {passCmd2, passPP}: extccomp.addLinkOption(arg) of "cincludes": expectArg(switch, arg, pass, info) - if pass in {passCmd2, passPP}: cIncludes.add arg.processPath + if pass in {passCmd2, passPP}: cIncludes.add arg.processPath(info) of "clibdir": expectArg(switch, arg, pass, info) - if pass in {passCmd2, passPP}: cLibs.add arg.processPath + if pass in {passCmd2, passPP}: cLibs.add arg.processPath(info) of "clib": expectArg(switch, arg, pass, info) - if pass in {passCmd2, passPP}: cLinkedLibs.add arg.processPath + if pass in {passCmd2, passPP}: cLinkedLibs.add arg.processPath(info) of "header": headerFile = arg incl(gGlobalOptions, optGenIndex) @@ -540,6 +559,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = gNotes = NotesVerbosity[gVerbosity] incl(gNotes, enableNotes) excl(gNotes, disableNotes) + gMainPackageNotes = gNotes of "parallelbuild": expectArg(switch, arg, pass, info) gNumberOfProcessors = parseInt(arg) @@ -572,7 +592,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = of "colors": processOnOffSwitchG({optUseColors}, arg, pass, info) of "lib": expectArg(switch, arg, pass, info) - libpath = processPath(arg, notRelativeToProj=true) + libpath = processPath(arg, info, notRelativeToProj=true) of "putenv": expectArg(switch, arg, pass, info) splitSwitch(arg, key, val, pass, info) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 60e8f2826..bcd9592e6 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -19,8 +19,8 @@ var gSymbols: StringTableRef const catNone = "false" -proc defineSymbol*(symbol: string) = - gSymbols[symbol] = "true" +proc defineSymbol*(symbol: string, value: string = "true") = + gSymbols[symbol] = value proc undefSymbol*(symbol: string) = gSymbols[symbol] = catNone @@ -62,6 +62,11 @@ proc isDefined*(symbol: string): bool = proc isDefined*(symbol: PIdent): bool = isDefined(symbol.s) +proc lookupSymbol*(symbol: string): string = + result = if isDefined(symbol): gSymbols[symbol] else: nil + +proc lookupSymbol*(symbol: PIdent): string = lookupSymbol(symbol.s) + iterator definedSymbolNames*: string = for key, val in pairs(gSymbols): if val != catNone: yield key @@ -93,3 +98,4 @@ proc initDefines*() = defineSymbol("nimtypedescfixed") defineSymbol("nimKnowsNimvm") defineSymbol("nimArrIdx") + defineSymbol("nimImmediateDeprecated") diff --git a/compiler/debuginfo.nim b/compiler/debuginfo.nim new file mode 100644 index 000000000..8589730b9 --- /dev/null +++ b/compiler/debuginfo.nim @@ -0,0 +1,81 @@ +# +# +# The Nim Compiler +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## The compiler can generate debuginfo to help debuggers in translating back from C/C++/JS code +## to Nim. The data structure has been designed to produce something useful with Nim's marshal +## module. + +type + FilenameHash* = uint32 + FilenameMapping* = object + package*, file*: string + mangled*: FilenameHash + EnumDesc* = object + size*: int + owner*: FilenameHash + id*: int + name*: string + values*: seq[(string, int)] + DebugInfo* = object + version*: int + files*: seq[FilenameMapping] + enums*: seq[EnumDesc] + conflicts*: bool + +proc sdbmHash(hash: FilenameHash, c: char): FilenameHash {.inline.} = + return FilenameHash(c) + (hash shl 6) + (hash shl 16) - hash + +proc sdbmHash(package, file: string): FilenameHash = + template `&=`(x, c) = x = sdbmHash(x, c) + result = 0 + for i in 0..<package.len: + result &= package[i] + result &= '.' + for i in 0..<file.len: + result &= file[i] + +proc register*(self: var DebugInfo; package, file: string): FilenameHash = + result = sdbmHash(package, file) + for f in self.files: + if f.mangled == result: + if f.package == package and f.file == file: return + self.conflicts = true + break + self.files.add(FilenameMapping(package: package, file: file, mangled: result)) + +proc hasEnum*(self: DebugInfo; ename: string; id: int; owner: FilenameHash): bool = + for en in self.enums: + if en.owner == owner and en.name == ename and en.id == id: return true + +proc registerEnum*(self: var DebugInfo; ed: EnumDesc) = + self.enums.add ed + +proc init*(self: var DebugInfo) = + self.version = 1 + self.files = @[] + self.enums = @[] + +var gDebugInfo*: DebugInfo +debuginfo.init gDebugInfo + +import marshal, streams + +proc writeDebugInfo*(self: var DebugInfo; file: string) = + let s = newFileStream(file, fmWrite) + store(s, self) + s.close + +proc writeDebugInfo*(file: string) = writeDebugInfo(gDebugInfo, file) + +proc loadDebugInfo*(self: var DebugInfo; file: string) = + let s = newFileStream(file, fmRead) + load(s, self) + s.close + +proc loadDebugInfo*(file: string) = loadDebugInfo(gDebugInfo, file) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 8555ec4f0..bbbec081a 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -14,7 +14,7 @@ import ast, strutils, strtabs, options, msgs, os, ropes, idents, wordrecg, syntaxes, renderer, lexer, rstast, rst, rstgen, times, highlite, - importer, sempass2, json, xmltree, cgi, typesrenderer + importer, sempass2, json, xmltree, cgi, typesrenderer, astalgo type TSections = array[TSymKind, Rope] @@ -25,9 +25,32 @@ type indexValFilename: string analytics: string # Google Analytics javascript, "" if doesn't exist seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML. + jArray: JsonNode + types: TStrTable PDoc* = ref TDocumentor ## Alias to type less. +proc whichType(d: PDoc; n: PNode): PSym = + if n.kind == nkSym: + if d.types.strTableContains(n.sym): + result = n.sym + else: + for i in 0..<safeLen(n): + let x = whichType(d, n[i]) + if x != nil: return x + +proc attachToType(d: PDoc; p: PSym): PSym = + let params = p.ast.sons[paramsPos] + # first check the first parameter, then the return type, + # then the other parameter: + template check(i) = + result = whichType(d, params[i]) + if result != nil: return result + + if params.len > 1: check(1) + if params.len > 0: check(0) + for i in 2..<params.len: check(i) + proc compilerMsgHandler(filename: string, line, col: int, msgKind: rst.MsgKind, arg: string) {.procvar.} = # translate msg kind: @@ -81,6 +104,8 @@ proc newDocumentor*(filename: string, config: StringTableRef): PDoc = result.seenSymbols = newStringTable(modeCaseInsensitive) result.id = 100 + result.jArray = newJArray() + initStrTable result.types proc dispA(dest: var Rope, xml, tex: string, args: openArray[Rope]) = if gCmd != cmdRst2tex: addf(dest, xml, args) @@ -226,10 +251,27 @@ proc getName(d: PDoc, n: PNode, splitAfter = -1): string = result = esc(d.target, "`") for i in 0.. <n.len: result.add(getName(d, n[i], splitAfter)) result.add esc(d.target, "`") + of nkOpenSymChoice, nkClosedSymChoice: + result = getName(d, n[0], splitAfter) else: internalError(n.info, "getName()") result = "" +proc getNameIdent(n: PNode): PIdent = + case n.kind + of nkPostfix: result = getNameIdent(n.sons[1]) + of nkPragmaExpr: result = getNameIdent(n.sons[0]) + of nkSym: result = n.sym.name + of nkIdent: result = n.ident + of nkAccQuoted: + var r = "" + for i in 0.. <n.len: r.add(getNameIdent(n[i]).s) + result = getIdent(r) + of nkOpenSymChoice, nkClosedSymChoice: + result = getNameIdent(n[0]) + else: + result = nil + proc getRstName(n: PNode): PRstNode = case n.kind of nkPostfix: result = getRstName(n.sons[1]) @@ -239,6 +281,8 @@ proc getRstName(n: PNode): PRstNode = of nkAccQuoted: result = getRstName(n.sons[0]) for i in 1 .. <n.len: result.text.add(getRstName(n[i]).text) + of nkOpenSymChoice, nkClosedSymChoice: + result = getRstName(n[0]) else: internalError(n.info, "getRstName()") result = nil @@ -311,7 +355,7 @@ proc docstringSummary(rstText: string): string = ## Also, we hope to not break the rst, but maybe we do. If there is any ## trimming done, an ellipsis unicode char is added. const maxDocstringChars = 100 - assert (rstText.len < 2 or (rstText[0] == '#' and rstText[1] == '#')) + assert(rstText.len < 2 or (rstText[0] == '#' and rstText[1] == '#')) result = rstText.substr(2).strip var pos = result.find('\L') if pos > 0: @@ -380,13 +424,27 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = "\\spanIdentifier{$1}", [rope(esc(d.target, literal))]) of tkSpaces, tkInvalid: add(result, literal) + of tkCurlyDotLe: + dispA(result, """<span class="Other pragmabegin">$1</span><div class="pragma">""", + "\\spanOther{$1}", + [rope(esc(d.target, literal))]) + of tkCurlyDotRi: + dispA(result, "</div><span class=\"Other pragmaend\">$1</span>", + "\\spanOther{$1}", + [rope(esc(d.target, literal))]) of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi, - tkBracketDotLe, tkBracketDotRi, tkCurlyDotLe, tkCurlyDotRi, tkParDotLe, + tkBracketDotLe, tkBracketDotRi, tkParDotLe, tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot, tkAccent, tkColonColon, tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr: dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}", [rope(esc(d.target, literal))]) + + if k in routineKinds and nameNode.kind == nkSym: + let att = attachToType(d, nameNode.sym) + if att != nil: + dispA(result, """<span class="attachedType" style="visibility:hidden">$1</span>""", "", + [rope esc(d.target, att.name.s)]) inc(d.id) let plainNameRope = rope(xmltree.escape(plainName.strip)) @@ -402,12 +460,11 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = var seeSrcRope: Rope = nil let docItemSeeSrc = getConfigVar("doc.item.seesrc") if docItemSeeSrc.len > 0 and options.docSeeSrcUrl.len > 0: - # XXX toFilename doesn't really work. We need to ensure that this keeps - # returning a relative path. + let path = n.info.toFilename.extractFilename.rope let urlRope = ropeFormatNamedVars(options.docSeeSrcUrl, - ["path", "line"], [n.info.toFilename.rope, rope($n.info.line)]) + ["path", "line"], [path, rope($n.info.line)]) dispA(seeSrcRope, "$1", "", [ropeFormatNamedVars(docItemSeeSrc, - ["path", "line", "url"], [n.info.toFilename.rope, + ["path", "line", "url"], [path, rope($n.info.line), urlRope])]) add(d.section[k], ropeFormatNamedVars(getConfigVar("doc.item"), @@ -431,8 +488,10 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = setIndexTerm(d[], symbolOrId, name, linkTitle, xmltree.escape(plainDocstring.docstringSummary)) + if k == skType and nameNode.kind == nkSym: + d.types.strTableAdd nameNode.sym -proc genJSONItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = +proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = if not isVisible(nameNode): return var name = getName(d, nameNode) @@ -441,8 +500,8 @@ proc genJSONItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments}) - result = %{ "name": %name, "type": %($k) } - + result = %{ "name": %name, "type": %($k), "line": %n.info.line, + "col": %n.info.col} if comm != nil and comm != "": result["description"] = %comm if r.buf != nil: @@ -492,46 +551,45 @@ proc generateDoc*(d: PDoc, n: PNode) = of nkFromStmt, nkImportExceptStmt: traceDeps(d, n.sons[0]) else: discard -proc generateJson(d: PDoc, n: PNode, jArray: JsonNode = nil): JsonNode = +proc add(d: PDoc; j: JsonNode) = + if j != nil: d.jArray.add j + +proc generateJson*(d: PDoc, n: PNode) = case n.kind of nkCommentStmt: if n.comment != nil and startsWith(n.comment, "##"): let stripped = n.comment.substr(2).strip - result = %{ "comment": %stripped } + d.add %{ "comment": %stripped, "line": %n.info.line, + "col": %n.info.col } of nkProcDef: when useEffectSystem: documentRaises(n) - result = genJSONItem(d, n, n.sons[namePos], skProc) + d.add genJsonItem(d, n, n.sons[namePos], skProc) of nkMethodDef: when useEffectSystem: documentRaises(n) - result = genJSONItem(d, n, n.sons[namePos], skMethod) + d.add genJsonItem(d, n, n.sons[namePos], skMethod) of nkIteratorDef: when useEffectSystem: documentRaises(n) - result = genJSONItem(d, n, n.sons[namePos], skIterator) + d.add genJsonItem(d, n, n.sons[namePos], skIterator) of nkMacroDef: - result = genJSONItem(d, n, n.sons[namePos], skMacro) + d.add genJsonItem(d, n, n.sons[namePos], skMacro) of nkTemplateDef: - result = genJSONItem(d, n, n.sons[namePos], skTemplate) + d.add genJsonItem(d, n, n.sons[namePos], skTemplate) of nkConverterDef: when useEffectSystem: documentRaises(n) - result = genJSONItem(d, n, n.sons[namePos], skConverter) + d.add genJsonItem(d, n, n.sons[namePos], skConverter) of nkTypeSection, nkVarSection, nkLetSection, nkConstSection: for i in countup(0, sonsLen(n) - 1): if n.sons[i].kind != nkCommentStmt: # order is always 'type var let const': - result = genJSONItem(d, n.sons[i], n.sons[i].sons[0], + d.add genJsonItem(d, n.sons[i], n.sons[i].sons[0], succ(skType, ord(n.kind)-ord(nkTypeSection))) of nkStmtList: - result = if jArray != nil: jArray else: newJArray() - for i in countup(0, sonsLen(n) - 1): - var r = generateJson(d, n.sons[i], result) - if r != nil: - result.add(r) - + generateJson(d, n.sons[i]) of nkWhenStmt: # generate documentation for the first branch only: - if not checkForFalse(n.sons[0].sons[0]) and jArray != nil: - discard generateJson(d, lastSon(n.sons[0]), jArray) + if not checkForFalse(n.sons[0].sons[0]): + generateJson(d, lastSon(n.sons[0])) else: discard proc genSection(d: PDoc, kind: TSymKind) = @@ -593,12 +651,36 @@ proc generateIndex*(d: PDoc) = writeIndexFile(d[], splitFile(options.outFile).dir / splitFile(d.filename).name & IndexExt) +proc getOutFile2(filename, ext, dir: string): string = + if gWholeProject: + let d = if options.outFile != "": options.outFile else: dir + createDir(d) + result = d / changeFileExt(filename, ext) + else: + result = getOutFile(filename, ext) + proc writeOutput*(d: PDoc, filename, outExt: string, useWarning = false) = var content = genOutFile(d) if optStdout in gGlobalOptions: writeRope(stdout, content) else: - writeRope(content, getOutFile(filename, outExt), useWarning) + writeRope(content, getOutFile2(filename, outExt, "htmldocs"), useWarning) + +proc writeOutputJson*(d: PDoc, filename, outExt: string, + useWarning = false) = + let content = %*{"orig": d.filename, + "nimble": getPackageName(d.filename), + "entries": d.jArray} + if optStdout in gGlobalOptions: + write(stdout, $content) + else: + var f: File + if open(f, getOutFile2(splitFile(filename).name, + outExt, "jsondocs"), fmWrite): + write(f, $content) + close(f) + else: + discard "fixme: error report" proc commandDoc*() = var ast = parseFile(gProjectMainIdx) @@ -629,18 +711,19 @@ proc commandRst2TeX*() = splitter = "\\-" commandRstAux(gProjectFull, TexExt) -proc commandJSON*() = +proc commandJson*() = var ast = parseFile(gProjectMainIdx) if ast == nil: return var d = newDocumentor(gProjectFull, options.gConfigVars) d.hasToc = true - var json = generateJson(d, ast) - var content = rope(pretty(json)) + generateJson(d, ast) + let json = d.jArray + let content = rope(pretty(json)) if optStdout in gGlobalOptions: writeRope(stdout, content) else: - echo getOutFile(gProjectFull, JsonExt) + #echo getOutFile(gProjectFull, JsonExt) writeRope(content, getOutFile(gProjectFull, JsonExt), useWarning = false) proc commandBuildIndex*() = diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index 27de06811..d70d5406c 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -19,21 +19,36 @@ type module: PSym PGen = ref TGen -proc close(p: PPassContext, n: PNode): PNode = +template closeImpl(body: untyped) {.dirty.} = var g = PGen(p) let useWarning = sfMainModule notin g.module.flags - if gWholeProject or sfMainModule in g.module.flags: - writeOutput(g.doc, g.module.filename, HtmlExt, useWarning) + #echo g.module.name.s, " ", g.module.owner.id, " ", gMainPackageId + if (g.module.owner.id == gMainPackageId and gWholeProject) or + sfMainModule in g.module.flags: + body try: generateIndex(g.doc) except IOError: discard +proc close(p: PPassContext, n: PNode): PNode = + closeImpl: + writeOutput(g.doc, g.module.filename, HtmlExt, useWarning) + +proc closeJson(p: PPassContext, n: PNode): PNode = + closeImpl: + writeOutputJson(g.doc, g.module.filename, ".json", useWarning) + proc processNode(c: PPassContext, n: PNode): PNode = result = n var g = PGen(c) generateDoc(g.doc, n) +proc processNodeJson(c: PPassContext, n: PNode): PNode = + result = n + var g = PGen(c) + generateJson(g.doc, n) + proc myOpen(module: PSym): PPassContext = var g: PGen new(g) @@ -44,6 +59,8 @@ proc myOpen(module: PSym): PPassContext = result = g const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close) +const docgen2JsonPass* = makePass(open = myOpen, process = processNodeJson, + close = closeJson) proc finishDoc2Pass*(project: string) = discard diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim index a5a132005..96ede44fd 100644 --- a/compiler/evaltempl.nim +++ b/compiler/evaltempl.nim @@ -59,7 +59,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) = evalTemplateAux(templ.sons[i], actual, c, res) result.add res -proc evalTemplateArgs(n: PNode, s: PSym): PNode = +proc evalTemplateArgs(n: PNode, s: PSym; fromHlo: bool): PNode = # if the template has zero arguments, it can be called without ``()`` # `n` is then a nkSym or something similar var totalParams = case n.kind @@ -75,11 +75,11 @@ proc evalTemplateArgs(n: PNode, s: PSym): PNode = # their bodies. We could try to fix this, but it may be # wiser to just deprecate immediate templates and macros # now that we have working untyped parameters. - genericParams = if sfImmediate in s.flags: 0 + genericParams = if sfImmediate in s.flags or fromHlo: 0 else: s.ast[genericParamsPos].len expectedRegularParams = <s.typ.len givenRegularParams = totalParams - genericParams - + if givenRegularParams < 0: givenRegularParams = 0 if totalParams > expectedRegularParams + genericParams: globalError(n.info, errWrongNumberOfArguments) @@ -104,14 +104,14 @@ proc evalTemplateArgs(n: PNode, s: PSym): PNode = var evalTemplateCounter* = 0 # to prevent endless recursion in templates instantiation -proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym): PNode = +proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym; fromHlo=false): PNode = inc(evalTemplateCounter) if evalTemplateCounter > 100: globalError(n.info, errTemplateInstantiationTooNested) result = n # replace each param by the corresponding node: - var args = evalTemplateArgs(n, tmpl) + var args = evalTemplateArgs(n, tmpl, fromHlo) var ctx: TemplCtx ctx.owner = tmpl ctx.genSymOwner = genSymOwner diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 16a8b8bd4..2dcfc0226 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -16,6 +16,8 @@ import lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs, securehash, streams +from debuginfo import writeDebugInfo + type TSystemCC* = enum ccNone, ccGcc, ccLLVM_Gcc, ccCLang, ccLcc, ccBcc, ccDmc, ccWcc, ccVcc, @@ -59,7 +61,7 @@ type # When adding new compilers, the cmake sources could be a good reference: # http://cmake.org/gitweb?p=cmake.git;a=tree;f=Modules/Platform; -template compiler(name: expr, settings: stmt): stmt {.immediate.} = +template compiler(name, settings: untyped): untyped = proc name: TInfoCC {.compileTime.} = settings # GNU C and C++ Compiler @@ -736,6 +738,8 @@ proc callCCompiler*(projectfile: string) = if not noAbsolutePaths(): if not exefile.isAbsolute(): exefile = joinPath(splitFile(projectfile).dir, exefile) + if optCDebug in gGlobalOptions: + writeDebugInfo(exefile.changeFileExt("ndb")) exefile = quoteShell(exefile) let linkOptions = getLinkOptions() & " " & getConfigVar(cCompiler, ".options.linker") @@ -750,7 +754,7 @@ proc callCCompiler*(projectfile: string) = "lib", quoteShell(libpath)]) if optCompileOnly notin gGlobalOptions: execExternalProgram(linkCmd, - if gVerbosity > 1: hintExecuting else: hintLinking) + if optListCmd in gGlobalOptions or gVerbosity > 1: hintExecuting else: hintLinking) else: linkCmd = "" if optGenScript in gGlobalOptions: diff --git a/compiler/guards.nim b/compiler/guards.nim index 5ad932e48..4e887d3e3 100644 --- a/compiler/guards.nim +++ b/compiler/guards.nim @@ -728,12 +728,12 @@ proc simpleSlice*(a, b: PNode): BiggestInt = result = -1 -template isMul(x): expr = x.getMagic in someMul -template isDiv(x): expr = x.getMagic in someDiv -template isAdd(x): expr = x.getMagic in someAdd -template isSub(x): expr = x.getMagic in someSub -template isVal(x): expr = x.kind in {nkCharLit..nkUInt64Lit} -template isIntVal(x, y): expr = x.intVal == y +template isMul(x): untyped = x.getMagic in someMul +template isDiv(x): untyped = x.getMagic in someDiv +template isAdd(x): untyped = x.getMagic in someAdd +template isSub(x): untyped = x.getMagic in someSub +template isVal(x): untyped = x.kind in {nkCharLit..nkUInt64Lit} +template isIntVal(x, y): untyped = x.intVal == y import macros @@ -776,8 +776,8 @@ proc isMinusOne(n: PNode): bool = proc pleViaModel(model: TModel; aa, bb: PNode): TImplication proc ple(m: TModel; a, b: PNode): TImplication = - template `<=?`(a,b): expr = ple(m,a,b) == impYes - template `>=?`(a,b): expr = ple(m, nkIntLit.newIntNode(b), a) == impYes + template `<=?`(a,b): untyped = ple(m,a,b) == impYes + template `>=?`(a,b): untyped = ple(m, nkIntLit.newIntNode(b), a) == impYes # 0 <= 3 if a.isValue and b.isValue: diff --git a/compiler/hlo.nim b/compiler/hlo.nim index 6cc9567af..de0fa6216 100644 --- a/compiler/hlo.nim +++ b/compiler/hlo.nim @@ -24,7 +24,7 @@ proc evalPattern(c: PContext, n, orig: PNode): PNode = of skMacro: result = semMacroExpr(c, n, orig, s) of skTemplate: - result = semTemplateExpr(c, n, s) + result = semTemplateExpr(c, n, s, {efFromHlo}) else: result = semDirectOp(c, n, {}) if optHints in gOptions and hintPattern in gNotes: diff --git a/compiler/idgen.nim b/compiler/idgen.nim index 906c16546..333772705 100644 --- a/compiler/idgen.nim +++ b/compiler/idgen.nim @@ -11,7 +11,7 @@ import idents, strutils, os, options -var gFrontEndId, gBackendId*: int +var gFrontEndId*: int const debugIds* = false @@ -30,10 +30,6 @@ proc getID*(): int {.inline.} = result = gFrontEndId inc(gFrontEndId) -proc backendId*(): int {.inline.} = - result = gBackendId - inc(gBackendId) - proc setId*(id: int) {.inline.} = gFrontEndId = max(gFrontEndId, id + 1) @@ -49,7 +45,6 @@ proc toGid(f: string): string = proc saveMaxIds*(project: string) = var f = open(project.toGid, fmWrite) f.writeLine($gFrontEndId) - f.writeLine($gBackendId) f.close() proc loadMaxIds*(project: string) = @@ -61,5 +56,4 @@ proc loadMaxIds*(project: string) = if f.readLine(line): var backEndId = parseInt(line) gFrontEndId = max(gFrontEndId, frontEndId) - gBackendId = max(gBackendId, backEndId) f.close() diff --git a/compiler/importer.nim b/compiler/importer.nim index 86993358b..dd2c4d954 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -22,7 +22,11 @@ proc getModuleName*(n: PNode): string = # The proc won't perform any checks that the path is actually valid case n.kind of nkStrLit, nkRStrLit, nkTripleStrLit: - result = unixToNativePath(n.strVal) + try: + result = pathSubs(n.strVal, n.info.toFullPath().splitFile().dir) + except ValueError: + localError(n.info, "invalid path: " & n.strVal) + result = n.strVal of nkIdent: result = n.ident.s of nkSym: @@ -172,7 +176,7 @@ proc evalImport(c: PContext, n: PNode): PNode = var m = myImportModule(c, n.sons[i]) if m != nil: # ``addDecl`` needs to be done before ``importAllSymbols``! - addDecl(c, m) # add symbol to symbol table of module + addDecl(c, m, n.info) # add symbol to symbol table of module importAllSymbolsExcept(c, m, emptySet) #importForwarded(c, m.ast, emptySet) @@ -182,7 +186,7 @@ proc evalFrom(c: PContext, n: PNode): PNode = var m = myImportModule(c, n.sons[0]) if m != nil: n.sons[0] = newSymNode(m) - addDecl(c, m) # add symbol to symbol table of module + addDecl(c, m, n.info) # add symbol to symbol table of module for i in countup(1, sonsLen(n) - 1): if n.sons[i].kind != nkNilLit: importSymbol(c, n.sons[i], m) @@ -193,7 +197,7 @@ proc evalImportExcept*(c: PContext, n: PNode): PNode = var m = myImportModule(c, n.sons[0]) if m != nil: n.sons[0] = newSymNode(m) - addDecl(c, m) # add symbol to symbol table of module + addDecl(c, m, n.info) # add symbol to symbol table of module var exceptSet = initIntSet() for i in countup(1, sonsLen(n) - 1): let ident = lookups.considerQuotedIdent(n.sons[i]) diff --git a/compiler/installer.ini b/compiler/installer.ini index 12d9baf82..b802f08f1 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -8,7 +8,7 @@ Platforms: """ windows: i386;amd64 linux: i386;amd64;powerpc64;arm;sparc;mips;mipsel;powerpc;powerpc64el;arm64 macosx: i386;amd64;powerpc64 - solaris: i386;amd64;sparc + solaris: i386;amd64;sparc;sparc64 freebsd: i386;amd64 netbsd: i386;amd64 openbsd: i386;amd64 @@ -48,8 +48,8 @@ Start: "doc/overview.html" [Other] Files: "readme.txt;install.txt;contributors.txt;copying.txt" Files: "makefile" -Files: "*.ini" Files: "koch.nim" +Files: "install_nimble.nims" Files: "icons/nim.ico" Files: "icons/nim.rc" @@ -60,176 +60,26 @@ Files: "icons/koch.rc" Files: "icons/koch.res" Files: "icons/koch_icon.o" -Files: "compiler/readme.txt" -Files: "compiler/installer.ini" -Files: "compiler/*.cfg" -Files: "compiler/*.nim" -Files: "doc/*.txt" -Files: "doc/manual/*.txt" -Files: "doc/*.nim" -Files: "doc/*.cfg" -Files: "compiler/nimfix/*.nim" -Files: "compiler/nimfix/*.cfg" -Files: "compiler/nimsuggest/*.nim" -Files: "compiler/nimsuggest/*.cfg" -Files: "compiler/plugins/locals/*.nim" -Files: "compiler/plugins/*.nim" -Files: "tools/*.nim" -Files: "tools/*.cfg" -Files: "tools/*.tmpl" -Files: "tools/niminst/*.nim" -Files: "tools/niminst/*.cfg" -Files: "tools/niminst/*.tmpl" -Files: "tools/niminst/*.nsh" +Files: "compiler" +Files: "doc" +Files: "tools" Files: "web/website.ini" +Files: "web/ticker.html" Files: "web/*.nim" -Files: "web/*.txt" +Files: "web/*.rst" +Files: "web/*.csv" +Files: "web/news/*.rst" Files: "bin/nimblepkg/*.nim" Files: "bin/nimblepkg/*.cfg" [Lib] -Files: "lib/nimbase.h" -Files: "lib/*.nim" -Files: "lib/*.cfg" -Files: "lib/*.nimble" - -Files: "lib/system/*.nim" -Files: "lib/core/*.nim" -Files: "lib/pure/*.nim" -Files: "lib/pure/*.cfg" -Files: "lib/pure/collections/*.nim" -Files: "lib/pure/concurrency/*.nim" -Files: "lib/pure/unidecode/*.nim" -Files: "lib/pure/concurrency/*.cfg" -Files: "lib/impure/*.nim" -Files: "lib/impure/nre/private/*.nim" -Files: "lib/wrappers/*.nim" -Files: "lib/arch/*.nim" - -Files: "lib/wrappers/readline/*.nim" -Files: "lib/wrappers/linenoise/*.nim" -Files: "lib/wrappers/linenoise/*.c" -Files: "lib/wrappers/linenoise/*.h" - -Files: "lib/windows/*.nim" -Files: "lib/posix/*.nim" -Files: "lib/js/*.nim" -Files: "lib/packages/docutils/*.nim" - -Files: "lib/deprecated/core/*.nim" -Files: "lib/deprecated/pure/*.nim" -Files: "lib/deprecated/pure/*.cfg" +Files: "lib" [Other] -Files: "examples/*.nim" -Files: "examples/c++iface/*.nim" -Files: "examples/objciface/*.nim" -Files: "examples/cross_calculator/" - -Files: "examples/*.html" -Files: "examples/*.txt" -Files: "examples/*.cfg" -Files: "examples/*.tmpl" +Files: "examples" +#Files: "dist/nimble" -Files: "tests/actiontable/*.nim" -Files: "tests/alias/*.nim" -Files: "tests/ambsym/*.nim" -Files: "tests/array/*.nim" -Files: "tests/assign/*.nim" -Files: "tests/astoverload/*.nim" -Files: "tests/async/*.nim" -Files: "tests/benchmarks/*.nim" -Files: "tests/bind/*.nim" -Files: "tests/borrow/*.nim" -Files: "tests/casestmt/*.nim" -Files: "tests/ccgbugs/*.nim" -Files: "tests/clearmsg/*.nim" -Files: "tests/closure/*.nim" -Files: "tests/cnstseq/*.nim" -Files: "tests/collections/*.nim" -Files: "tests/compiles/*.nim" -Files: "tests/concat/*.nim" -Files: "tests/concepts/*.nim" -Files: "tests/constr/*.nim" -Files: "tests/constraints/*.nim" -Files: "tests/controlflow/*.nim" -Files: "tests/converter/*.nim" -Files: "tests/cpp/*.nim" -Files: "tests/defaultprocparam/*.nim" -Files: "tests/deprecated/*.nim" -Files: "tests/destructor/*.nim" -Files: "tests/dir with space/*.nim" -Files: "tests/discard/*.nim" -Files: "tests/distinct/*.nim" -Files: "tests/dll/*.nim" -Files: "tests/effects/*.nim" -Files: "tests/enum/*.nim" -Files: "tests/exception/*.nim" -Files: "tests/exprs/*.nim" -Files: "tests/fields/*.nim" -Files: "tests/float/*.nim" -Files: "tests/friends/*.nim" -Files: "tests/gc/*.nim" -Files: "tests/generics/*.nim" -Files: "tests/gensym/*.nim" -Files: "tests/global/*.nim" -Files: "tests/implicit/*.nim" -Files: "tests/init/*.nim" -Files: "tests/iter/*.nim" -Files: "tests/js/*.nim" -Files: "tests/js/*.cfg" -Files: "tests/let/*.nim" -Files: "tests/lexer/*.nim" -Files: "tests/lookups/*.nim" -Files: "tests/macros/*.nim" -Files: "tests/magics/*.nim" -Files: "tests/metatype/*.nim" -Files: "tests/method/*.nim" -Files: "tests/misc/*.nim" -Files: "tests/modules/*.nim" -Files: "tests/namedparams/*.nim" -Files: "tests/notnil/*.nim" -Files: "tests/objects/*.nim" -Files: "tests/objvariant/*.nim" -Files: "tests/openarray/*.nim" -Files: "tests/osproc/*.nim" -Files: "tests/overflw/*.nim" -Files: "tests/overload/*.nim" -Files: "tests/parallel/*.nim" -Files: "tests/parallel/*.cfg" -Files: "tests/parser/*.nim" -Files: "tests/pragmas/*.nim" -Files: "tests/proc/*.nim" -Files: "tests/procvar/*.nim" -Files: "tests/range/*.nim" -Files: "tests/rodfiles/*.nim" -Files: "tests/seq/*.nim" -Files: "tests/sets/*.nim" -Files: "tests/showoff/*.nim" -Files: "tests/specialops/*.nim" -Files: "tests/stdlib/*.nim" -Files: "tests/system/*.nim" -Files: "tests/template/*.nim" -Files: "tests/testament/*.nim" -Files: "tests/testdata/*.csv" -Files: "tests/testdata/*.html" -Files: "tests/testdata/*.json" -Files: "tests/testdata/*.txt" -Files: "tests/testdata/*.xml" -Files: "tests/threads/*.nim" -Files: "tests/threads/*.cfg" -Files: "tests/trmacros/*.nim" -Files: "tests/tuples/*.nim" -Files: "tests/typerel/*.nim" -Files: "tests/types/*.nim" -Files: "tests/usingstmt/*.nim" -Files: "tests/varres/*.nim" -Files: "tests/varstmt/*.nim" -Files: "tests/vm/*.nim" -Files: "tests/readme.txt" -Files: "tests/testament/css/*.css" -Files: "tests/testament/*.cfg" -Files: "lib/pure/unidecode/unidecode.dat" +Files: "tests" [Windows] Files: "bin/nim.exe" @@ -249,7 +99,7 @@ BinPath: r"bin;dist\mingw\bin;dist" Download: r"Documentation|doc|docs.zip|13824|http://nim-lang.org/download/docs-${version}.zip|overview.html" Download: r"C Compiler (MingW)|dist|mingw.zip|82944|http://nim-lang.org/download/${mingw}.zip" Download: r"Support DLL's|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls.zip" -Download: r"Aporia IDE|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.3.0.zip|aporia-0.3.0\bin\aporia.exe" +Download: r"Aporia IDE|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe" ; for now only NSIS supports optional downloads [UnixBin] @@ -279,3 +129,7 @@ buildDepends: "gcc (>= 4:4.3.2)" pkgDepends: "gcc (>= 4:4.3.2)" shortDesc: "The Nim Compiler" licenses: "bin/nim,MIT;lib/*,MIT;" + +[nimble] +pkgName: "compiler" +pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt" diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 2da7db900..80bcd2b0e 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -8,7 +8,7 @@ # # This is the JavaScript code generator. -# Soon also a PHP code generator. ;-) +# Also a PHP code generator. ;-) discard """ The JS code generator contains only 2 tricks: @@ -48,6 +48,7 @@ type etyNull, # null type etyProc, # proc type etyBool, # bool type + etySeq, # Nim seq or string type etyInt, # JavaScript's int etyFloat, # JavaScript's float etyString, # JavaScript's string @@ -71,7 +72,7 @@ type isLoop: bool # whether it's a 'block' or 'while' TGlobals = object - typeInfo, code: Rope + typeInfo, constants, code: Rope forwarded: seq[PSym] generatedSyms: IntSet typeInfoGenerated: IntSet @@ -94,7 +95,7 @@ type up: PProc # up the call chain; required for closure support declaredGlobals: IntSet -template `|`(a, b: expr): expr {.immediate, dirty.} = +template `|`(a, b: untyped): untyped {.dirty.} = (if p.target == targetJS: a else: b) proc newGlobals(): PGlobals = @@ -156,15 +157,18 @@ proc mapType(typ: PType): TJSTypeKind = of tyBool: result = etyBool of tyFloat..tyFloat128: result = etyFloat of tySet: result = etyObject # map a set to a table - of tyString, tySequence: result = etyInt # little hack to get right semantics + of tyString, tySequence: result = etySeq of tyObject, tyArray, tyArrayConstr, tyTuple, tyOpenArray, tyBigNum, tyVarargs: result = etyObject of tyNil: result = etyNull of tyGenericInst, tyGenericParam, tyGenericBody, tyGenericInvocation, tyNone, tyFromExpr, tyForward, tyEmpty, tyFieldAccessor, - tyExpr, tyStmt, tyStatic, tyTypeDesc, tyTypeClasses: + tyExpr, tyStmt, tyTypeDesc, tyTypeClasses, tyVoid: result = etyNone + of tyStatic: + if t.n != nil: result = mapType(lastSon t) + else: result = etyNone of tyProc: result = etyProc of tyCString: result = etyString @@ -237,7 +241,7 @@ proc useMagic(p: PProc, name: string) = internalAssert s.kind in {skProc, skMethod, skConverter} if not p.g.generatedSyms.containsOrIncl(s.id): let code = genProc(p, s) - add(p.g.code, code) + add(p.g.constants, code) else: # we used to exclude the system module from this check, but for DLL # generation support this sloppyness leads to hard to detect bugs, so @@ -359,8 +363,8 @@ const # magic checked op; magic unchecked op; checked op; unchecked op ["", "", "($1 == $2)", "($1 == $2)"], # EqUntracedRef ["", "", "($1 <= $2)", "($1 <= $2)"], # LePtr ["", "", "($1 < $2)", "($1 < $2)"], # LtPtr - ["", "", "($1 == $2)", "($1 == $2)"], # EqCString ["", "", "($1 != $2)", "($1 != $2)"], # Xor + ["", "", "($1 == $2)", "($1 == $2)"], # EqCString ["", "", "($1 == $2)", "($1 == $2)"], # EqProc ["negInt", "", "negInt($1)", "-($1)"], # UnaryMinusI ["negInt64", "", "negInt64($1)", "-($1)"], # UnaryMinusI64 @@ -817,7 +821,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = addf(p.body, "$#[$#] = chr($#);$n", [a.rdLoc, b.rdLoc, c.rdLoc]) return - let xtyp = mapType(p, x.typ) + var xtyp = mapType(p, x.typ) if x.kind == nkHiddenDeref and x.sons[0].kind == nkCall and xtyp != etyObject: gen(p, x.sons[0], a) @@ -829,7 +833,18 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = gen(p, y, b) + # we don't care if it's an etyBaseIndex (global) of a string, it's + # still a string that needs to be copied properly: + if x.typ.skipTypes(abstractInst).kind in {tySequence, tyString}: + xtyp = etySeq case xtyp + of etySeq: + if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: + addf(p.body, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) + else: + useMagic(p, "nimCopy") + addf(p.body, "$1 = nimCopy(null, $2, $3);$n", + [a.rdLoc, b.res, genTypeInfo(p, y.typ)]) of etyObject: if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: addf(p.body, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) @@ -981,7 +996,7 @@ proc genArrayAccess(p: PProc, n: PNode, r: var TCompRes) = r.res = "nimAt($1, $2)" % [r.address, r.res] elif ty.kind in {tyString, tyCString}: # XXX this needs to be more like substr($1,$2) - r.res = "ord($1[$2])" % [r.address, r.res] + r.res = "ord(@$1[$2])" % [r.address, r.res] else: r.res = "$1[$2]" % [r.address, r.res] else: @@ -1049,6 +1064,8 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) = else: internalError(n.sons[0].info, "expr(nkBracketExpr, " & $kindOfIndexedExpr & ')') of nkObjDownConv: gen(p, n.sons[0], r) + of nkHiddenDeref: + gen(p, n.sons[0].sons[0], r) else: internalError(n.sons[0].info, "genAddr: " & $n.sons[0].kind) proc thisParam(p: PProc; typ: PType): PType = @@ -1215,11 +1232,13 @@ proc genOtherArg(p: PProc; n: PNode; i: int; typ: PType; genArgNoParam(p, it, r) else: genArg(p, it, paramType.sym, r) + inc generated proc genPatternCall(p: PProc; n: PNode; pat: string; typ: PType; r: var TCompRes) = var i = 0 var j = 1 + r.kind = resExpr while i < pat.len: case pat[i] of '@': @@ -1233,31 +1252,43 @@ proc genPatternCall(p: PProc; n: PNode; pat: string; typ: PType; genOtherArg(p, n, j, typ, generated, r) inc j inc i + of '\31': + # unit separator + add(r.res, "#") + inc i + of '\29': + # group separator + add(r.res, "@") + inc i else: let start = i while i < pat.len: - if pat[i] notin {'@', '#'}: inc(i) + if pat[i] notin {'@', '#', '\31', '\29'}: inc(i) else: break if i - 1 >= start: add(r.res, substr(pat, start, i - 1)) proc genInfixCall(p: PProc, n: PNode, r: var TCompRes) = # don't call '$' here for efficiency: - let pat = n.sons[0].sym.loc.r.data - internalAssert pat != nil - if pat.contains({'#', '(', '@'}): - var typ = skipTypes(n.sons[0].typ, abstractInst) - assert(typ.kind == tyProc) - genPatternCall(p, n, pat, typ, r) - return - gen(p, n.sons[1], r) - if r.typ == etyBaseIndex: - if r.address == nil: - globalError(n.info, "cannot invoke with infix syntax") - r.res = "$1[$2]" % [r.address, r.res] - r.address = nil - r.typ = etyNone - add(r.res, "." | "->") + let f = n[0].sym + if f.loc.r == nil: f.loc.r = mangleName(f, p.target) + if sfInfixCall in f.flags: + let pat = n.sons[0].sym.loc.r.data + internalAssert pat != nil + if pat.contains({'#', '(', '@'}): + var typ = skipTypes(n.sons[0].typ, abstractInst) + assert(typ.kind == tyProc) + genPatternCall(p, n, pat, typ, r) + return + if n.len != 1: + gen(p, n.sons[1], r) + if r.typ == etyBaseIndex: + if r.address == nil: + globalError(n.info, "cannot invoke with infix syntax") + r.res = "$1[$2]" % [r.address, r.res] + r.address = nil + r.typ = etyNone + add(r.res, "." | "->") var op: TCompRes if p.target == targetPHP: op.kind = resCallee @@ -1266,7 +1297,7 @@ proc genInfixCall(p: PProc, n: PNode, r: var TCompRes) = genArgs(p, n, r, 2) proc genCall(p: PProc, n: PNode, r: var TCompRes) = - if thisParam(p, n.sons[0].typ) != nil: + if n.sons[0].kind == nkSym and thisParam(p, n.sons[0].typ) != nil: genInfixCall(p, n, r) return if p.target == targetPHP: @@ -1400,6 +1431,12 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope = result = putToSeq("null", indirect) of tySequence, tyString, tyCString, tyPointer, tyProc: result = putToSeq("null", indirect) + of tyStatic: + if t.n != nil: + result = createVar(p, lastSon t, indirect) + else: + internalError("createVar: " & $t.kind) + result = nil else: internalError("createVar: " & $t.kind) result = nil @@ -1409,13 +1446,16 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = a: TCompRes s: Rope if n.kind == nkEmpty: + let mname = mangleName(v, p.target) addf(p.body, "var $1 = $2;$n" | "$$$1 = $2;$n", - [mangleName(v, p.target), createVar(p, v.typ, isIndirect(v))]) + [mname, createVar(p, v.typ, isIndirect(v))]) + if v.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, v.typ) == etyBaseIndex: + addf(p.body, "var $1_Idx = 0;$n", [ mname ]) else: discard mangleName(v, p.target) gen(p, n, a) case mapType(p, v.typ) - of etyObject: + of etyObject, etySeq: if needsNoCopy(p, n): s = a.res else: @@ -1458,7 +1498,7 @@ proc genConstant(p: PProc, c: PSym) = p.body = nil #genLineDir(p, c.ast) genVarInit(p, c, c.ast) - add(p.g.code, p.body) + add(p.g.constants, p.body) p.body = oldBody proc genNew(p: PProc, n: PNode) = @@ -1554,8 +1594,16 @@ proc genRepr(p: PProc, n: PNode, r: var TCompRes) = of tyEnum, tyOrdinal: gen(p, n.sons[1], r) useMagic(p, "cstrToNimstr") + var offset = "" + if t.kind == tyEnum: + let firstFieldOffset = t.n.sons[0].sym.position + if firstFieldOffset < 0: + offset = "+" & $(-firstFieldOffset) + elif firstFieldOffset > 0: + offset = "-" & $firstFieldOffset + r.kind = resExpr - r.res = "cstrToNimstr($1.node.sons[$2].name)" % [genTypeInfo(p, t), r.res] + r.res = "cstrToNimstr($1.node.sons[$2$3].name)" % [genTypeInfo(p, t), r.res, rope(offset)] else: # XXX: internalError(n.info, "genRepr: Not implemented") @@ -1645,8 +1693,11 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = of mChr, mArrToSeq: gen(p, n.sons[1], r) # nothing to do of mOrd: genOrd(p, n, r) of mLengthStr: - unaryExpr(p, n, r, "", "($1 != null ? $1.length-1 : 0)" | - "strlen($1)") + if p.target == targetJS and n.sons[1].typ.skipTypes(abstractInst).kind == tyCString: + unaryExpr(p, n, r, "", "($1 != null ? $1.length : 0)") + else: + unaryExpr(p, n, r, "", "($1 != null ? $1.length-1 : 0)" | + "strlen($1)") of mXLenStr: unaryExpr(p, n, r, "", "$1.length-1" | "strlen($1)") of mLengthSeq, mLengthOpenArray, mLengthArray: unaryExpr(p, n, r, "", "($1 != null ? $1.length : 0)" | @@ -1672,8 +1723,9 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = else: if optOverflowCheck notin p.options: binaryExpr(p, n, r, "", "$1 -= $2") else: binaryExpr(p, n, r, "subInt", "$1 = subInt($1, $2)") - of mSetLengthStr: binaryExpr(p, n, r, "", "$1.length = $2+1; $1[$1.length-1] = 0" | "") - of mSetLengthSeq: binaryExpr(p, n, r, "", "$1.length = $2" | "") + of mSetLengthStr: + binaryExpr(p, n, r, "", "$1.length = $2+1; $1[$1.length-1] = 0" | "$1 = substr($1, 0, $2)") + of mSetLengthSeq: binaryExpr(p, n, r, "", "$1.length = $2" | "$1 = array_slice($1, 0, $2)") of mCard: unaryExpr(p, n, r, "SetCard", "SetCard($1)") of mLtSet: binaryExpr(p, n, r, "SetLt", "SetLt($1, $2)") of mLeSet: binaryExpr(p, n, r, "SetLe", "SetLe($1, $2)") @@ -1683,8 +1735,31 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = of mMinusSet: binaryExpr(p, n, r, "SetMinus", "SetMinus($1, $2)") of mIncl: binaryExpr(p, n, r, "", "$1[$2] = true") of mExcl: binaryExpr(p, n, r, "", "delete $1[$2]" | "unset $1[$2]") - of mInSet: binaryExpr(p, n, r, "", "($1[$2] != undefined)" | "isset($1[$2])") + of mInSet: + if p.target == targetJS: + binaryExpr(p, n, r, "", "($1[$2] != undefined)") + else: + let s = n.sons[1] + if s.kind == nkCurly: + var a, b, x: TCompRes + gen(p, n.sons[2], x) + r.res = rope("(") + r.kind = resExpr + for i in countup(0, sonsLen(s) - 1): + if i > 0: add(r.res, " || ") + var it = s.sons[i] + if it.kind == nkRange: + gen(p, it.sons[0], a) + gen(p, it.sons[1], b) + addf(r.res, "($1 >= $2 && $1 <= $3)", [x.res, a.res, b.res,]) + else: + gen(p, it, a) + addf(r.res, "($1 == $2)", [x.res, a.res]) + add(r.res, ")") + else: + binaryExpr(p, n, r, "", "isset($1[$2])") of mNewSeq: genNewSeq(p, n) + of mNewSeqOfCap: unaryExpr(p, n, r, "", "[]" | "array()") of mOf: genOf(p, n, r) of mReset: genReset(p, n) of mEcho: genEcho(p, n, r) @@ -1877,9 +1952,13 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = let header = generateHeader(p, prc.typ) if prc.typ.sons[0] != nil and sfPure notin prc.flags: resultSym = prc.ast.sons[resultPos].sym + let mname = mangleName(resultSym, p.target) resultAsgn = ("var $# = $#;$n" | "$$$# = $#;$n") % [ - mangleName(resultSym, p.target), + mname, createVar(p, resultSym.typ, isIndirect(resultSym))] + if resultSym.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, resultSym.typ) == etyBaseIndex: + resultAsgn.add "var $#_Idx = 0;$n" % [ + mname ]; gen(p, prc.ast.sons[resultPos], a) if mapType(p, resultSym.typ) == etyBaseIndex: returnStmt = "return [$#, $#];$n" % [a.address, a.res] @@ -1983,7 +2062,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = if (n.sons[0].kind == nkSym) and (n.sons[0].sym.magic != mNone): genMagic(p, n, r) elif n.sons[0].kind == nkSym and sfInfixCall in n.sons[0].sym.flags and - n.len >= 2: + n.len >= 1: genInfixCall(p, n, r) else: genCall(p, n, r) @@ -2131,7 +2210,7 @@ proc wholeCode*(m: BModule): Rope = var p = newProc(globals, m, nil, m.module.options) attachProc(p, prc) - result = globals.typeInfo & globals.code + result = globals.typeInfo & globals.constants & globals.code proc getClassName(t: PType): Rope = var s = t.sym diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim index 8d109e48a..cf1679ee4 100644 --- a/compiler/jstypes.nim +++ b/compiler/jstypes.nim @@ -165,4 +165,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope = of tyEnum: genEnumInfo(p, t, result) of tyObject: genObjectInfo(p, t, result) of tyTuple: genTupleInfo(p, t, result) + of tyStatic: + if t.n != nil: result = genTypeInfo(p, lastSon t) + else: internalError("genTypeInfo(" & $t.kind & ')') else: internalError("genTypeInfo(" & $t.kind & ')') diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 53fca4863..753602c80 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -716,11 +716,16 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; #localError(n.info, "internal error: closure to closure created") # now we know better, so patch it: n.sons[0] = x.sons[0] + n.sons[1] = x.sons[1] of nkLambdaKinds, nkIteratorDef: if n.typ != nil and n[namePos].kind == nkSym: let m = newSymNode(n[namePos].sym) m.typ = n.typ result = liftCapturedVars(m, owner, d, c) + of nkHiddenStdConv: + if n.len == 2: + n.sons[1] = liftCapturedVars(n[1], owner, d, c) + if n[1].kind == nkClosure: result = n[1] else: if owner.isIterator: if n.kind == nkYieldStmt: diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 0032b97df..0eee5004e 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -138,6 +138,8 @@ proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} = proc isKeyword*(kind: TTokType): bool = result = (kind >= tokKeywordLow) and (kind <= tokKeywordHigh) +template ones(n): untyped = ((1 shl n)-1) # for utf-8 conversion + proc isNimIdentifier*(s: string): bool = if s[0] in SymStartChars: var i = 1 @@ -589,12 +591,29 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) = of '\\': add(tok.literal, '\\') inc(L.bufpos) - of 'x', 'X': + of 'x', 'X', 'u', 'U': + var tp = L.buf[L.bufpos] inc(L.bufpos) var xi = 0 handleHexChar(L, xi) handleHexChar(L, xi) - add(tok.literal, chr(xi)) + if tp in {'u', 'U'}: + handleHexChar(L, xi) + handleHexChar(L, xi) + # inlined toUTF-8 to avoid unicode and strutils dependencies. + if xi <=% 127: + add(tok.literal, xi.char ) + elif xi <=% 0x07FF: + add(tok.literal, ((xi shr 6) or 0b110_00000).char ) + add(tok.literal, ((xi and ones(6)) or 0b10_0000_00).char ) + elif xi <=% 0xFFFF: + add(tok.literal, (xi shr 12 or 0b1110_0000).char ) + add(tok.literal, (xi shr 6 and ones(6) or 0b10_0000_00).char ) + add(tok.literal, (xi and ones(6) or 0b10_0000_00).char ) + else: # value is 0xFFFF + add(tok.literal, "\xef\xbf\xbf" ) + else: + add(tok.literal, chr(xi)) of '0'..'9': if matchTwoChars(L, '0', {'0'..'9'}): lexMessage(L, warnOctalEscape) @@ -716,7 +735,8 @@ proc getSymbol(L: var TLexer, tok: var TToken) = if c == '\226' and buf[pos+1] == '\128' and buf[pos+2] == '\147': # It's a 'magic separator' en-dash Unicode - if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars: + if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars or + isMagicIdentSeparatorRune(buf, pos+magicIdentSeparatorRuneByteWidth) or pos == L.bufpos: lexMessage(L, errInvalidToken, "–") break inc(pos, magicIdentSeparatorRuneByteWidth) @@ -728,7 +748,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) = h = h !& ord(c) inc(pos) of '_': - if buf[pos+1] notin SymChars: + if buf[pos+1] notin SymChars or isMagicIdentSeparatorRune(buf, pos+1): lexMessage(L, errInvalidToken, "_") break inc(pos) @@ -811,10 +831,6 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; break dec nesting inc pos - of '\t': - lexMessagePos(L, errTabulatorsAreNotAllowed, pos) - inc(pos) - if isDoc: tok.literal.add '\t' of CR, LF: pos = handleCRLF(L, pos) buf = L.buf @@ -877,10 +893,10 @@ proc scanComment(L: var TLexer, tok: var TToken) = inc(indent) when defined(nimfix): - template doContinue(): expr = + template doContinue(): untyped = buf[pos] == '#' and (col == indent or lastBackslash > 0) else: - template doContinue(): expr = + template doContinue(): untyped = buf[pos] == '#' and buf[pos+1] == '#' if doContinue(): tok.literal.add "\n" @@ -926,9 +942,9 @@ proc skip(L: var TLexer, tok: var TToken) = break tok.strongSpaceA = 0 when defined(nimfix): - template doBreak(): expr = buf[pos] > ' ' + template doBreak(): untyped = buf[pos] > ' ' else: - template doBreak(): expr = + template doBreak(): untyped = buf[pos] > ' ' and (buf[pos] != '#' or buf[pos+1] == '#') if doBreak(): tok.indent = indent @@ -1041,7 +1057,8 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = inc(L.bufpos) of '_': inc(L.bufpos) - if L.buf[L.bufpos] notin SymChars: + if L.buf[L.bufpos] notin SymChars+{'_'} and not + isMagicIdentSeparatorRune(L.buf, L.bufpos): tok.tokType = tkSymbol tok.ident = getIdent("_") else: @@ -1062,6 +1079,9 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = tok.tokType = tkCharLit of '0'..'9': getNumber(L, tok) + let c = L.buf[L.bufpos] + if c in SymChars+{'_'}: + lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')') else: if c in OpChars: getOperator(L, tok) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index a337bf0f3..df19a6afb 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -99,8 +99,11 @@ proc debugScopes*(c: PContext; limit=0) {.deprecated.} = proc searchInScopes*(c: PContext, s: PIdent, filter: TSymKinds): PSym = for scope in walkScopes(c.currentScope): - result = strTableGet(scope.symbols, s) - if result != nil and result.kind in filter: return + var ti: TIdentIter + var candidate = initIdentIter(ti, scope.symbols, s) + while candidate != nil: + if candidate.kind in filter: return candidate + candidate = nextIdentIter(ti, scope.symbols) result = nil proc errorSym*(c: PContext, n: PNode): PSym = @@ -153,7 +156,8 @@ proc ensureNoMissingOrUnusedSymbols(scope: PScope) = if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam}: # XXX: implicit type params are currently skTypes # maybe they can be made skGenericParam as well. - if s.typ != nil and tfImplicitTypeParam notin s.typ.flags: + if s.typ != nil and tfImplicitTypeParam notin s.typ.flags and + s.typ.kind != tyGenericParam: message(s.info, hintXDeclaredButNotUsed, getSymRepr(s)) s = nextIter(it, scope.symbols) @@ -161,6 +165,10 @@ proc wrongRedefinition*(info: TLineInfo, s: string) = if gCmd != cmdInteractive: localError(info, errAttemptToRedefine, s) +proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) = + if not c.currentScope.addUniqueSym(sym): + wrongRedefinition(info, sym.name.s) + proc addDecl*(c: PContext, sym: PSym) = if not c.currentScope.addUniqueSym(sym): wrongRedefinition(sym.info, sym.name.s) @@ -212,14 +220,27 @@ when defined(nimfix): of 'a'..'z': result = getIdent(toLower(x.s[0]) & x.s.substr(1)) else: result = x - template fixSpelling(n: PNode; ident: PIdent; op: expr) = + template fixSpelling(n: PNode; ident: PIdent; op: untyped) = let alt = ident.altSpelling result = op(c, alt).skipAlias(n) if result != nil: prettybase.replaceDeprecated(n.info, ident, alt) return result else: - template fixSpelling(n: PNode; ident: PIdent; op: expr) = discard + template fixSpelling(n: PNode; ident: PIdent; op: untyped) = discard + +proc errorUseQualifier*(c: PContext; info: TLineInfo; s: PSym) = + var err = "Error: ambiguous identifier: '" & s.name.s & "'" + var ti: TIdentIter + var candidate = initIdentIter(ti, c.importTable.symbols, s.name) + var i = 0 + while candidate != nil: + if i == 0: err.add " --use " + else: err.add " or " + err.add candidate.owner.name.s & "." & candidate.name.s + candidate = nextIdentIter(ti, c.importTable.symbols) + inc i + localError(info, errGenerated, err) proc lookUp*(c: PContext, n: PNode): PSym = # Looks up a symbol. Generates an error in case of nil. @@ -243,32 +264,36 @@ proc lookUp*(c: PContext, n: PNode): PSym = internalError(n.info, "lookUp") return if contains(c.ambiguousSymbols, result.id): - localError(n.info, errUseQualifier, result.name.s) + errorUseQualifier(c, n.info, result) if result.kind == skStub: loadStub(result) type TLookupFlag* = enum - checkAmbiguity, checkUndeclared + checkAmbiguity, checkUndeclared, checkModule -proc qualifiedLookUp*(c: PContext, n: PNode, flags = {checkUndeclared}): PSym = +proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym = + const allExceptModule = {low(TSymKind)..high(TSymKind)}-{skModule,skPackage} case n.kind of nkIdent, nkAccQuoted: var ident = considerQuotedIdent(n) - result = searchInScopes(c, ident).skipAlias(n) + if checkModule in flags: + result = searchInScopes(c, ident).skipAlias(n) + else: + result = searchInScopes(c, ident, allExceptModule).skipAlias(n) if result == nil and checkUndeclared in flags: fixSpelling(n, ident, searchInScopes) localError(n.info, errUndeclaredIdentifier, ident.s) result = errorSym(c, n) elif checkAmbiguity in flags and result != nil and contains(c.ambiguousSymbols, result.id): - localError(n.info, errUseQualifier, ident.s) + errorUseQualifier(c, n.info, result) of nkSym: result = n.sym if checkAmbiguity in flags and contains(c.ambiguousSymbols, result.id): - localError(n.info, errUseQualifier, n.sym.name.s) + errorUseQualifier(c, n.info, n.sym) of nkDotExpr: result = nil - var m = qualifiedLookUp(c, n.sons[0], flags*{checkUndeclared}) + var m = qualifiedLookUp(c, n.sons[0], (flags*{checkUndeclared})+{checkModule}) if m != nil and m.kind == skModule: var ident: PIdent = nil if n.sons[1].kind == nkIdent: @@ -313,7 +338,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = o.mode = oimDone of nkDotExpr: o.mode = oimOtherModule - o.m = qualifiedLookUp(c, n.sons[0]) + o.m = qualifiedLookUp(c, n.sons[0], {checkUndeclared, checkModule}) if o.m != nil and o.m.kind == skModule: var ident: PIdent = nil if n.sons[1].kind == nkIdent: diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 7a5c7f44b..36eb24653 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -471,6 +471,7 @@ proc setupArgsForParallelism(n: PNode; objType: PType; scratchObj: PSym; let slice = newNodeI(nkCall, n.info, 4) slice.typ = n.typ slice.sons[0] = newSymNode(createMagic("slice", mSlice)) + slice.sons[0].typ = getSysType(tyInt) # fake type var fieldB = newSym(skField, tmpName, objType.owner, n.info) fieldB.typ = getSysType(tyInt) objType.addField(fieldB) @@ -577,6 +578,8 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType; var fn = n.sons[0] # templates and macros are in fact valid here due to the nature of # the transformation: + if fn.kind == nkClosure: + localError(n.info, "closure in spawn environment is not allowed") if not (fn.kind == nkSym and fn.sym.kind in {skProc, skTemplate, skMacro, skMethod, skConverter}): # for indirect calls we pass the function pointer in the scratchObj diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index 40e0728ae..13b365d04 100644 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -41,11 +41,15 @@ proc getSysSym*(name: string): PSym = proc getSysMagic*(name: string, m: TMagic): PSym = var ti: TIdentIter let id = getIdent(name) - result = initIdentIter(ti, systemModule.tab, id) - while result != nil: - if result.kind == skStub: loadStub(result) - if result.magic == m: return result - result = nextIdentIter(ti, systemModule.tab) + var r = initIdentIter(ti, systemModule.tab, id) + while r != nil: + if r.kind == skStub: loadStub(r) + if r.magic == m: + # prefer the tyInt variant: + if r.typ.sons[0].kind == tyInt: return r + result = r + r = nextIdentIter(ti, systemModule.tab) + if result != nil: return result rawMessage(errSystemNeeds, name) result = newSym(skError, id, systemModule, systemModule.info) result.typ = newType(tyError, systemModule) diff --git a/compiler/main.nim b/compiler/main.nim index ece21a817..0db66b53e 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -41,14 +41,16 @@ proc commandGenDepend = proc commandCheck = msgs.gErrorMax = high(int) # do not stop after first error + defineSymbol("nimcheck") semanticPasses() # use an empty backend for semantic checking only rodPass() compileProject() -proc commandDoc2 = +proc commandDoc2(json: bool) = msgs.gErrorMax = high(int) # do not stop after first error semanticPasses() - registerPass(docgen2Pass) + if json: registerPass(docgen2JsonPass) + else: registerPass(docgen2Pass) #registerPass(cleanupPass()) compileProject() finishDoc2Pass(gProjectName) @@ -241,7 +243,7 @@ proc mainCommand* = clearPasses() gLastCmdTime = epochTime() appendStr(searchPaths, options.libpath) - if gProjectFull.len != 0: + when false: # gProjectFull.len != 0: # current path is always looked first for modules prependStr(searchPaths, gProjectPath) setId(100) @@ -280,7 +282,7 @@ proc mainCommand* = gCmd = cmdDoc loadConfigs(DocConfig) defineSymbol("nimdoc") - commandDoc2() + commandDoc2(false) of "rst2html": gCmd = cmdRst2html loadConfigs(DocConfig) @@ -295,7 +297,13 @@ proc mainCommand* = loadConfigs(DocConfig) wantMainModule() defineSymbol("nimdoc") - commandJSON() + commandJson() + of "jsondoc2": + gCmd = cmdDoc + loadConfigs(DocConfig) + wantMainModule() + defineSymbol("nimdoc") + commandDoc2(true) of "buildindex": gCmd = cmdDoc loadConfigs(DocConfig) diff --git a/compiler/modules.nim b/compiler/modules.nim index ef727e200..9120bd1b6 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -29,12 +29,16 @@ var gMemCacheData*: seq[TModuleInMemory] = @[] ## XXX: we should implement recycling of file IDs ## if the user keeps renaming modules, the file IDs will keep growing + gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why. + packageSyms: TStrTable + +initStrTable(packageSyms) proc getModule*(fileIdx: int32): PSym = if fileIdx >= 0 and fileIdx < gCompiledModules.len: result = gCompiledModules[fileIdx] -template hash(x: PSym): expr = +template hash(x: PSym): untyped = gMemCacheData[x.position].hash proc hashChanged(fileIdx: int32): bool = @@ -45,7 +49,7 @@ proc hashChanged(fileIdx: int32): bool = else: hashNotChanged # echo "TESTING Hash: ", fileIdx.toFilename, " ", result - case gMemCacheData[fileIdx].hashStatus: + case gMemCacheData[fileIdx].hashStatus of hashHasChanged: result = true of hashNotChanged: @@ -90,6 +94,7 @@ proc resetAllModules* = if gCompiledModules[i] != nil: resetModule(i.int32) resetPackageCache() + initStrTable(packageSyms) # for m in cgenModules(): echo "CGEN MODULE FOUND" proc resetAllModulesHard* = @@ -97,6 +102,7 @@ proc resetAllModulesHard* = gCompiledModules.setLen 0 gMemCacheData.setLen 0 magicsys.resetSysTypes() + initStrTable(packageSyms) # XXX #gOwners = @[] @@ -105,12 +111,16 @@ proc checkDepMem(fileIdx: int32): TNeedRecompile = resetModule(fileIdx) return Yes - if gMemCacheData[fileIdx].needsRecompile != Maybe: - return gMemCacheData[fileIdx].needsRecompile + if gFuzzyGraphChecking: + if gMemCacheData[fileIdx].needsRecompile != Maybe: + return gMemCacheData[fileIdx].needsRecompile + else: + # cycle detection: We claim that a cycle does no harm. + if gMemCacheData[fileIdx].needsRecompile == Probing: + return No - if optForceFullMake in gGlobalOptions or - hashChanged(fileIdx): - markDirty + if optForceFullMake in gGlobalOptions or hashChanged(fileIdx): + markDirty() if gMemCacheData[fileIdx].deps != nil: gMemCacheData[fileIdx].needsRecompile = Probing @@ -118,7 +128,7 @@ proc checkDepMem(fileIdx: int32): TNeedRecompile = let d = checkDepMem(dep) if d in {Yes, Recompiled}: # echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d - markDirty + markDirty() gMemCacheData[fileIdx].needsRecompile = No return No @@ -135,8 +145,16 @@ proc newModule(fileIdx: int32): PSym = rawMessage(errInvalidModuleName, result.name.s) result.info = newLineInfo(fileIdx, 1, 1) - result.owner = newSym(skPackage, getIdent(getPackageName(filename)), nil, - result.info) + let pack = getIdent(getPackageName(filename)) + var packSym = packageSyms.strTableGet(pack) + if packSym == nil: + let pck = getPackageName(filename) + let pck2 = if pck.len > 0: pck else: "unknown" + packSym = newSym(skPackage, getIdent(pck2), nil, result.info) + initStrTable(packSym.tab) + packageSyms.strTableAdd(packSym) + + result.owner = packSym result.position = fileIdx growCache gMemCacheData, fileIdx @@ -146,6 +164,11 @@ proc newModule(fileIdx: int32): PSym = incl(result.flags, sfUsed) initStrTable(result.tab) strTableAdd(result.tab, result) # a module knows itself + let existing = strTableGet(packSym.tab, result.name) + if existing != nil and existing.info.fileIndex != result.info.fileIndex: + localError(result.info, "module names need to be unique per Nimble package; module clashes with " & existing.info.fileIndex.toFullPath) + # strTableIncl() for error corrections: + discard strTableIncl(packSym.tab, result) proc compileModule*(fileIdx: int32, flags: TSymFlags): PSym = result = getModule(fileIdx) @@ -156,6 +179,9 @@ proc compileModule*(fileIdx: int32, flags: TSymFlags): PSym = #var rd = handleSymbolFile(result) var rd: PRodReader result.flags = result.flags + flags + if sfMainModule in result.flags: + gMainPackageId = result.owner.id + if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}: rd = handleSymbolFile(result) if result.id < 0: @@ -181,8 +207,11 @@ proc importModule*(s: PSym, fileIdx: int32): PSym {.procvar.} = # this is called by the semantic checking phase result = compileModule(fileIdx, {}) if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx) - if sfSystemModule in result.flags: - localError(result.info, errAttemptToRedefine, result.name.s) + #if sfSystemModule in result.flags: + # localError(result.info, errAttemptToRedefine, result.name.s) + # restore the notes for outer module: + gNotes = if s.owner.id == gMainPackageId: gMainPackageNotes + else: ForeignPackageNotes proc includeModule*(s: PSym, fileIdx: int32): PNode {.procvar.} = result = syntaxes.parseFile(fileIdx) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index a556ad0c5..4a9980066 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -471,6 +471,8 @@ type shortName*: string # short name of the module quotedName*: Rope # cached quoted short name for codegen # purposes + quotedFullName*: Rope # cached quoted full name for codegen + # purposes lines*: seq[Rope] # the source code of the module # used for better error messages and @@ -505,7 +507,7 @@ const warnProveField, warnProveIndex, warnGcUnsafe, hintSuccessX, hintPath, hintConf, - hintProcessing, + hintProcessing, hintPattern, hintDependency, hintExecuting, hintLinking, hintCodeBegin, hintCodeEnd, @@ -514,9 +516,8 @@ const {low(TNoteKind)..high(TNoteKind)} - {warnShadowIdent, warnUninit, warnProveField, warnProveIndex, warnGcUnsafe, - hintPath, hintConf, + hintPath, hintDependency, - hintExecuting, hintCodeBegin, hintCodeEnd, hintSource, hintStackTrace, hintGCStats}, @@ -527,6 +528,8 @@ const InvalidFileIDX* = int32(-1) var + ForeignPackageNotes*: TNoteKinds = {hintProcessing, warnUnknownMagic, + hintQuitCalled, hintExecuting} filenameToIndexTbl = initTable[string, int32]() fileInfos*: seq[TFileInfo] = @[] systemFileIdx*: int32 @@ -561,6 +564,7 @@ proc newFileInfo(fullPath, projPath: string): TFileInfo = let fileName = projPath.extractFilename result.shortName = fileName.changeFileExt("") result.quotedName = fileName.makeCString + result.quotedFullName = fullPath.makeCString if optEmbedOrigSrc in gGlobalOptions or true: result.lines = @[] @@ -616,6 +620,7 @@ var gHintCounter*: int = 0 gWarnCounter*: int = 0 gErrorMax*: int = 1 # stop after gErrorMax errors + gMainPackageNotes*: TNoteKinds = NotesVerbosity[1] proc unknownLineInfo*(): TLineInfo = result.line = int16(-1) @@ -755,7 +760,7 @@ proc msgWriteln*(s: string, flags: MsgFlags = {}) = flushFile(stderr) macro callIgnoringStyle(theProc: typed, first: typed, - args: varargs[expr]): stmt = + args: varargs[typed]): untyped = let typForegroundColor = bindSym"ForegroundColor".getType let typBackgroundColor = bindSym"BackgroundColor".getType let typStyle = bindSym"Style".getType @@ -772,7 +777,7 @@ macro callIgnoringStyle(theProc: typed, first: typed, typ != typTerminalCmd: result.add(arg) -macro callStyledWriteLineStderr(args: varargs[expr]): stmt = +macro callStyledWriteLineStderr(args: varargs[typed]): untyped = result = newCall(bindSym"styledWriteLine") result.add(bindSym"stderr") for arg in children(args[0][1]): @@ -784,7 +789,7 @@ template callWritelnHook(args: varargs[string, `$`]) = s.add arg writelnHook s -template styledMsgWriteln*(args: varargs[expr]) = +template styledMsgWriteln*(args: varargs[typed]) = if not isNil(writelnHook): callIgnoringStyle(callWritelnHook, nil, args) elif optStdout in gGlobalOptions: @@ -995,11 +1000,11 @@ proc internalError*(errMsg: string) = writeContext(unknownLineInfo()) rawMessage(errInternal, errMsg) -template assertNotNil*(e: expr): expr = +template assertNotNil*(e): untyped = if e == nil: internalError($instantiationInfo()) e -template internalAssert*(e: bool): stmt = +template internalAssert*(e: bool) = if not e: internalError($instantiationInfo()) proc addSourceLine*(fileIdx: int32, line: string) = @@ -1022,7 +1027,10 @@ proc sourceLine*(i: TLineInfo): Rope = proc quotedFilename*(i: TLineInfo): Rope = internalAssert i.fileIndex >= 0 - result = fileInfos[i.fileIndex].quotedName + if optExcessiveStackTrace in gGlobalOptions: + result = fileInfos[i.fileIndex].quotedFullName + else: + result = fileInfos[i.fileIndex].quotedName ropes.errorHandler = proc (err: RopesError, msg: string, useWarning: bool) = case err diff --git a/compiler/nim.cfg b/compiler/nim.cfg index 4f9962ea8..0ff128ba3 100644 --- a/compiler/nim.cfg +++ b/compiler/nim.cfg @@ -7,7 +7,7 @@ path:"$projectPath/.." path:"$lib/packages/docutils" define:booting -import:testability +#import:"$projectpath/testability" @if windows: cincludes: "$lib/wrappers/libffi/common" diff --git a/compiler/nim.nim b/compiler/nim.nim index b242052ec..a58afd593 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -56,12 +56,12 @@ proc handleCmdLine() = loadConfigs(DefaultConfig) # load all config files let scriptFile = gProjectFull.changeFileExt("nims") if fileExists(scriptFile): - runNimScript(scriptFile) + runNimScript(scriptFile, freshDefines=false) # 'nim foo.nims' means to just run the NimScript file and do nothing more: if scriptFile == gProjectFull: return elif fileExists(gProjectPath / "config.nims"): # directory wide NimScript file - runNimScript(gProjectPath / "config.nims") + runNimScript(gProjectPath / "config.nims", freshDefines=false) # now process command line arguments again, because some options in the # command line can overwite the config file's settings extccomp.initVars() diff --git a/compiler/nimfix/nimfix.nim.cfg b/compiler/nimfix/nimfix.nim.cfg index 73219d6f8..0d9dbfa4b 100644 --- a/compiler/nimfix/nimfix.nim.cfg +++ b/compiler/nimfix/nimfix.nim.cfg @@ -5,7 +5,7 @@ hint[XDeclaredButNotUsed]:off path:"$projectPath/.." path:"$lib/packages/docutils" -path:"../../compiler" +path:"$nim" define:useStdoutAsStdmsg symbol:nimfix diff --git a/compiler/nimfix/pretty.nim b/compiler/nimfix/pretty.nim index 1123afb9e..8ba922927 100644 --- a/compiler/nimfix/pretty.nim +++ b/compiler/nimfix/pretty.nim @@ -66,7 +66,7 @@ proc beautifyName(s: string, k: TSymKind): string = ]: result.add s[i] else: - result.add toUpper(s[i]) + result.add toUpperAscii(s[i]) of skConst, skEnumField: # for 'const' we keep how it's spelt; either upper case or lower case: result.add s[0] @@ -74,7 +74,7 @@ proc beautifyName(s: string, k: TSymKind): string = # as a special rule, don't transform 'L' to 'l' if s.len == 1 and s[0] == 'L': result.add 'L' elif '_' in s: result.add(s[i]) - else: result.add toLower(s[0]) + else: result.add toLowerAscii(s[0]) inc i while i < s.len: if s[i] == '_': @@ -85,9 +85,9 @@ proc beautifyName(s: string, k: TSymKind): string = result.add s[i] else: inc i - result.add toUpper(s[i]) + result.add toUpperAscii(s[i]) elif allUpper: - result.add toLower(s[i]) + result.add toLowerAscii(s[i]) else: result.add s[i] inc i diff --git a/compiler/nimsets.nim b/compiler/nimsets.nim index f15ad6368..94507adf0 100644 --- a/compiler/nimsets.nim +++ b/compiler/nimsets.nim @@ -121,7 +121,7 @@ proc toTreeSet(s: TBitSet, settype: PType, info: TLineInfo): PNode = e = b inc(e) -template nodeSetOp(a, b: PNode, op: expr) {.dirty.} = +template nodeSetOp(a, b: PNode, op: untyped) {.dirty.} = var x, y: TBitSet toBitSet(a, x) toBitSet(b, y) diff --git a/compiler/options.nim b/compiler/options.nim index 2716a98d3..a5154cb48 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -67,6 +67,8 @@ type # please make sure we have under 32 options optIdeDebug # idetools: debug mode optIdeTerse # idetools: use terse descriptions optNoCppExceptions # use C exception handling even with CPP + optExcessiveStackTrace # fully qualified module filenames + TGlobalOptions* = set[TGlobalOption] TCommands* = enum # Nim's commands # **keep binary compatible** @@ -127,10 +129,10 @@ var proc importantComments*(): bool {.inline.} = gCmd in {cmdDoc, cmdIdeTools} proc usesNativeGC*(): bool {.inline.} = gSelectedGC >= gcRefc -template compilationCachePresent*: expr = +template compilationCachePresent*: untyped = {optCaasEnabled, optSymbolFiles} * gGlobalOptions != {} -template optPreserveOrigSource*: expr = +template optPreserveOrigSource*: untyped = optEmbedOrigSrc in gGlobalOptions const @@ -149,6 +151,7 @@ const var gConfigVars* = newStringTable(modeStyleInsensitive) gDllOverrides = newStringTable(modeCaseInsensitive) + gModuleOverrides* = newStringTable(modeStyleInsensitive) gPrefixDir* = "" # Overrides the default prefix dir in getPrefixDir proc. libpath* = "" gProjectName* = "" # holds a name like 'nim' @@ -209,9 +212,7 @@ proc setDefaultLibpath*() = # Special rule to support other tools (nimble) which import the compiler # modules and make use of them. - let realNimPath = # Make sure we expand the symlink - if symlinkExists(findExe("nim")): expandSymlink(findExe("nim")) - else: findExe("nim") + let realNimPath = findExe("nim") # Find out if $nim/../../lib/system.nim exists. let parentNimLibPath = realNimPath.parentDir().parentDir() / "lib" if not fileExists(libpath / "system.nim") and @@ -219,7 +220,7 @@ proc setDefaultLibpath*() = libpath = parentNimLibPath proc canonicalizePath*(path: string): string = - when not FileSystemCaseSensitive: result = path.expandFilename.toLower + when not FileSystemCaseSensitive: result = path.expandFilename.toLowerAscii else: result = path.expandFilename proc shortenDir*(dir: string): string = @@ -238,57 +239,26 @@ proc removeTrailingDirSep*(path: string): string = else: result = path +include packagehandling + proc getNimcacheDir*: string = result = if nimcacheDir.len > 0: nimcacheDir else: gProjectPath.shortenDir / genSubDir -template newPackageCache(): expr = - newStringTable(when FileSystemCaseSensitive: - modeCaseInsensitive - else: - modeCaseSensitive) - -var packageCache = newPackageCache() - -proc resetPackageCache*() = packageCache = newPackageCache() - -iterator myParentDirs(p: string): string = - # XXX os's parentDirs is stupid (multiple yields) and triggers an old bug... - var current = p - while true: - current = current.parentDir - if current.len == 0: break - yield current - -proc getPackageName*(path: string): string = - var parents = 0 - block packageSearch: - for d in myParentDirs(path): - if packageCache.hasKey(d): - #echo "from cache ", d, " |", packageCache[d], "|", path.splitFile.name - return packageCache[d] - inc parents - for file in walkFiles(d / "*.nimble"): - result = file.splitFile.name - break packageSearch - for file in walkFiles(d / "*.babel"): - result = file.splitFile.name - break packageSearch - # we also store if we didn't find anything: - if result.isNil: result = "" - for d in myParentDirs(path): - #echo "set cache ", d, " |", result, "|", parents - packageCache[d] = result - dec parents - if parents <= 0: break - -proc withPackageName*(path: string): string = - let x = path.getPackageName - if x.len == 0: - result = path - else: - let (p, file, ext) = path.splitFile - result = (p / (x & '_' & file)) & ext + +proc pathSubs*(p, config: string): string = + let home = removeTrailingDirSep(os.getHomeDir()) + result = unixToNativePath(p % [ + "nim", getPrefixDir(), + "lib", libpath, + "home", home, + "config", config, + "projectname", options.gProjectName, + "projectpath", options.gProjectPath, + "projectdir", options.gProjectPath, + "nimcache", getNimcacheDir()]) + if '~' in result: + result = result.replace("~", home) proc toGeneratedFile*(path, ext: string): string = ## converts "/home/a/mymodule.nim", "rod" to "/home/a/nimcache/mymodule.rod" @@ -357,17 +327,25 @@ proc rawFindFile2(f: string): string = it = PStrEntry(it.next) result = "" +template patchModule() {.dirty.} = + if result.len > 0 and gModuleOverrides.len > 0: + let key = getPackageName(result) & "_" & splitFile(result).name + if gModuleOverrides.hasKey(key): + let ov = gModuleOverrides[key] + if ov.len > 0: result = ov + proc findFile*(f: string): string {.procvar.} = if f.isAbsolute: result = if f.existsFile: f else: "" else: result = f.rawFindFile if result.len == 0: - result = f.toLower.rawFindFile + result = f.toLowerAscii.rawFindFile if result.len == 0: result = f.rawFindFile2 if result.len == 0: - result = f.toLower.rawFindFile2 + result = f.toLowerAscii.rawFindFile2 + patchModule() proc findModule*(modulename, currentModule: string): string = # returns path to module @@ -386,6 +364,7 @@ proc findModule*(modulename, currentModule: string): string = result = currentPath / m if not existsFile(result): result = findFile(m) + patchModule() proc libCandidates*(s: string, dest: var seq[string]) = var le = strutils.find(s, '(') @@ -426,10 +405,10 @@ proc binaryStrSearch*(x: openArray[string], y: string): int = return mid result = - 1 -template nimdbg*: expr = c.module.fileIdx == gProjectMainIdx -template cnimdbg*: expr = p.module.module.fileIdx == gProjectMainIdx -template pnimdbg*: expr = p.lex.fileIdx == gProjectMainIdx -template lnimdbg*: expr = L.fileIdx == gProjectMainIdx +template nimdbg*: untyped = c.module.fileIdx == gProjectMainIdx +template cnimdbg*: untyped = p.module.module.fileIdx == gProjectMainIdx +template pnimdbg*: untyped = p.lex.fileIdx == gProjectMainIdx +template lnimdbg*: untyped = L.fileIdx == gProjectMainIdx proc parseIdeCmd*(s: string): IdeCmd = case s: diff --git a/compiler/packagehandling.nim b/compiler/packagehandling.nim new file mode 100644 index 000000000..6a98ffe60 --- /dev/null +++ b/compiler/packagehandling.nim @@ -0,0 +1,56 @@ +# +# +# The Nim Compiler +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +iterator myParentDirs(p: string): string = + # XXX os's parentDirs is stupid (multiple yields) and triggers an old bug... + var current = p + while true: + current = current.parentDir + if current.len == 0: break + yield current + +template newPackageCache(): untyped = + newStringTable(when FileSystemCaseSensitive: + modeCaseInsensitive + else: + modeCaseSensitive) + +var packageCache = newPackageCache() + +proc resetPackageCache*() = packageCache = newPackageCache() + +proc getPackageName*(path: string): string = + var parents = 0 + block packageSearch: + for d in myParentDirs(path): + if packageCache.hasKey(d): + #echo "from cache ", d, " |", packageCache[d], "|", path.splitFile.name + return packageCache[d] + inc parents + for file in walkFiles(d / "*.nimble"): + result = file.splitFile.name + break packageSearch + for file in walkFiles(d / "*.babel"): + result = file.splitFile.name + break packageSearch + # we also store if we didn't find anything: + if result.isNil: result = "" + for d in myParentDirs(path): + #echo "set cache ", d, " |", result, "|", parents + packageCache[d] = result + dec parents + if parents <= 0: break + +proc withPackageName*(path: string): string = + let x = path.getPackageName + if x.len == 0: + result = path + else: + let (p, file, ext) = path.splitFile + result = (p / (x & '_' & file)) & ext diff --git a/compiler/parser.nim b/compiler/parser.nim index f22177ac1..19ef0960a 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -98,7 +98,7 @@ proc parMessage(p: TParser, msg: TMsgKind, tok: TToken) = ## Produce and emit a parser message to output about the token `tok` parMessage(p, msg, prettyTok(tok)) -template withInd(p: expr, body: stmt) {.immediate.} = +template withInd(p, body: untyped) = let oldInd = p.currInd p.currInd = p.tok.indent body @@ -202,7 +202,7 @@ proc isRightAssociative(tok: TToken): bool {.inline.} = proc getPrecedence(tok: TToken, strongSpaces: bool): int = ## Calculates the precedence of the given token. - template considerStrongSpaces(x): expr = + template considerStrongSpaces(x): untyped = x + (if strongSpaces: 100 - tok.strongSpaceA.int*10 else: 0) case tok.tokType @@ -214,7 +214,7 @@ proc getPrecedence(tok: TToken, strongSpaces: bool): int = if L > 1 and tok.ident.s[L-1] == '>' and tok.ident.s[L-2] in {'-', '~', '='}: return considerStrongSpaces(1) - template considerAsgn(value: expr) = + template considerAsgn(value: untyped) = result = if tok.ident.s[L-1] == '=': 1 else: value case relevantChar @@ -326,6 +326,7 @@ proc parseSymbol(p: var TParser, allowNil = false): PNode = getTok(p) else: parMessage(p, errIdentifierExpected, p.tok) + break eat(p, tkAccent) else: if allowNil and p.tok.tokType == tkNil: @@ -685,11 +686,19 @@ proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode = #| | '{' optInd indexExprList optPar '}' #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax result = r + + template somePar() = + if p.tok.strongSpaceA > 0: + if p.strongSpaces: + break + else: + parMessage(p, warnDeprecated, + "a [b] will be parsed as command syntax; spacing") while p.tok.indent < 0 or (p.tok.tokType == tkDot and p.tok.indent >= baseIndent): case p.tok.tokType of tkParLe: - if p.strongSpaces and p.tok.strongSpaceA > 0: break + somePar() result = namedParams(p, result, nkCall, tkParRi) if result.len > 1 and result.sons[1].kind == nkExprColonExpr: result.kind = nkObjConstr @@ -704,10 +713,10 @@ proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode = result = dotExpr(p, result) result = parseGStrLit(p, result) of tkBracketLe: - if p.strongSpaces and p.tok.strongSpaceA > 0: break + somePar() result = namedParams(p, result, nkBracketExpr, tkBracketRi) of tkCurlyLe: - if p.strongSpaces and p.tok.strongSpaceA > 0: break + somePar() result = namedParams(p, result, nkCurlyExpr, tkCurlyRi) of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast, tkAddr, tkType: if p.inPragma == 0: @@ -963,8 +972,9 @@ proc parseDoBlock(p: var TParser): PNode = proc parseDoBlocks(p: var TParser, call: PNode) = #| doBlocks = doBlock ^* IND{=} if p.tok.tokType == tkDo: - addSon(call, parseDoBlock(p)) - while sameInd(p) and p.tok.tokType == tkDo: + #withInd(p): + # addSon(call, parseDoBlock(p)) + while sameOrNoInd(p) and p.tok.tokType == tkDo: addSon(call, parseDoBlock(p)) proc parseProcExpr(p: var TParser, isExpr: bool): PNode = diff --git a/compiler/passaux.nim b/compiler/passaux.nim index 9b9fdca4e..d4361a671 100644 --- a/compiler/passaux.nim +++ b/compiler/passaux.nim @@ -24,7 +24,7 @@ proc verboseProcess(context: PPassContext, n: PNode): PNode = # system.nim deactivates all hints, for verbosity:3 we want the processing # messages nonetheless, so we activate them again unconditionally: incl(msgs.gNotes, hintProcessing) - message(n.info, hintProcessing, $idgen.gBackendId) + message(n.info, hintProcessing, $idgen.gFrontendId) const verbosePass* = makePass(open = verboseOpen, process = verboseProcess) diff --git a/compiler/patterns.nim b/compiler/patterns.nim index 604d3521d..09b8d3305 100644 --- a/compiler/patterns.nim +++ b/compiler/patterns.nim @@ -75,7 +75,7 @@ proc checkTypes(c: PPatternContext, p: PSym, n: PNode): bool = result = matchNodeKinds(p.constraint, n) if not result: return if isNil(n.typ): - result = p.typ.kind in {tyEmpty, tyStmt} + result = p.typ.kind in {tyVoid, tyStmt} else: result = sigmatch.argtypeMatches(c.c, p.typ, n.typ) @@ -129,7 +129,7 @@ proc matchNested(c: PPatternContext, p, n: PNode, rpn: bool): bool = result = bindOrCheck(c, p.sons[2].sym, arglist) proc matches(c: PPatternContext, p, n: PNode): bool = - # hidden conversions (?) + let n = skipHidden(n) if nfNoRewrite in n.flags: result = false elif isPatternParam(c, p): diff --git a/compiler/platform.nim b/compiler/platform.nim index dc414bfeb..36ec10477 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -159,7 +159,7 @@ type # alias conditionals to condsyms (end of module). cpuNone, cpuI386, cpuM68k, cpuAlpha, cpuPowerpc, cpuPowerpc64, cpuPowerpc64el, cpuSparc, cpuVm, cpuIa64, cpuAmd64, cpuMips, cpuMipsel, - cpuArm, cpuArm64, cpuJS, cpuNimrodVM, cpuAVR, cpuMSP430 + cpuArm, cpuArm64, cpuJS, cpuNimrodVM, cpuAVR, cpuMSP430, cpuSparc64 type TEndian* = enum @@ -187,7 +187,8 @@ const (name: "js", intSize: 32, endian: bigEndian,floatSize: 64,bit: 32), (name: "nimrodvm", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32), (name: "avr", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16), - (name: "msp430", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16)] + (name: "msp430", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16), + (name: "sparc64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64)] var targetCPU*, hostCPU*: TSystemCPU diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 2280ef712..db5f8e727 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -63,11 +63,12 @@ const wImportCpp, wImportObjC, wError, wNoInit, wCompileTime, wGlobal, wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims} constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl, - wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject, wExportNims} + wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject, wExportNims, + wIntDefine, wStrDefine} letPragmas* = varPragmas procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNosideeffect, wThread, wRaises, wLocks, wTags, wGcSafe} - allRoutinePragmas* = procPragmas + iteratorPragmas + lambdaPragmas + allRoutinePragmas* = methodPragmas + iteratorPragmas + lambdaPragmas proc pragma*(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords) # implementation @@ -89,43 +90,38 @@ proc pragmaAsm*(c: PContext, n: PNode): char = else: invalidPragma(it) -proc setExternName(s: PSym, extname: string) = - s.loc.r = rope(extname % s.name.s) +proc setExternName(s: PSym, extname: string, info: TLineInfo) = + # special cases to improve performance: + if extname == "$1": + s.loc.r = rope(s.name.s) + elif '$' notin extname: + s.loc.r = rope(extname) + else: + try: + s.loc.r = rope(extname % s.name.s) + except ValueError: + localError(info, "invalid extern name: '" & extname & "'. (Forgot to escape '$'?)") if gCmd == cmdPretty and '$' notin extname: # note that '{.importc.}' is transformed into '{.importc: "$1".}' s.loc.flags.incl(lfFullExternalName) -proc makeExternImport(s: PSym, extname: string) = - setExternName(s, extname) +proc makeExternImport(s: PSym, extname: string, info: TLineInfo) = + setExternName(s, extname, info) incl(s.flags, sfImportc) excl(s.flags, sfForward) -proc validateExternCName(s: PSym, info: TLineInfo) = - ## Validates that the symbol name in s.loc.r is a valid C identifier. - ## - ## Valid identifiers are those alphanumeric including the underscore not - ## starting with a number. If the check fails, a generic error will be - ## displayed to the user. - let target = $s.loc.r - if target.len < 1 or target[0] notin IdentStartChars or - not target.allCharsInSet(IdentChars): - localError(info, errGenerated, "invalid exported symbol") - proc makeExternExport(s: PSym, extname: string, info: TLineInfo) = - setExternName(s, extname) - # XXX to fix make it work with nimrtl. - #if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC}: - # validateExternCName(s, info) + setExternName(s, extname, info) incl(s.flags, sfExportc) -proc processImportCompilerProc(s: PSym, extname: string) = - setExternName(s, extname) +proc processImportCompilerProc(s: PSym, extname: string, info: TLineInfo) = + setExternName(s, extname, info) incl(s.flags, sfImportc) excl(s.flags, sfForward) incl(s.loc.flags, lfImportCompilerProc) -proc processImportCpp(s: PSym, extname: string) = - setExternName(s, extname) +proc processImportCpp(s: PSym, extname: string, info: TLineInfo) = + setExternName(s, extname, info) incl(s.flags, sfImportc) incl(s.flags, sfInfixCall) excl(s.flags, sfForward) @@ -133,8 +129,8 @@ proc processImportCpp(s: PSym, extname: string) = incl(m.flags, sfCompileToCpp) extccomp.gMixedMode = true -proc processImportObjC(s: PSym, extname: string) = - setExternName(s, extname) +proc processImportObjC(s: PSym, extname: string, info: TLineInfo) = + setExternName(s, extname, info) incl(s.flags, sfImportc) incl(s.flags, sfNamedParamCall) excl(s.flags, sfForward) @@ -256,8 +252,9 @@ proc expectDynlibNode(c: PContext, n: PNode): PNode = proc processDynLib(c: PContext, n: PNode, sym: PSym) = if (sym == nil) or (sym.kind == skModule): - POptionEntry(c.optionStack.tail).dynlib = getLib(c, libDynamic, - expectDynlibNode(c, n)) + let lib = getLib(c, libDynamic, expectDynlibNode(c, n)) + if not lib.isOverriden: + POptionEntry(c.optionStack.tail).dynlib = lib else: if n.kind == nkExprColonExpr: var lib = getLib(c, libDynamic, expectDynlibNode(c, n)) @@ -276,6 +273,7 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym) = proc processNote(c: PContext, n: PNode) = if (n.kind == nkExprColonExpr) and (sonsLen(n) == 2) and (n.sons[0].kind == nkBracketExpr) and + (n.sons[0].sons.len == 2) and (n.sons[0].sons[1].kind == nkIdent) and (n.sons[0].sons[0].kind == nkIdent): #and (n.sons[1].kind == nkIdent): @@ -391,21 +389,25 @@ type TLinkFeature = enum linkNormal, linkSys -proc processCompile(c: PContext, n: PNode) = +proc relativeFile(c: PContext; n: PNode; ext=""): string = var s = expectStrLit(c, n) - var found = findFile(s) - if found == "": found = s - var trunc = changeFileExt(found, "") - if not isAbsolute(found): - found = parentDir(n.info.toFullPath) / found + if ext.len > 0 and splitFile(s).ext == "": + s = addFileExt(s, ext) + result = parentDir(n.info.toFullPath) / s + if not fileExists(result): + if isAbsolute(s): result = s + else: + result = findFile(s) + if result.len == 0: result = s + +proc processCompile(c: PContext, n: PNode) = + let found = relativeFile(c, n) + let trunc = found.changeFileExt("") extccomp.addExternalFileToCompile(found) extccomp.addFileToLink(completeCFilePath(trunc, false)) proc processCommonLink(c: PContext, n: PNode, feature: TLinkFeature) = - var f = expectStrLit(c, n) - if splitFile(f).ext == "": f = addFileExt(f, CC[cCompiler].objExt) - var found = findFile(f) - if found == "": found = f # use the default + let found = relativeFile(c, n, CC[cCompiler].objExt) case feature of linkNormal: extccomp.addFileToLink(found) of linkSys: @@ -564,7 +566,7 @@ proc deprecatedStmt(c: PContext; pragma: PNode) = localError(pragma.info, "list of key:value pairs expected"); return for n in pragma: if n.kind in {nkExprColonExpr, nkExprEqExpr}: - let dest = qualifiedLookUp(c, n[1]) + let dest = qualifiedLookUp(c, n[1], {checkUndeclared}) let src = considerQuotedIdent(n[0]) let alias = newSym(skAlias, src, dest, n[0].info) incl(alias.flags, sfExported) @@ -589,7 +591,7 @@ proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym = # and perform the lookup on demand instead. result = newSym(skUnknown, considerQuotedIdent(n), nil, n.info) else: - result = qualifiedLookUp(c, n) + result = qualifiedLookUp(c, n, {checkUndeclared}) proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, validPragmas: TSpecialWords): bool = @@ -616,20 +618,23 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, of wExportc: makeExternExport(sym, getOptionalStr(c, it, "$1"), it.info) incl(sym.flags, sfUsed) # avoid wrong hints - of wImportc: makeExternImport(sym, getOptionalStr(c, it, "$1")) + of wImportc: makeExternImport(sym, getOptionalStr(c, it, "$1"), it.info) of wImportCompilerProc: - processImportCompilerProc(sym, getOptionalStr(c, it, "$1")) - of wExtern: setExternName(sym, expectStrLit(c, it)) + processImportCompilerProc(sym, getOptionalStr(c, it, "$1"), it.info) + of wExtern: setExternName(sym, expectStrLit(c, it), it.info) of wImmediate: - if sym.kind in {skTemplate, skMacro}: incl(sym.flags, sfImmediate) + if sym.kind in {skTemplate, skMacro}: + incl(sym.flags, sfImmediate) + incl(sym.flags, sfAllUntyped) + message(n.info, warnDeprecated, "use 'untyped' parameters instead; immediate") else: invalidPragma(it) of wDirty: if sym.kind == skTemplate: incl(sym.flags, sfDirty) else: invalidPragma(it) of wImportCpp: - processImportCpp(sym, getOptionalStr(c, it, "$1")) + processImportCpp(sym, getOptionalStr(c, it, "$1"), it.info) of wImportObjC: - processImportObjC(sym, getOptionalStr(c, it, "$1")) + processImportObjC(sym, getOptionalStr(c, it, "$1"), it.info) of wAlign: if sym.typ == nil: invalidPragma(it) var align = expectIntLit(c, it) @@ -894,6 +899,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, of wBase: noVal(it) sym.flags.incl sfBase + of wIntDefine: + sym.magic = mIntDefine + of wStrDefine: + sym.magic = mStrDefine else: invalidPragma(it) else: invalidPragma(it) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 03f6d4832..0e733d643 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -704,7 +704,10 @@ proc gcase(g: var TSrcGen, n: PNode) = proc gproc(g: var TSrcGen, n: PNode) = var c: TContext if n.sons[namePos].kind == nkSym: - put(g, tkSymbol, renderDefinitionName(n.sons[namePos].sym)) + let s = n.sons[namePos].sym + put(g, tkSymbol, renderDefinitionName(s)) + if sfGenSym in s.flags: + put(g, tkIntLit, $s.id) else: gsub(g, n.sons[namePos]) @@ -712,7 +715,11 @@ proc gproc(g: var TSrcGen, n: PNode) = gpattern(g, n.sons[patternPos]) let oldCheckAnon = g.checkAnon g.checkAnon = true - gsub(g, n.sons[genericParamsPos]) + if renderNoBody in g.flags and n[miscPos].kind != nkEmpty and + n[miscPos][1].kind != nkEmpty: + gsub(g, n[miscPos][1]) + else: + gsub(g, n.sons[genericParamsPos]) g.checkAnon = oldCheckAnon gsub(g, n.sons[paramsPos]) gsub(g, n.sons[pragmasPos]) @@ -794,7 +801,8 @@ proc gident(g: var TSrcGen, n: PNode) = else: t = tkOpr put(g, t, s) - if n.kind == nkSym and renderIds in g.flags: put(g, tkIntLit, $n.sym.id) + if n.kind == nkSym and (renderIds in g.flags or sfGenSym in n.sym.flags): + put(g, tkIntLit, $n.sym.id) proc doParamsAux(g: var TSrcGen, params: PNode) = if params.len > 1: @@ -802,7 +810,7 @@ proc doParamsAux(g: var TSrcGen, params: PNode) = gsemicolon(g, params, 1) put(g, tkParRi, ")") - if params.sons[0].kind != nkEmpty: + if params.len > 0 and params.sons[0].kind != nkEmpty: putWithSpace(g, tkOpr, "->") gsub(g, params.sons[0]) diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim index 91f93d279..d24e8f560 100644 --- a/compiler/rodutils.nim +++ b/compiler/rodutils.nim @@ -21,7 +21,7 @@ proc toStrMaxPrecision*(f: BiggestFloat): string = if f > 0.0: result = "INF" else: result = "-INF" else: - var buf: array [0..80, char] + var buf: array[0..80, char] c_sprintf(buf, "%#.16e", f) result = $buf @@ -64,7 +64,7 @@ const const vintDelta = 5 -template encodeIntImpl(self: expr) = +template encodeIntImpl(self) = var d: char var v = x var rem = v mod 190 diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index 22cd282fd..8b2653bc9 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -13,10 +13,10 @@ import ast, modules, passes, passaux, condsyms, options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs, - os, times, osproc + os, times, osproc, wordrecg, strtabs # we support 'cmpIgnoreStyle' natively for efficiency: -from strutils import cmpIgnoreStyle +from strutils import cmpIgnoreStyle, contains proc listDirs(a: VmArgs, filter: set[PathComponent]) = let dir = getString(a, 0) @@ -116,12 +116,27 @@ proc setupVM*(module: PSym; scriptName: string): PEvalContext = setResult(a, options.command) cbconf switch: processSwitch(a.getString 0, a.getString 1, passPP, unknownLineInfo()) - - -proc runNimScript*(scriptName: string) = + cbconf hintImpl: + processSpecificNote(a.getString 0, wHint, passPP, unknownLineInfo(), + a.getString 1) + cbconf warningImpl: + processSpecificNote(a.getString 0, wWarning, passPP, unknownLineInfo(), + a.getString 1) + cbconf patchFile: + let key = a.getString(0) & "_" & a.getString(1) + var val = a.getString(2).addFileExt(NimExt) + if {'$', '~'} in val: + val = pathSubs(val, vthisDir) + elif not isAbsolute(val): + val = vthisDir / val + gModuleOverrides[key] = val + cbconf selfExe: + setResult(a, os.getAppFilename()) + +proc runNimScript*(scriptName: string; freshDefines=true) = passes.gIncludeFile = includeModule passes.gImportModule = importModule - initDefines() + if freshDefines: initDefines() defineSymbol("nimscript") defineSymbol("nimconfig") diff --git a/compiler/sem.nim b/compiler/sem.nim index 97a20a4da..7db4ae47e 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -59,21 +59,11 @@ template semIdeForTemplateOrGeneric(c: PContext; n: PNode; # templates perform some quick check whether the cursor is actually in # the generic or template. when defined(nimsuggest): - assert gCmd == cmdIdeTools - if requiresCheck: + if gCmd == cmdIdeTools and requiresCheck: #if optIdeDebug in gGlobalOptions: # echo "passing to safeSemExpr: ", renderTree(n) discard safeSemExpr(c, n) -proc typeMismatch(n: PNode, formal, actual: PType) = - if formal.kind != tyError and actual.kind != tyError: - let named = typeToString(formal) - let desc = typeToString(formal, preferDesc) - let x = if named == desc: named else: named & " = " & desc - localError(n.info, errGenerated, msgKindToString(errTypeMismatch) & - typeToString(actual) & ") " & - `%`(msgKindToString(errButExpectedX), [x])) - proc fitNode(c: PContext, formal: PType, arg: PNode): PNode = if arg.typ.isNil: localError(arg.info, errExprXHasNoType, @@ -174,7 +164,7 @@ proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym = result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info) proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym = - proc `$`(kind: TSymKind): string = substr(system.`$`(kind), 2).toLower + proc `$`(kind: TSymKind): string = substr(system.`$`(kind), 2).toLowerAscii # like newSymS, but considers gensym'ed symbols if n.kind == nkSym: @@ -316,6 +306,13 @@ proc semConstExpr(c: PContext, n: PNode): PNode = include hlo, seminst, semcall +when false: + # hopefully not required: + proc resetSemFlag(n: PNode) = + excl n.flags, nfSem + for i in 0 ..< n.safeLen: + resetSemFlag(n[i]) + proc semAfterMacroCall(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = ## Semantically check the output of a macro. @@ -329,6 +326,8 @@ proc semAfterMacroCall(c: PContext, n: PNode, s: PSym, c.friendModules.add(s.owner.getModule) result = n + excl(n.flags, nfSem) + #resetSemFlag n if s.typ.sons[0] == nil: result = semStmt(c, result) else: @@ -363,7 +362,6 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym, #if c.evalContext == nil: # c.evalContext = c.createEvalContext(emStatic) - result = evalMacroCall(c.module, n, nOrig, sym) if efNoSemCheck notin flags: result = semAfterMacroCall(c, result, sym, flags) @@ -419,17 +417,42 @@ proc myOpen(module: PSym): PPassContext = c.importTable.addSym(module) # a module knows itself if sfSystemModule in module.flags: magicsys.systemModule = module # set global variable! - else: - c.importTable.addSym magicsys.systemModule # import the "System" identifier - importAllSymbols(c, magicsys.systemModule) c.topLevelScope = openScope(c) + # don't be verbose unless the module belongs to the main package: + if module.owner.id == gMainPackageId: + gNotes = gMainPackageNotes + else: + if gMainPackageNotes == {}: gMainPackageNotes = gNotes + gNotes = ForeignPackageNotes result = c proc myOpenCached(module: PSym, rd: PRodReader): PPassContext = result = myOpen(module) for m in items(rd.methods): methodDef(m, true) +proc isImportSystemStmt(n: PNode): bool = + if magicsys.systemModule == nil: return false + case n.kind + of nkImportStmt: + for x in n: + let f = checkModuleName(x) + if f == magicsys.systemModule.info.fileIndex: + return true + of nkImportExceptStmt, nkFromStmt: + let f = checkModuleName(n[0]) + if f == magicsys.systemModule.info.fileIndex: + return true + else: discard + proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = + if c.topStmts == 0 and not isImportSystemStmt(n): + if sfSystemModule notin c.module.flags and + n.kind notin {nkEmpty, nkCommentStmt}: + c.importTable.addSym magicsys.systemModule # import the "System" identifier + importAllSymbols(c, magicsys.systemModule) + inc c.topStmts + else: + inc c.topStmts if sfNoForward in c.module.flags: result = semAllTypeSections(c, n) else: diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index a1e209263..9702128ba 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -183,7 +183,7 @@ proc newSeqCall(c: PContext; x, y: PNode): PNode = proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = case t.kind - of tyNone, tyEmpty: discard + of tyNone, tyEmpty, tyVoid: discard of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString, tyPtr, tyString, tyRef: defaultOp(c, t, body, x, y) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 916d897c9..97209167d 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -53,7 +53,18 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, if symx.kind in filter: syms.add((symx, o.lastOverloadScope)) symx = nextOverloadIter(o, c, headSymbol) - if syms.len == 0: return + if syms.len == 0: + when false: + if skIterator notin filter: + # also try iterators, but these are 2nd class: + symx = initOverloadIter(o, c, headSymbol) + while symx != nil: + if symx.kind == skIterator: + syms.add((symx, 100)) + symx = nextOverloadIter(o, c, headSymbol) + if syms.len == 0: return + else: + return var z: TCandidate initCandidate(c, best, syms[0][0], initialBinding, symScope) @@ -69,7 +80,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, # gDebug = true matches(c, n, orig, z) if errors != nil: - errors.safeAdd(sym) + errors.safeAdd((sym, int z.mutabilityProblem)) if z.errors != nil: for err in z.errors: errors.add(err) @@ -111,7 +122,7 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = let proto = describeArgs(c, n, 1, preferName) var prefer = preferName - for err in errors: + for err, mut in items(errors): var errProto = "" let n = err.typ.n for i in countup(1, n.len - 1): @@ -123,14 +134,21 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = if errProto == proto: prefer = preferModuleInfo break + # now use the information stored in 'prefer' to produce a nice error message: var result = msgKindToString(errTypeMismatch) add(result, describeArgs(c, n, 1, prefer)) add(result, ')') var candidates = "" - for err in errors: - add(candidates, err.getProcHeader(prefer)) + for err, mut in items(errors): + if err.kind in routineKinds and err.ast != nil: + add(candidates, renderTree(err.ast, + {renderNoBody, renderNoComments,renderNoPragmas})) + else: + add(candidates, err.getProcHeader(prefer)) add(candidates, "\n") + if mut != 0 and mut < n.len: + add(candidates, "for a 'var' type a variable needs to be passed, but '" & renderTree(n[mut]) & "' is immutable\n") if candidates != "": add(result, "\n" & msgKindToString(errButExpected) & "\n" & candidates) if c.compilesContextId > 0 and optReportConceptFailures in gGlobalOptions: @@ -138,6 +156,20 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = else: localError(n.info, errGenerated, result) +proc bracketNotFoundError(c: PContext; n: PNode) = + var errors: CandidateErrors = @[] + var o: TOverloadIter + let headSymbol = n[0] + var symx = initOverloadIter(o, c, headSymbol) + while symx != nil: + if symx.kind in routineKinds: + errors.add((symx, 0)) + symx = nextOverloadIter(o, c, headSymbol) + if errors.len == 0: + localError(n.info, "could not resolve: " & $n) + else: + notFoundError(c, n, errors) + proc resolveOverloads(c: PContext, n, orig: PNode, filter: TSymKinds; errors: var CandidateErrors): TCandidate = @@ -154,8 +186,6 @@ proc resolveOverloads(c: PContext, n, orig: PNode, template pickBest(headSymbol) = pickBestCandidate(c, headSymbol, n, orig, initialBinding, filter, result, alt, errors) - - pickBest(f) let overloadsState = result.state @@ -226,7 +256,6 @@ proc resolveOverloads(c: PContext, n, orig: PNode, #notFoundError(c, n, errors) return - if alt.state == csMatch and cmpCandidates(result, alt) == 0 and not sameMethodDispatcher(result.calleeSym, alt.calleeSym): internalAssert result.state == csMatch @@ -295,8 +324,7 @@ proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode = var finalCallee = x.calleeSym markUsed(n.sons[0].info, finalCallee) styleCheckUse(n.sons[0].info, finalCallee) - if finalCallee.ast == nil: - internalError(n.info, "calleeSym.ast is nil") # XXX: remove this check! + assert finalCallee.ast != nil if x.hasFauxMatch: result = x.call result.sons[0] = newSymNode(finalCallee, result.sons[0].info) @@ -315,12 +343,12 @@ proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode = # are added as normal params. for s in instantiateGenericParamList(c, gp, x.bindings): case s.kind - of skConst: - x.call.add s.ast - of skType: - x.call.add newSymNode(s, n.info) - else: - internalAssert false + of skConst: + x.call.add s.ast + of skType: + x.call.add newSymNode(s, n.info) + else: + internalAssert false result = x.call instGenericConvertersSons(c, result, x) @@ -361,7 +389,14 @@ proc explicitGenericInstError(n: PNode): PNode = proc explicitGenericSym(c: PContext, n: PNode, s: PSym): PNode = var m: TCandidate - initCandidate(c, m, s, n) + # binding has to stay 'nil' for this to work! + initCandidate(c, m, s, nil) + + for i in 1..sonsLen(n)-1: + let formal = s.ast.sons[genericParamsPos].sons[i-1].typ + let arg = n[i].typ + let tm = typeRel(m, formal, arg, true) + if tm in {isNone, isConvertible}: return nil var newInst = generateInstance(c, s, m.bindings, n.info) markUsed(n.info, s) styleCheckUse(n.info, s) @@ -382,6 +417,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode = "; got " & $(n.len-1) & " type(s) but expected " & $expected) return n result = explicitGenericSym(c, n, s) + if result == nil: result = explicitGenericInstError(n) elif a.kind in {nkClosedSymChoice, nkOpenSymChoice}: # choose the generic proc with the proper number of type parameters. # XXX I think this could be improved by reusing sigmatch.paramTypesMatch. @@ -394,11 +430,12 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode = # it suffices that the candidate has the proper number of generic # type parameters: if safeLen(candidate.ast.sons[genericParamsPos]) == n.len-1: - result.add(explicitGenericSym(c, n, candidate)) + let x = explicitGenericSym(c, n, candidate) + if x != nil: result.add(x) # get rid of nkClosedSymChoice if not ambiguous: if result.len == 1 and a.kind == nkClosedSymChoice: result = result[0] - # candidateCount != 1: return explicitGenericInstError(n) + elif result.len == 0: result = explicitGenericInstError(n) else: result = explicitGenericInstError(n) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index a13f2c232..30b6e261d 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -47,7 +47,7 @@ type efLValue, efWantIterator, efInTypeof, efWantStmt, efAllowStmt, efDetermineType, efAllowDestructor, efWantValue, efOperand, efNoSemCheck, - efNoProcvarCheck + efNoProcvarCheck, efNoEvaluateGeneric, efInCall, efFromHlo TExprFlags* = set[TExprFlag] TTypeAttachedOp* = enum @@ -99,11 +99,12 @@ type unknownIdents*: IntSet # ids of all unknown identifiers to prevent # naming it multiple times generics*: seq[TInstantiationPair] # pending list of instantiated generics to compile + topStmts*: int # counts the number of encountered top level statements lastGenericIdx*: int # used for the generics stack hloLoopDetector*: int # used to prevent endless loops in the HLO inParallelStmt*: int instTypeBoundOp*: proc (c: PContext; dc: PSym; t: PType; info: TLineInfo; - op: TTypeAttachedOp): PSym {.nimcall.} + op: TTypeAttachedOp; col: int): PSym {.nimcall.} selfName*: PIdent signatures*: TStrTable diff --git a/compiler/semdestruct.nim b/compiler/semdestruct.nim index 1261dd460..9ea581f3a 100644 --- a/compiler/semdestruct.nim +++ b/compiler/semdestruct.nim @@ -87,7 +87,7 @@ proc destroyCase(c: PContext, n: PNode, holder: PNode): PNode = result = nil proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode = - template maybeAddLine(e: expr): stmt = + template maybeAddLine(e) = let stmt = e if stmt != nil: if result == nil: result = newNode(nkStmtList) @@ -139,6 +139,7 @@ proc instantiateDestructor(c: PContext, typ: PType): PType = t = t.skipTypes({tyGenericInst}) case t.kind of tySequence, tyArray, tyArrayConstr, tyOpenArray, tyVarargs: + t.destructor = analyzingDestructor if instantiateDestructor(c, t.sons[0]) != nil: t.destructor = getCompilerProc"nimDestroyRange" return t diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 16b4ee479..fc31829ba 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -15,7 +15,7 @@ proc semTemplateExpr(c: PContext, n: PNode, s: PSym, markUsed(n.info, s) styleCheckUse(n.info, s) pushInfoContext(n.info) - result = evalTemplate(n, s, getCurrOwner()) + result = evalTemplate(n, s, getCurrOwner(), efFromHlo in flags) if efNoSemCheck notin flags: result = semAfterMacroCall(c, result, s, flags) popInfoContext() @@ -24,16 +24,15 @@ proc semFieldAccess(c: PContext, n: PNode, flags: TExprFlags = {}): PNode proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = # same as 'semExprWithType' but doesn't check for proc vars result = semExpr(c, n, flags + {efOperand}) - if result.kind == nkEmpty and result.typ.isNil: + #if result.kind == nkEmpty and result.typ.isNil: # do not produce another redundant error message: #raiseRecoverableError("") - result = errorNode(c, n) + # result = errorNode(c, n) if result.typ != nil: # XXX tyGenericInst here? if result.typ.kind == tyVar: result = newDeref(result) elif {efWantStmt, efAllowStmt} * flags != {}: - result.typ = newTypeS(tyEmpty, c) - result.typ.flags.incl tfVoid + result.typ = newTypeS(tyVoid, c) else: localError(n.info, errExprXHasNoType, renderTree(result, {renderNoComments})) @@ -51,7 +50,6 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = renderTree(result, {renderNoComments})) result.typ = errorType(c) else: - # XXX tyGenericInst here? if efNoProcvarCheck notin flags: semProcvarCheck(c, result) if result.typ.kind == tyVar: result = newDeref(result) semDestructorCheck(c, result, flags) @@ -74,8 +72,12 @@ proc semSymGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode = proc inlineConst(n: PNode, s: PSym): PNode {.inline.} = result = copyTree(s.ast) - result.typ = s.typ - result.info = n.info + if result.isNil: + localError(n.info, "constant of type '" & typeToString(s.typ) & "' has no value") + result = newSymNode(s) + else: + result.typ = s.typ + result.info = n.info type TConvStatus = enum @@ -83,8 +85,9 @@ type convNotNeedeed, convNotLegal -proc checkConversionBetweenObjects(castDest, src: PType): TConvStatus = - return if inheritanceDiff(castDest, src) == high(int): +proc checkConversionBetweenObjects(castDest, src: PType; pointers: int): TConvStatus = + let diff = inheritanceDiff(castDest, src) + return if diff == high(int) or (pointers > 1 and diff != 0): convNotLegal else: convOK @@ -101,13 +104,15 @@ proc checkConvertible(c: PContext, castDest, src: PType): TConvStatus = return var d = skipTypes(castDest, abstractVar) var s = skipTypes(src, abstractVar-{tyTypeDesc}) + var pointers = 0 while (d != nil) and (d.kind in {tyPtr, tyRef}) and (d.kind == s.kind): d = d.lastSon s = s.lastSon + inc pointers if d == nil: result = convNotLegal elif d.kind == tyObject and s.kind == tyObject: - result = checkConversionBetweenObjects(d, s) + result = checkConversionBetweenObjects(d, s, pointers) elif (skipTypes(castDest, abstractVarRange).kind in IntegralTypes) and (skipTypes(src, abstractVarRange-{tyTypeDesc}).kind in IntegralTypes): # accept conversion between integral types @@ -186,17 +191,17 @@ proc semConv(c: PContext, n: PNode): PNode = case status of convOK: # handle SomeProcType(SomeGenericProc) - # XXX: This needs fixing. checkConvertible uses typeRel internally, but - # doesn't bother to perform the work done in paramTypeMatchAux/fitNode - # so we are redoing the typeRel work here. Why does semConv exist as a - # separate proc from fitNode? if op.kind == nkSym and op.sym.isGenericRoutine: result.sons[1] = fitNode(c, result.typ, result.sons[1]) + elif op.kind == nkPar and targetType.kind == tyTuple: + op = fitNode(c, targetType, op) of convNotNeedeed: message(n.info, hintConvFromXtoItselfNotNeeded, result.typ.typeToString) of convNotLegal: - localError(n.info, errGenerated, msgKindToString(errIllegalConvFromXtoY)% - [op.typ.typeToString, result.typ.typeToString]) + result = fitNode(c, result.typ, result.sons[1]) + if result == nil: + localError(n.info, errGenerated, msgKindToString(errIllegalConvFromXtoY)% + [op.typ.typeToString, result.typ.typeToString]) else: for i in countup(0, sonsLen(op) - 1): let it = op.sons[i] @@ -206,7 +211,7 @@ proc semConv(c: PContext, n: PNode): PNode = styleCheckUse(n.info, it.sym) markIndirect(c, it.sym) return it - localError(n.info, errUseQualifier, op.sons[0].sym.name.s) + errorUseQualifier(c, n.info, op.sons[0].sym) proc semCast(c: PContext, n: PNode): PNode = ## Semantically analyze a casting ("cast[type](param)") @@ -594,6 +599,9 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode = result = n if n.kind notin nkCallKinds or n.sons[0].kind != nkSym: return var callee = n.sons[0].sym + # workaround for bug #537 (overly aggressive inlining leading to + # wrong NimNode semantics): + if n.typ != nil and tfTriggersCompileTime in n.typ.flags: return # constant folding that is necessary for correctness of semantic pass: if callee.magic != mNone and callee.magic in ctfeWhitelist and n.typ != nil: @@ -710,25 +718,60 @@ proc resolveIndirectCall(c: PContext; n, nOrig: PNode; initCandidate(c, result, t) matches(c, n, nOrig, result) +proc bracketedMacro(n: PNode): PSym = + if n.len >= 1 and n[0].kind == nkSym: + result = n[0].sym + if result.kind notin {skMacro, skTemplate}: + result = nil + +proc semBracketedMacro(c: PContext; outer, inner: PNode; s: PSym; + flags: TExprFlags): PNode = + # We received untransformed bracket expression coming from macroOrTmpl[]. + # Transform it to macro or template call, where first come normal + # arguments, next come generic template arguments. + var sons = newSeq[PNode]() + sons.add inner.sons[0] + # Normal arguments: + for i in 1..<outer.len: + sons.add outer.sons[i] + # Generic template arguments from bracket expression: + for i in 1..<inner.len: + sons.add inner.sons[i] + shallowCopy(outer.sons, sons) + # FIXME: Shouldn't we check sfImmediate and call semDirectOp? + # However passing to semDirectOp doesn't work here. + case s.kind + of skMacro: result = semMacroExpr(c, outer, outer, s, flags) + of skTemplate: result = semTemplateExpr(c, outer, s, flags) + else: assert(false) + return + proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = result = nil checkMinSonsLen(n, 1) var prc = n.sons[0] if n.sons[0].kind == nkDotExpr: checkSonsLen(n.sons[0], 2) - n.sons[0] = semFieldAccess(c, n.sons[0]) - if n.sons[0].kind == nkDotCall: + let n0 = semFieldAccess(c, n.sons[0]) + if n0.kind == nkDotCall: # it is a static call! - result = n.sons[0] + result = n0 result.kind = nkCall result.flags.incl nfExplicitCall for i in countup(1, sonsLen(n) - 1): addSon(result, n.sons[i]) return semExpr(c, result, flags) + else: + n.sons[0] = n0 else: - n.sons[0] = semExpr(c, n.sons[0]) + n.sons[0] = semExpr(c, n.sons[0], {efInCall}) let t = n.sons[0].typ if t != nil and t.kind == tyVar: n.sons[0] = newDeref(n.sons[0]) + elif n.sons[0].kind == nkBracketExpr: + let s = bracketedMacro(n.sons[0]) + if s != nil: + return semBracketedMacro(c, n, n.sons[0], s, flags) + let nOrig = n.copyTree semOpAux(c, n) var t: PType = nil @@ -958,8 +1001,20 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = else: result = newSymNode(s, n.info) else: result = newSymNode(s, n.info) - of skMacro: result = semMacroExpr(c, n, n, s, flags) - of skTemplate: result = semTemplateExpr(c, n, s, flags) + of skMacro: + if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0: + markUsed(n.info, s) + styleCheckUse(n.info, s) + result = newSymNode(s, n.info) + else: + result = semMacroExpr(c, n, n, s, flags) + of skTemplate: + if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0: + markUsed(n.info, s) + styleCheckUse(n.info, s) + result = newSymNode(s, n.info) + else: + result = semTemplateExpr(c, n, s, flags) of skParam: markUsed(n.info, s) styleCheckUse(n.info, s) @@ -1011,8 +1066,11 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = result = newSymNode(s, n.info) result.typ = makeTypeDesc(c, s.typ) of skField: - if c.p != nil and c.p.selfSym != nil: - var ty = skipTypes(c.p.selfSym.typ, {tyGenericInst, tyVar, tyPtr, tyRef}) + var p = c.p + while p != nil and p.selfSym == nil: + p = p.next + if p != nil and p.selfSym != nil: + var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyPtr, tyRef}) while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct}) var check: PNode = nil if ty.kind == tyObject: @@ -1025,7 +1083,7 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = markUsed(n.info, f) styleCheckUse(n.info, f) result = newNodeIT(nkDotExpr, n.info, f.typ) - result.add makeDeref(newSymNode(c.p.selfSym)) + result.add makeDeref(newSymNode(p.selfSym)) result.add newSymNode(f) # we now have the correct field if check != nil: check.sons[0] = result @@ -1051,7 +1109,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = # here at all! #if isSymChoice(n.sons[1]): return - var s = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared}) + var s = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared, checkModule}) if s != nil: if s.kind in OverloadableSyms: result = symChoice(c, n, s, scClosed) @@ -1186,7 +1244,9 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = result.add(x[0]) return checkMinSonsLen(n, 2) - n.sons[0] = semExprWithType(c, n.sons[0], {efNoProcvarCheck}) + # make sure we don't evaluate generic macros/templates + n.sons[0] = semExprWithType(c, n.sons[0], + {efNoProcvarCheck, efNoEvaluateGeneric}) let arr = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyPtr, tyRef}) case arr.kind of tyArray, tyOpenArray, tyVarargs, tyArrayConstr, tySequence, tyString, @@ -1229,12 +1289,30 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = let s = if n.sons[0].kind == nkSym: n.sons[0].sym elif n[0].kind in nkSymChoices: n.sons[0][0].sym else: nil - if s != nil and s.kind in {skProc, skMethod, skConverter, skIterator}: - # type parameters: partial generic specialization - n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s) - result = explicitGenericInstantiation(c, n, s) - elif s != nil and s.kind == skType: - result = symNodeFromType(c, semTypeNode(c, n, nil), n.info) + if s != nil: + case s.kind + of skProc, skMethod, skConverter, skIterator: + # type parameters: partial generic specialization + n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s) + result = explicitGenericInstantiation(c, n, s) + of skMacro, skTemplate: + if efInCall in flags: + # We are processing macroOrTmpl[] in macroOrTmpl[](...) call. + # Return as is, so it can be transformed into complete macro or + # template call in semIndirectOp caller. + result = n + else: + # We are processing macroOrTmpl[] not in call. Transform it to the + # macro or template call with generic arguments here. + n.kind = nkCall + case s.kind + of skMacro: result = semMacroExpr(c, n, n, s, flags) + of skTemplate: result = semTemplateExpr(c, n, s, flags) + else: discard + of skType: + result = symNodeFromType(c, semTypeNode(c, n, nil), n.info) + else: + c.p.bracketExpr = n.sons[0] else: c.p.bracketExpr = n.sons[0] @@ -1287,7 +1365,7 @@ proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} = n.sons[1] = takeImplicitAddr(c, ri) x.typ.flags.incl tfVarIsPtr -template resultTypeIsInferrable(typ: PType): expr = +template resultTypeIsInferrable(typ: PType): untyped = typ.isMetaType and typ.kind != tyTypeDesc proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = @@ -1313,15 +1391,16 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = # --> `[]=`(a, i, x) let oldBracketExpr = c.p.bracketExpr a = semSubscript(c, a, {efLValue}) - if a == nil and mode != noOverloadedSubscript: + if a == nil: result = buildOverloadedSubscripts(n.sons[0], getIdent"[]=") add(result, n[1]) - result = semExprNoType(c, result) - c.p.bracketExpr = oldBracketExpr - return result - elif a == nil: - localError(n.info, "could not resolve: " & $n[0]) - return n + if mode == noOverloadedSubscript: + bracketNotFoundError(c, result) + return n + else: + result = semExprNoType(c, result) + c.p.bracketExpr = oldBracketExpr + return result c.p.bracketExpr = oldBracketExpr of nkCurlyExpr: # a{i} = x --> `{}=`(a, i, x) @@ -1557,8 +1636,8 @@ proc newAnonSym(kind: TSymKind, info: TLineInfo, result.flags = {sfGenSym} proc semExpandToAst(c: PContext, n: PNode): PNode = - var macroCall = n[1] - var expandedSym = expectMacroOrTemplateCall(c, macroCall) + let macroCall = n[1] + let expandedSym = expectMacroOrTemplateCall(c, macroCall) if expandedSym.kind == skError: return n macroCall.sons[0] = newSymNode(expandedSym, macroCall.info) @@ -1566,11 +1645,12 @@ proc semExpandToAst(c: PContext, n: PNode): PNode = styleCheckUse(n.info, expandedSym) for i in countup(1, macroCall.len-1): + #if macroCall.sons[0].typ.sons[i].kind != tyExpr: macroCall.sons[i] = semExprWithType(c, macroCall[i], {}) # Preserve the magic symbol in order to be handled in evals.nim internalAssert n.sons[0].sym.magic == mExpandToAst - #n.typ = getSysSym("PNimrodNode").typ # expandedSym.getReturnType + #n.typ = getSysSym("NimNode").typ # expandedSym.getReturnType n.typ = if getCompilerProc("NimNode") != nil: sysTypeFromName"NimNode" else: sysTypeFromName"PNimrodNode" result = n @@ -1810,8 +1890,8 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = # the correct branch. Otherwise the AST will be passed through semStmt. result = nil - template setResult(e: expr) = - if semCheck: result = semStmt(c, e) # do not open a new scope! + template setResult(e: untyped) = + if semCheck: result = semExpr(c, e) # do not open a new scope! else: result = e # Check if the node is "when nimvm" @@ -1820,6 +1900,7 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = # else: # ... var whenNimvm = false + var typ = commonTypeBegin if n.sons.len == 2 and n.sons[0].kind == nkElifBranch and n.sons[1].kind == nkElse: let exprNode = n.sons[0].sons[0] @@ -1836,7 +1917,8 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = checkSonsLen(it, 2) if whenNimvm: if semCheck: - it.sons[1] = semStmt(c, it.sons[1]) + it.sons[1] = semExpr(c, it.sons[1]) + typ = commonType(typ, it.sons[1].typ) result = n # when nimvm is not elimited until codegen else: var e = semConstExpr(c, it.sons[0]) @@ -1850,12 +1932,14 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = checkSonsLen(it, 1) if result == nil or whenNimvm: if semCheck: - it.sons[0] = semStmt(c, it.sons[0]) + it.sons[0] = semExpr(c, it.sons[0]) + typ = commonType(typ, it.sons[0].typ) if result == nil: result = it.sons[0] else: illFormedAst(n) if result == nil: result = newNodeI(nkEmpty, n.info) + if whenNimvm: result.typ = typ # The ``when`` statement implements the mechanism for platform dependent # code. Thus we try to ensure here consistent ID allocation after the # ``when`` statement. @@ -2132,7 +2216,9 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = if nfSem in n.flags: return case n.kind of nkIdent, nkAccQuoted: - var s = lookUp(c, n) + let checks = if efNoEvaluateGeneric in flags: {checkUndeclared} + else: {checkUndeclared, checkModule, checkAmbiguity} + var s = qualifiedLookUp(c, n, checks) if c.inTypeClass == 0: semCaptureSym(s, c.p.owner) result = semSym(c, n, s, flags) if s.kind in {skProc, skMethod, skConverter, skIterator}: @@ -2220,7 +2306,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = elif n.len == 1: result = semObjConstr(c, n, flags) elif contains(c.ambiguousSymbols, s.id): - localError(n.info, errUseQualifier, s.name.s) + errorUseQualifier(c, n.info, s) elif s.magic == mNone: result = semDirectOp(c, n, flags) else: result = semMagic(c, n, s, flags) of skProc, skMethod, skConverter, skIterator: @@ -2336,7 +2422,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = if not isTopLevel(c): localError(n.info, errXOnlyAtModuleScope, "from") result = evalFrom(c, n) of nkIncludeStmt: - if not isTopLevel(c): localError(n.info, errXOnlyAtModuleScope, "include") + #if not isTopLevel(c): localError(n.info, errXOnlyAtModuleScope, "include") result = evalInclude(c, n) of nkExportStmt, nkExportExceptStmt: if not isTopLevel(c): localError(n.info, errXOnlyAtModuleScope, "export") diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 5fe4e3299..02f238ae6 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -143,7 +143,7 @@ proc getIntervalType*(m: TMagic, n: PNode): PType = const ordIntLit = {nkIntLit..nkUInt64Lit} result = n.typ - template commutativeOp(opr: expr) {.immediate.} = + template commutativeOp(opr: untyped) = let a = n.sons[1] let b = n.sons[2] if isIntRangeOrLit(a.typ) and isIntRangeOrLit(b.typ): @@ -151,7 +151,7 @@ proc getIntervalType*(m: TMagic, n: PNode): PType = opr(pickMinInt(a), pickMinInt(b)), opr(pickMaxInt(a), pickMaxInt(b))) - template binaryOp(opr: expr) {.immediate.} = + template binaryOp(opr: untyped) = let a = n.sons[1] let b = n.sons[2] if isIntRange(a.typ) and b.kind in {nkIntLit..nkUInt64Lit}: @@ -286,13 +286,14 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode = of mNot: result = newIntNodeT(1 - getInt(a), n) of mCard: result = newIntNodeT(nimsets.cardSet(a), n) of mBitnotI: result = newIntNodeT(not getInt(a), n) - of mLengthStr, mXLenStr: - if a.kind == nkNilLit: result = newIntNodeT(0, n) - else: result = newIntNodeT(len(getStr(a)), n) of mLengthArray: result = newIntNodeT(lengthOrd(a.typ), n) - of mLengthSeq, mLengthOpenArray, mXLenSeq: - if a.kind == nkNilLit: result = newIntNodeT(0, n) - else: result = newIntNodeT(sonsLen(a), n) # BUGFIX + of mLengthSeq, mLengthOpenArray, mXLenSeq, mLengthStr, mXLenStr: + if a.kind == nkNilLit: + result = newIntNodeT(0, n) + elif a.kind in {nkStrLit..nkTripleStrLit}: + result = newIntNodeT(len a.strVal, n) + else: + result = newIntNodeT(sonsLen(a), n) # BUGFIX of mUnaryPlusI, mUnaryPlusF64: result = a # throw `+` away of mToFloat, mToBiggestFloat: result = newFloatNodeT(toFloat(int(getInt(a))), n) @@ -419,7 +420,14 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode = result = newStrNodeT(substr(getStr(a), int(getOrdValue(b)), int(getOrdValue(c))), n) of mFloatToStr: result = newStrNodeT($getFloat(a), n) - of mCStrToStr, mCharToStr: result = newStrNodeT(getStrOrChar(a), n) + of mCStrToStr, mCharToStr: + if a.kind == nkBracket: + var s = "" + for b in a.sons: + s.add b.getStrOrChar + result = newStrNodeT(s, n) + else: + result = newStrNodeT(getStrOrChar(a), n) of mStrToStr: result = a of mEnumToStr: result = newStrNodeT(ordinalValToString(a), n) of mArrToSeq: @@ -482,7 +490,7 @@ proc leValueConv(a, b: PNode): bool = of nkCharLit..nkUInt64Lit: case b.kind of nkCharLit..nkUInt64Lit: result = a.intVal <= b.intVal - of nkFloatLit..nkFloat128Lit: result = a.intVal <= round(b.floatVal) + of nkFloatLit..nkFloat128Lit: result = a.intVal <= round(b.floatVal).int else: internalError(a.info, "leValueConv") of nkFloatLit..nkFloat128Lit: case b.kind @@ -627,14 +635,20 @@ proc getConstExpr(m: PSym, n: PNode): PNode = of mCompileDate: result = newStrNodeT(times.getDateStr(), n) of mCompileTime: result = newStrNodeT(times.getClockStr(), n) of mCpuEndian: result = newIntNodeT(ord(CPU[targetCPU].endian), n) - of mHostOS: result = newStrNodeT(toLower(platform.OS[targetOS].name), n) - of mHostCPU: result = newStrNodeT(platform.CPU[targetCPU].name.toLower, n) + of mHostOS: result = newStrNodeT(toLowerAscii(platform.OS[targetOS].name), n) + of mHostCPU: result = newStrNodeT(platform.CPU[targetCPU].name.toLowerAscii, n) of mAppType: result = getAppType(n) of mNaN: result = newFloatNodeT(NaN, n) of mInf: result = newFloatNodeT(Inf, n) of mNegInf: result = newFloatNodeT(NegInf, n) + of mIntDefine: + if isDefined(s.name): + result = newIntNodeT(lookupSymbol(s.name).parseInt, n) + of mStrDefine: + if isDefined(s.name): + result = newStrNodeT(lookupSymbol(s.name), n) else: - if sfFakeConst notin s.flags: result = copyTree(s.ast) + result = copyTree(s.ast) of {skProc, skMethod}: result = n of skType: diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index 6651de78e..b8451865e 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -47,8 +47,11 @@ proc semGenericStmtScope(c: PContext, n: PNode, result = semGenericStmt(c, n, flags, ctx) closeScope(c) -template macroToExpand(s: expr): expr = - s.kind in {skMacro, skTemplate} and (s.typ.len == 1 or sfImmediate in s.flags) +template macroToExpand(s): untyped = + s.kind in {skMacro, skTemplate} and (s.typ.len == 1 or sfAllUntyped in s.flags) + +template macroToExpandSym(s): untyped = + s.kind in {skMacro, skTemplate} and (s.typ.len == 1) proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym, ctx: var GenericCtx): PNode = @@ -61,14 +64,14 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym, of skProc, skMethod, skIterator, skConverter, skModule: result = symChoice(c, n, s, scOpen) of skTemplate: - if macroToExpand(s): + if macroToExpandSym(s): styleCheckUse(n.info, s) result = semTemplateExpr(c, n, s, {efNoSemCheck}) result = semGenericStmt(c, result, {}, ctx) else: result = symChoice(c, n, s, scOpen) of skMacro: - if macroToExpand(s): + if macroToExpandSym(s): styleCheckUse(n.info, s) result = semMacroExpr(c, n, n, s, {efNoSemCheck}) result = semGenericStmt(c, result, {}, ctx) @@ -124,7 +127,7 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags, assert n.kind == nkDotExpr semIdeForTemplateOrGenericCheck(n, ctx.cursorInBody) - let luf = if withinMixin notin flags: {checkUndeclared} else: {} + let luf = if withinMixin notin flags: {checkUndeclared, checkModule} else: {checkModule} var s = qualifiedLookUp(c, n, luf) if s != nil: @@ -199,24 +202,25 @@ proc semGenericStmt(c: PContext, n: PNode, if s != nil: incl(s.flags, sfUsed) mixinContext = s.magic in {mDefined, mDefinedInScope, mCompiles} - let scOption = if s.name.id in ctx.toMixin: scForceOpen else: scOpen + let sc = symChoice(c, fn, s, + if s.name.id in ctx.toMixin: scForceOpen else: scOpen) case s.kind of skMacro: - if macroToExpand(s): + if macroToExpand(s) and sc.safeLen <= 1: styleCheckUse(fn.info, s) result = semMacroExpr(c, n, n, s, {efNoSemCheck}) - result = semGenericStmt(c, result, {}, ctx) + result = semGenericStmt(c, result, flags, ctx) else: - n.sons[0] = symChoice(c, fn, s, scOption) + n.sons[0] = sc result = n mixinContext = true of skTemplate: - if macroToExpand(s): + if macroToExpand(s) and sc.safeLen <= 1: styleCheckUse(fn.info, s) result = semTemplateExpr(c, n, s, {efNoSemCheck}) - result = semGenericStmt(c, result, {}, ctx) + result = semGenericStmt(c, result, flags, ctx) else: - n.sons[0] = symChoice(c, fn, s, scOption) + n.sons[0] = sc result = n # BUGFIX: we must not return here, we need to do first phase of # symbol lookup. Also since templates and macros can do scope injections @@ -227,7 +231,7 @@ proc semGenericStmt(c: PContext, n: PNode, # Leave it as an identifier. discard of skProc, skMethod, skIterator, skConverter, skModule: - result.sons[0] = symChoice(c, fn, s, scOption) + result.sons[0] = sc # do not check of 's.magic==mRoof' here because it might be some # other '^' but after overload resolution the proper one: if ctx.bracketExpr != nil and n.len == 2 and s.name.s == "^": @@ -426,7 +430,12 @@ proc semGenericStmt(c: PContext, n: PNode, n.sons[paramsPos] = semGenericStmt(c, n.sons[paramsPos], flags, ctx) n.sons[pragmasPos] = semGenericStmt(c, n.sons[pragmasPos], flags, ctx) var body: PNode - if n.sons[namePos].kind == nkSym: body = n.sons[namePos].sym.getBody + if n.sons[namePos].kind == nkSym: + let s = n.sons[namePos].sym + if sfGenSym in s.flags and s.ast == nil: + body = n.sons[bodyPos] + else: + body = s.getBody else: body = n.sons[bodyPos] n.sons[bodyPos] = semGenericStmtScope(c, body, flags, ctx) closeScope(c) diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 4a45dee9d..460db4f7c 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -43,9 +43,11 @@ proc rawHandleSelf(c: PContext; owner: PSym) = if arg.name.id == c.selfName.id: c.p.selfSym = arg arg.flags.incl sfIsSelf - let t = c.p.selfSym.typ.skipTypes(abstractPtrs) - if t.kind == tyObject: + var t = c.p.selfSym.typ.skipTypes(abstractPtrs) + while t.kind == tyObject: addObjFieldsToLocalScope(c, t.n) + if t.sons[0] == nil: break + t = t.sons[0].skipTypes(abstractPtrs) proc pushProcCon*(c: PContext; owner: PSym) = rawPushProcCon(c, owner) @@ -111,9 +113,9 @@ proc removeDefaultParamValues(n: PNode) = # not possible... XXX We don't solve this issue here. a.sons[L-1] = ast.emptyNode -proc freshGenSyms(n: PNode, owner: PSym, symMap: var TIdTable) = +proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) = # we need to create a fresh set of gensym'ed symbols: - if n.kind == nkSym and sfGenSym in n.sym.flags: + if n.kind == nkSym and sfGenSym in n.sym.flags and n.sym.owner == orig: let s = n.sym var x = PSym(idTableGet(symMap, s)) if x == nil: @@ -122,7 +124,7 @@ proc freshGenSyms(n: PNode, owner: PSym, symMap: var TIdTable) = idTablePut(symMap, s, x) n.sym = x else: - for i in 0 .. <safeLen(n): freshGenSyms(n.sons[i], owner, symMap) + for i in 0 .. <safeLen(n): freshGenSyms(n.sons[i], owner, orig, symMap) proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind) @@ -137,7 +139,7 @@ proc addProcDecls(c: PContext, fn: PSym) = maybeAddResult(c, fn, fn.ast) -proc instantiateBody(c: PContext, n, params: PNode, result: PSym) = +proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) = if n.sons[bodyPos].kind != nkEmpty: inc c.inGenericInst # add it here, so that recursive generic procs are possible: @@ -149,7 +151,7 @@ proc instantiateBody(c: PContext, n, params: PNode, result: PSym) = let param = params[i].sym if sfGenSym in param.flags: idTablePut(symMap, params[i].sym, result.typ.n[param.position+1].sym) - freshGenSyms(b, result, symMap) + freshGenSyms(b, result, orig, symMap) b = semProcBody(c, b) b = hloBody(c, b) n.sons[bodyPos] = transformBody(c.module, b, result) @@ -165,7 +167,7 @@ proc fixupInstantiatedSymbols(c: PContext, s: PSym) = openScope(c) var n = oldPrc.ast n.sons[bodyPos] = copyTree(s.getBody) - instantiateBody(c, n, nil, oldPrc) + instantiateBody(c, n, nil, oldPrc, s) closeScope(c) popInfoContext() @@ -203,7 +205,7 @@ proc instantiateProcType(c: PContext, pt: TIdTable, # The solution would be to move this logic into semtypinst, but # at this point semtypinst have to become part of sem, because it # will need to use openScope, addDecl, etc. - addDecl(c, prc) + #addDecl(c, prc) pushInfoContext(info) var cl = initTypeVars(c, pt, info, nil) @@ -312,7 +314,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable, pragma(c, result, n.sons[pragmasPos], allRoutinePragmas) if isNil(n.sons[bodyPos]): n.sons[bodyPos] = copyTree(fn.getBody) - instantiateBody(c, n, fn.typ.n, result) + instantiateBody(c, n, fn.typ.n, result, fn) sideEffectsCheck(c, result) paramsTypeCheck(c, result.typ) else: diff --git a/compiler/semmacrosanity.nim b/compiler/semmacrosanity.nim index 150680af7..6cd5c4a3c 100644 --- a/compiler/semmacrosanity.nim +++ b/compiler/semmacrosanity.nim @@ -12,25 +12,26 @@ import ast, astalgo, msgs, types -proc ithField(n: PNode, field: int): PSym = +proc ithField(n: PNode, field: var int): PSym = result = nil case n.kind of nkRecList: for i in countup(0, sonsLen(n) - 1): - result = ithField(n.sons[i], field-i) + result = ithField(n.sons[i], field) if result != nil: return of nkRecCase: if n.sons[0].kind != nkSym: internalError(n.info, "ithField") - result = ithField(n.sons[0], field-1) + result = ithField(n.sons[0], field) if result != nil: return for i in countup(1, sonsLen(n) - 1): case n.sons[i].kind of nkOfBranch, nkElse: - result = ithField(lastSon(n.sons[i]), field-1) + result = ithField(lastSon(n.sons[i]), field) if result != nil: return else: internalError(n.info, "ithField(record case branch)") of nkSym: if field == 0: result = n.sym + else: dec(field) else: discard proc annotateType*(n: PNode, t: PType) = @@ -39,10 +40,13 @@ proc annotateType*(n: PNode, t: PType) = # to not to skip tyGenericInst case n.kind of nkObjConstr: + let x = t.skipTypes(abstractPtrs) n.typ = t for i in 1 .. <n.len: - let field = x.n.ithField(i - 1) - if field.isNil: globalError n.info, "invalid field at index " & $i + var j = i-1 + let field = x.n.ithField(j) + if field.isNil: + globalError n.info, "invalid field at index " & $i else: internalAssert(n.sons[i].kind == nkExprColonExpr) annotateType(n.sons[i].sons[1], field.typ) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 1a70e4a12..cbe9bc176 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -42,7 +42,10 @@ proc semArrGet(c: PContext; n: PNode; flags: TExprFlags): PNode = result = semSubscript(c, result, flags) c.p.bracketExpr = oldBracketExpr if result.isNil: - localError(n.info, "could not resolve: " & $n) + let x = copyTree(n) + x.sons[0] = newIdentNode(getIdent"[]", n.info) + bracketNotFoundError(c, x) + #localError(n.info, "could not resolve: " & $n) result = n proc semArrPut(c: PContext; n: PNode; flags: TExprFlags): PNode = @@ -111,8 +114,12 @@ proc semTypeTraits(c: PContext, n: PNode): PNode = proc semOrd(c: PContext, n: PNode): PNode = result = n - result.typ = makeRangeType(c, firstOrd(n.sons[1].typ), - lastOrd(n.sons[1].typ), n.info) + let parType = n.sons[1].typ + if isOrdinalType(parType) or parType.kind == tySet: + result.typ = makeRangeType(c, firstOrd(parType), lastOrd(parType), n.info) + else: + localError(n.info, errOrdinalTypeExpected) + result.typ = errorType(c) proc semBindSym(c: PContext, n: PNode): PNode = result = copyNode(n) @@ -130,7 +137,7 @@ proc semBindSym(c: PContext, n: PNode): PNode = return errorNode(c, n) let id = newIdentNode(getIdent(sl.strVal), n.info) - let s = qualifiedLookUp(c, id) + let s = qualifiedLookUp(c, id, {checkUndeclared}) if s != nil: # we need to mark all symbols: var sc = symChoice(c, id, s, TSymChoiceRule(isMixin.intVal)) @@ -203,7 +210,9 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, result = n.sons[1] else: result = newNodeIT(nkCall, n.info, getSysType(tyInt)) - result.add newSymNode(getSysMagic("-", mSubI), n.info) + let subi = getSysMagic("-", mSubI) + #echo "got ", typeToString(subi.typ) + result.add newSymNode(subi, n.info) result.add lenExprB result.add n.sons[1] of mPlugin: diff --git a/compiler/semparallel.nim b/compiler/semparallel.nim index b04ba4657..90c1a315a 100644 --- a/compiler/semparallel.nim +++ b/compiler/semparallel.nim @@ -74,8 +74,6 @@ type currentSpawnId: int inLoop: int -let opSlice = createMagic("slice", mSlice) - proc initAnalysisCtx(): AnalysisCtx = result.locals = @[] result.slices = @[] @@ -123,7 +121,7 @@ proc checkLocal(c: AnalysisCtx; n: PNode) = else: for i in 0 .. <n.safeLen: checkLocal(c, n.sons[i]) -template `?`(x): expr = x.renderTree +template `?`(x): untyped = x.renderTree proc checkLe(c: AnalysisCtx; a, b: PNode) = case proveLe(c.guards, a, b) @@ -262,7 +260,7 @@ proc min(a, b: PNode): PNode = proc fromSystem(op: PSym): bool = sfSystemModule in getModule(op).flags -template pushSpawnId(c: expr, body: stmt) {.immediate, dirty.} = +template pushSpawnId(c, body) {.dirty.} = inc c.spawns let oldSpawnId = c.currentSpawnId c.currentSpawnId = c.spawns @@ -399,7 +397,9 @@ proc transformSlices(n: PNode): PNode = let op = n[0].sym if op.name.s == "[]" and op.fromSystem: result = copyNode(n) - result.add opSlice.newSymNode + let opSlice = newSymNode(createMagic("slice", mSlice)) + opSlice.typ = getSysType(tyInt) + result.add opSlice result.add n[1] let slice = n[2].skipStmtList result.add slice[1] diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index c3a9e01a0..b12ab5e96 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -151,7 +151,7 @@ proc guardDotAccess(a: PEffects; n: PNode) = guardGlobal(a, n, g) proc makeVolatile(a: PEffects; s: PSym) {.inline.} = - template compileToCpp(a): expr = + template compileToCpp(a): untyped = gCmd == cmdCompileToCpp or sfCompileToCpp in getModule(a.owner).flags if a.inTryStmt > 0 and not compileToCpp(a): incl(s.flags, sfVolatile) @@ -459,7 +459,7 @@ proc documentRaises*(n: PNode) = if p4 != nil: n.sons[pragmasPos].add p4 if p5 != nil: n.sons[pragmasPos].add p5 -template notGcSafe(t): expr = {tfGcSafe, tfNoSideEffect} * t.flags == {} +template notGcSafe(t): untyped = {tfGcSafe, tfNoSideEffect} * t.flags == {} proc importedFromC(n: PNode): bool = # when imported from C, we assume GC-safety. @@ -505,7 +505,7 @@ proc notNilCheck(tracked: PEffects, n: PNode, paramType: PType) = # addr(x[]) can't be proven, but addr(x) can: if not containsNode(n, {nkDerefExpr, nkHiddenDeref}): return elif (n.kind == nkSym and n.sym.kind in routineKinds) or - n.kind in procDefs+{nkObjConstr}: + n.kind in procDefs+{nkObjConstr, nkBracket}: # 'p' is not nil obviously: return case impliesNotNil(tracked.guards, n) @@ -532,7 +532,7 @@ proc isOwnedProcVar(n: PNode; owner: PSym): bool = proc trackOperand(tracked: PEffects, n: PNode, paramType: PType) = let a = skipConvAndClosure(n) let op = a.typ - if op != nil and op.kind == tyProc and n.kind != nkNilLit: + if op != nil and op.kind == tyProc and n.skipConv.kind != nkNilLit: internalAssert op.n.sons[0].kind == nkEffectList var effectList = op.n.sons[0] let s = n.skipConv diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 910267e57..5f9989ea2 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -17,7 +17,8 @@ proc semDiscard(c: PContext, n: PNode): PNode = checkSonsLen(n, 1) if n.sons[0].kind != nkEmpty: n.sons[0] = semExprWithType(c, n.sons[0]) - if isEmptyType(n.sons[0].typ): localError(n.info, errInvalidDiscard) + if isEmptyType(n.sons[0].typ) or n.sons[0].typ.kind == tyNone: + localError(n.info, errInvalidDiscard) proc semBreakOrContinue(c: PContext, n: PNode): PNode = result = n @@ -137,7 +138,7 @@ proc fixNilType(n: PNode) = proc discardCheck(c: PContext, result: PNode) = if c.inTypeClass > 0: return - if result.typ != nil and result.typ.kind notin {tyStmt, tyEmpty}: + if result.typ != nil and result.typ.kind notin {tyStmt, tyVoid}: if result.kind == nkNilLit: result.typ = nil message(result.info, warnNilStatement) @@ -303,8 +304,14 @@ proc semTry(c: PContext, n: PNode): PNode = proc fitRemoveHiddenConv(c: PContext, typ: PType, n: PNode): PNode = result = fitNode(c, typ, n) if result.kind in {nkHiddenStdConv, nkHiddenSubConv}: - changeType(result.sons[1], typ, check=true) - result = result.sons[1] + let r1 = result.sons[1] + if r1.kind in {nkCharLit..nkUInt64Lit} and typ.skipTypes(abstractRange).kind in {tyFloat..tyFloat128}: + result = newFloatNode(nkFloatLit, BiggestFloat r1.intVal) + result.info = n.info + result.typ = typ + else: + changeType(r1, typ, check=true) + result = r1 elif not sameType(result.typ, typ): changeType(result, typ, check=false) @@ -336,7 +343,7 @@ proc checkNilable(v: PSym) = {tfNotNil, tfNeedsInit} * v.typ.flags != {}: if v.ast.isNil: message(v.info, warnProveInit, v.name.s) - elif tfNotNil in v.typ.flags and tfNotNil notin v.ast.typ.flags: + elif tfNeedsInit in v.typ.flags and tfNotNil notin v.ast.typ.flags: message(v.info, warnProveInit, v.name.s) include semasgn @@ -410,6 +417,13 @@ proc semUsing(c: PContext; n: PNode): PNode = if a.sons[length-1].kind != nkEmpty: localError(a.info, "'using' sections cannot contain assignments") +proc hasEmpty(typ: PType): bool = + if typ.kind in {tySequence, tyArray, tySet}: + result = typ.lastSon.kind == tyEmpty + elif typ.kind == tyTuple: + for s in typ.sons: + result = result or hasEmpty(s) + proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = var b: PNode result = copyNode(n) @@ -444,10 +458,9 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = #changeType(def.skipConv, typ, check=true) else: typ = skipIntLit(def.typ) - if typ.kind in {tySequence, tyArray, tySet} and - typ.lastSon.kind == tyEmpty: + if hasEmpty(typ): localError(def.info, errCannotInferTypeOfTheLiteral, - ($typ.kind).substr(2).toLower) + ($typ.kind).substr(2).toLowerAscii) else: def = ast.emptyNode if symkind == skLet: localError(a.info, errLetNeedsInit) @@ -535,7 +548,7 @@ proc semConst(c: PContext, n: PNode): PNode = localError(a.sons[2].info, errConstExprExpected) continue if typeAllowed(typ, skConst) != nil and def.kind != nkNilLit: - localError(a.info, errXisNoType, typeToString(typ)) + localError(a.info, "invalid type for const: " & typeToString(typ)) continue v.typ = typ v.ast = def # no need to copy @@ -668,7 +681,7 @@ proc typeSectionLeftSidePass(c: PContext, n: PNode) = let name = a.sons[0] var s: PSym if name.kind == nkDotExpr: - s = qualifiedLookUp(c, name) + s = qualifiedLookUp(c, name, {checkUndeclared, checkModule}) if s.kind != skType or s.typ.skipTypes(abstractPtrs).kind != tyObject or tfPartial notin s.typ.skipTypes(abstractPtrs).flags: localError(name.info, "only .partial objects can be extended") else: @@ -710,7 +723,7 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) = a.sons[1] = s.typ.n s.typ.size = -1 # could not be computed properly # we fill it out later. For magic generics like 'seq', it won't be filled - # so we use tyEmpty instead of nil to not crash for strange conversions + # so we use tyNone instead of nil to not crash for strange conversions # like: mydata.seq rawAddSon(s.typ, newTypeS(tyNone, c)) s.ast = a @@ -778,11 +791,16 @@ proc typeSectionFinalPass(c: PContext, n: PNode) = var s = a.sons[0].sym # compute the type's size and check for illegal recursions: if a.sons[1].kind == nkEmpty: - if a.sons[2].kind in {nkSym, nkIdent, nkAccQuoted}: + var x = a[2] + while x.kind in {nkStmtList, nkStmtListExpr} and x.len > 0: + x = x.lastSon + if x.kind notin {nkObjectTy, nkDistinctTy, nkEnumTy, nkEmpty} and + s.typ.kind notin {tyObject, tyEnum}: # type aliases are hard: - #MessageOut('for type ' + typeToString(s.typ)); - var t = semTypeNode(c, a.sons[2], nil) - if t.kind in {tyObject, tyEnum}: + var t = semTypeNode(c, x, nil) + assert t != nil + if t.kind in {tyObject, tyEnum, tyDistinct}: + assert s.typ != nil assignType(s.typ, t) s.typ.id = t.id # same id checkConstructedType(s.info, s.typ) @@ -924,6 +942,17 @@ proc semProcAnnotation(c: PContext, prc: PNode; pragma(c, result[namePos].sym, result[pragmasPos], validPragmas) return +proc setGenericParamsMisc(c: PContext; n: PNode): PNode = + let orig = n.sons[genericParamsPos] + # we keep the original params around for better error messages, see + # issue https://github.com/nim-lang/Nim/issues/1713 + result = semGenericParamList(c, orig) + if n.sons[miscPos].kind == nkEmpty: + n.sons[miscPos] = newTree(nkBracket, ast.emptyNode, orig) + else: + n.sons[miscPos].sons[1] = orig + n.sons[genericParamsPos] = result + proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode = # XXX semProcAux should be good enough for this now, we will eventually # remove semLambda @@ -942,8 +971,7 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode = openScope(c) var gp: PNode if n.sons[genericParamsPos].kind != nkEmpty: - n.sons[genericParamsPos] = semGenericParamList(c, n.sons[genericParamsPos]) - gp = n.sons[genericParamsPos] + gp = setGenericParamsMisc(c, n) else: gp = newNodeI(nkGenericParams, n.info) @@ -1165,8 +1193,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, openScope(c) var gp: PNode if n.sons[genericParamsPos].kind != nkEmpty: - n.sons[genericParamsPos] = semGenericParamList(c, n.sons[genericParamsPos]) - gp = n.sons[genericParamsPos] + gp = setGenericParamsMisc(c, n) else: gp = newNodeI(nkGenericParams, n.info) # process parameters: @@ -1231,6 +1258,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, pushOwner(s) s.options = gOptions if sfOverriden in s.flags or s.name.s[0] == '=': semOverride(c, s, n) + if s.name.s[0] in {'.', '('}: + if s.name.s in [".", ".()", ".=", "()"] and not experimentalMode(c): + message(n.info, warnDeprecated, "overloaded '.' and '()' operators are now .experimental; " & s.name.s) if n.sons[bodyPos].kind != nkEmpty: # for DLL generation it is annoying to check for sfImportc! if sfBorrow in s.flags: @@ -1245,7 +1275,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, c.p.wasForwarded = proto != nil maybeAddResult(c, s, n) - if sfImportc notin s.flags: + if lfDynamicLib notin s.loc.flags: # no semantic checking for importc: let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos])) # unfortunately we cannot skip this step when in 'system.compiles' @@ -1285,6 +1315,7 @@ proc determineType(c: PContext, s: PSym) = proc semIterator(c: PContext, n: PNode): PNode = # gensym'ed iterator? + let isAnon = n[namePos].kind == nkEmpty if n[namePos].kind == nkSym: # gensym'ed iterators might need to become closure iterators: n[namePos].sym.owner = getCurrOwner() @@ -1294,6 +1325,8 @@ proc semIterator(c: PContext, n: PNode): PNode = var t = s.typ if t.sons[0] == nil and s.typ.callConv != ccClosure: localError(n.info, errXNeedsReturnType, "iterator") + if isAnon and s.typ.callConv == ccInline: + localError(n.info, "inline iterators are not first-class / cannot be assigned to variables") # iterators are either 'inline' or 'closure'; for backwards compatibility, # we require first class iterators to be marked with 'closure' explicitly # -- at least for 0.9.2. @@ -1325,9 +1358,24 @@ proc finishMethod(c: PContext, s: PSym) = proc semMethod(c: PContext, n: PNode): PNode = if not isTopLevel(c): localError(n.info, errXOnlyAtModuleScope, "method") result = semProcAux(c, n, skMethod, methodPragmas) - + # macros can transform methods to nothing: + if namePos >= result.safeLen: return result var s = result.sons[namePos].sym - if not isGenericRoutine(s): + if isGenericRoutine(s): + let tt = s.typ + var foundObj = false + # we start at 1 for now so that tparsecombnum continues to compile. + # XXX Revisit this problem later. + for col in countup(1, sonsLen(tt)-1): + let t = tt.sons[col] + if t != nil and t.kind == tyGenericInvocation: + var x = skipTypes(t.sons[0], {tyVar, tyPtr, tyRef, tyGenericInst, tyGenericInvocation, tyGenericBody}) + if x.kind == tyObject and t.len-1 == result.sons[genericParamsPos].len: + foundObj = true + x.methods.safeAdd((col,s)) + if not foundObj: + message(n.info, warnDeprecated, "generic method not attachable to object type") + else: # why check for the body? bug #2400 has none. Checking for sfForward makes # no sense either. # and result.sons[bodyPos].kind != nkEmpty: @@ -1340,6 +1388,8 @@ proc semConverterDef(c: PContext, n: PNode): PNode = if not isTopLevel(c): localError(n.info, errXOnlyAtModuleScope, "converter") checkSonsLen(n, bodyPos + 1) result = semProcAux(c, n, skConverter, converterPragmas) + # macros can transform converters to nothing: + if namePos >= result.safeLen: return result var s = result.sons[namePos].sym var t = s.typ if t.sons[0] == nil: localError(n.info, errXNeedsReturnType, "converter") @@ -1349,6 +1399,8 @@ proc semConverterDef(c: PContext, n: PNode): PNode = proc semMacroDef(c: PContext, n: PNode): PNode = checkSonsLen(n, bodyPos + 1) result = semProcAux(c, n, skMacro, macroPragmas) + # macros can transform macros to nothing: + if namePos >= result.safeLen: return result var s = result.sons[namePos].sym var t = s.typ if t.sons[0] == nil: localError(n.info, errXNeedsReturnType, "macro") diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index a4498a3ae..20b5071ac 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -51,9 +51,10 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule): PNode = var i = 0 a = initOverloadIter(o, c, n) while a != nil: + if a.kind != skModule: + inc(i) + if i > 1: break a = nextOverloadIter(o, c, n) - inc(i) - if i > 1: break if i <= 1 and r != scForceOpen: # XXX this makes more sense but breaks bootstrapping for now: # (s.kind notin routineKinds or s.magic != mNone): @@ -68,8 +69,9 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule): PNode = result = newNodeIT(kind, n.info, newTypeS(tyNone, c)) a = initOverloadIter(o, c, n) while a != nil: - incl(a.flags, sfUsed) - addSon(result, newSymNode(a, n.info)) + if a.kind != skModule: + incl(a.flags, sfUsed) + addSon(result, newSymNode(a, n.info)) a = nextOverloadIter(o, c, n) proc semBindStmt(c: PContext, n: PNode, toBind: var IntSet): PNode = @@ -80,7 +82,7 @@ proc semBindStmt(c: PContext, n: PNode, toBind: var IntSet): PNode = # the same symbol! # This is however not true anymore for hygienic templates as semantic # processing for them changes the symbol table... - let s = qualifiedLookUp(c, a) + let s = qualifiedLookUp(c, a, {checkUndeclared}) if s != nil: # we need to mark all symbols: let sc = symChoice(c, n, s, scClosed) @@ -577,13 +579,16 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = else: gp = newNodeI(nkGenericParams, n.info) # process parameters: + var allUntyped = true if n.sons[paramsPos].kind != nkEmpty: semParamList(c, n.sons[paramsPos], gp, s) # a template's parameters are not gensym'ed even if that was originally the # case as we determine whether it's a template parameter in the template # body by the absence of the sfGenSym flag: for i in 1 .. s.typ.n.len-1: - s.typ.n.sons[i].sym.flags.excl sfGenSym + let param = s.typ.n.sons[i].sym + param.flags.excl sfGenSym + if param.typ.kind != tyExpr: allUntyped = false if sonsLen(gp) > 0: if n.sons[genericParamsPos].kind == nkEmpty: # we have a list of implicit type parameters: @@ -599,6 +604,7 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = s.typ.n = newNodeI(nkFormalParams, n.info) rawAddSon(s.typ, newTypeS(tyStmt, c)) addSon(s.typ.n, newNodeIT(nkType, n.info, s.typ.sons[0])) + if allUntyped: incl(s.flags, sfAllUntyped) if n.sons[patternPos].kind != nkEmpty: n.sons[patternPos] = semPattern(c, n.sons[patternPos]) var ctx: TemplCtx @@ -628,8 +634,8 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = c.patterns.add(s) proc semPatternBody(c: var TemplCtx, n: PNode): PNode = - template templToExpand(s: expr): expr = - s.kind == skTemplate and (s.typ.len == 1 or sfImmediate in s.flags) + template templToExpand(s: untyped): untyped = + s.kind == skTemplate and (s.typ.len == 1 or sfAllUntyped in s.flags) proc newParam(c: var TemplCtx, n: PNode, s: PSym): PNode = # the param added in the current scope is actually wrong here for diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index ba17cc307..9d00c06ca 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -130,7 +130,7 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType = if n.len < 1: result = newConstraint(c, kind) else: - let isCall = ord(n.kind in nkCallKinds) + let isCall = ord(n.kind in nkCallKinds+{nkBracketExpr}) let n = if n[0].kind == nkBracket: n[0] else: n checkMinSonsLen(n, 1) var base = semTypeNode(c, n.lastSon, nil) @@ -370,7 +370,7 @@ proc semTuple(c: PContext, n: PNode, prev: PType): PType = result.n = newNodeI(nkRecList, n.info) var check = initIntSet() var counter = 0 - for i in countup(0, sonsLen(n) - 1): + for i in countup(ord(n.kind == nkBracketExpr), sonsLen(n) - 1): var a = n.sons[i] if (a.kind != nkIdentDefs): illFormedAst(a) checkMinSonsLen(a, 3) @@ -393,20 +393,24 @@ proc semTuple(c: PContext, n: PNode, prev: PType): PType = addSon(result.n, newSymNode(field)) addSonSkipIntLit(result, typ) if gCmd == cmdPretty: styleCheckDef(a.sons[j].info, field) + if result.n.len == 0: result.n = nil proc semIdentVis(c: PContext, kind: TSymKind, n: PNode, allowed: TSymFlags): PSym = # identifier with visibility if n.kind == nkPostfix: - if sonsLen(n) == 2 and n.sons[0].kind == nkIdent: + if sonsLen(n) == 2: # for gensym'ed identifiers the identifier may already have been # transformed to a symbol and we need to use that here: result = newSymG(kind, n.sons[1], c) - var v = n.sons[0].ident + var v = considerQuotedIdent(n.sons[0]) if sfExported in allowed and v.id == ord(wStar): incl(result.flags, sfExported) else: - localError(n.sons[0].info, errInvalidVisibilityX, v.s) + if not (sfExported in allowed): + localError(n.sons[0].info, errXOnlyAtModuleScope, "export") + else: + localError(n.sons[0].info, errInvalidVisibilityX, renderTree(n[0])) else: illFormedAst(n) else: @@ -751,18 +755,18 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, addDecl(c, s) # XXX: There are codegen errors if this is turned into a nested proc - template liftingWalk(typ: PType, anonFlag = false): expr = + template liftingWalk(typ: PType, anonFlag = false): untyped = liftParamType(c, procKind, genericParams, typ, paramName, info, anonFlag) #proc liftingWalk(paramType: PType, anon = false): PType = var paramTypId = if not anon and paramType.sym != nil: paramType.sym.name else: nil - template maybeLift(typ: PType): expr = + template maybeLift(typ: PType): untyped = let lifted = liftingWalk(typ) (if lifted != nil: lifted else: typ) - template addImplicitGeneric(e: expr): expr = + template addImplicitGeneric(e): untyped = addImplicitGenericImpl(e, paramTypId) case paramType.kind: @@ -942,7 +946,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, if isType: localError(a.info, "':' expected") if kind in {skTemplate, skMacro}: typ = newTypeS(tyExpr, c) - elif skipTypes(typ, {tyGenericInst}).kind == tyEmpty: + elif skipTypes(typ, {tyGenericInst}).kind == tyVoid: continue for j in countup(0, length-3): var arg = newSymG(skParam, a.sons[j], c) @@ -974,7 +978,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, if r != nil: # turn explicit 'void' return type into 'nil' because the rest of the # compiler only checks for 'nil': - if skipTypes(r, {tyGenericInst}).kind != tyEmpty: + if skipTypes(r, {tyGenericInst}).kind != tyVoid: # 'auto' as a return type does not imply a generic: if r.kind == tyAnything: # 'p(): auto' and 'p(): expr' are equivalent, but the rest of the @@ -1035,7 +1039,7 @@ proc semGenericParamInInvocation(c: PContext, n: PNode): PType = proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType = if s.typ == nil: localError(n.info, "cannot instantiate the '$1' $2" % - [s.name.s, ($s.kind).substr(2).toLower]) + [s.name.s, ($s.kind).substr(2).toLowerAscii]) return newOrPrevType(tyError, prev, c) var t = s.typ @@ -1094,10 +1098,14 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType = result = instGenericContainer(c, n.info, result, allowMetaTypes = false) -proc semTypeExpr(c: PContext, n: PNode): PType = +proc semTypeExpr(c: PContext, n: PNode; prev: PType): PType = var n = semExprWithType(c, n, {efDetermineType}) if n.typ.kind == tyTypeDesc: result = n.typ.base + # fix types constructed by macros: + if prev != nil and prev.sym != nil and result.sym.isNil: + result.sym = prev.sym + result.sym.typ = result else: localError(n.info, errTypeExpected, n.renderTree) result = errorType(c) @@ -1177,7 +1185,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = else: localError(n.info, errGenerated, "invalid type") elif n[0].kind notin nkIdentKinds: - result = semTypeExpr(c, n) + result = semTypeExpr(c, n, prev) else: let op = considerQuotedIdent(n.sons[0]) if op.id in {ord(wAnd), ord(wOr)} or op.s == "|": @@ -1218,7 +1226,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = let typExpr = semExprWithType(c, n.sons[1], {efInTypeof}) result = typExpr.typ else: - result = semTypeExpr(c, n) + result = semTypeExpr(c, n, prev) of nkWhenStmt: var whenResult = semWhen(c, n, false) if whenResult.kind == nkStmtList: whenResult.kind = nkStmtListType @@ -1241,6 +1249,19 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result = copyType(result, getCurrOwner(), false) for i in countup(1, n.len - 1): result.rawAddSon(semTypeNode(c, n.sons[i], nil)) + of mDistinct: + result = newOrPrevType(tyDistinct, prev, c) + addSonSkipIntLit(result, semTypeNode(c, n[1], nil)) + of mVar: + result = newOrPrevType(tyVar, prev, c) + var base = semTypeNode(c, n.sons[1], nil) + if base.kind == tyVar: + localError(n.info, errVarVarTypeNotAllowed) + base = base.sons[0] + addSonSkipIntLit(result, base) + of mRef: result = semAnyRef(c, n, tyRef, prev) + of mPtr: result = semAnyRef(c, n, tyPtr, prev) + of mTuple: result = semTuple(c, n, prev) else: result = semGeneric(c, n, s, prev) of nkDotExpr: var typeExpr = semExpr(c, n) @@ -1370,10 +1391,7 @@ proc processMagicType(c: PContext, m: PSym) = setMagicType(m, tyTypeDesc, 0) rawAddSon(m.typ, newTypeS(tyNone, c)) of mVoidType: - setMagicType(m, tyEmpty, 0) - # for historical reasons we conflate 'void' with 'empty' so that '@[]' - # has the type 'seq[void]'. - m.typ.flags.incl tfVoid + setMagicType(m, tyVoid, 0) of mArray: setMagicType(m, tyArray, 0) of mOpenArray: diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 7ff33f918..12620d55d 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -162,7 +162,7 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode = discard of nkSym: result.sym = replaceTypeVarsS(cl, n.sym) - if result.sym.typ.kind == tyEmpty: + if result.sym.typ.kind == tyVoid: # don't add the 'void' field result = newNode(nkRecList, n.info) of nkRecWhen: @@ -289,7 +289,8 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = # but we already raised an error! rawAddSon(result, header.sons[i]) - var newbody = replaceTypeVarsT(cl, lastSon(body)) + let bbody = lastSon body + var newbody = replaceTypeVarsT(cl, bbody) cl.skipTypedesc = oldSkipTypedesc newbody.flags = newbody.flags + (t.flags + body.flags - tfInstClearedFlags) result.flags = result.flags + newbody.flags - tfInstClearedFlags @@ -306,25 +307,31 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = # 'deepCopy' needs to be instantiated for # generics *when the type is constructed*: newbody.deepCopy = cl.c.instTypeBoundOp(cl.c, dc, result, cl.info, - attachedDeepCopy) + attachedDeepCopy, 1) let asgn = newbody.assignment if asgn != nil and sfFromGeneric notin asgn.flags: # '=' needs to be instantiated for generics when the type is constructed: newbody.assignment = cl.c.instTypeBoundOp(cl.c, asgn, result, cl.info, - attachedAsgn) + attachedAsgn, 1) + let methods = skipTypes(bbody, abstractPtrs).methods + for col, meth in items(methods): + # we instantiate the known methods belonging to that type, this causes + # them to be registered and that's enough, so we 'discard' the result. + discard cl.c.instTypeBoundOp(cl.c, meth, result, cl.info, + attachedAsgn, col) proc eraseVoidParams*(t: PType) = # transform '(): void' into '()' because old parts of the compiler really # don't deal with '(): void': - if t.sons[0] != nil and t.sons[0].kind == tyEmpty: + if t.sons[0] != nil and t.sons[0].kind == tyVoid: t.sons[0] = nil for i in 1 .. <t.sonsLen: # don't touch any memory unless necessary - if t.sons[i].kind == tyEmpty: + if t.sons[i].kind == tyVoid: var pos = i for j in i+1 .. <t.sonsLen: - if t.sons[j].kind != tyEmpty: + if t.sons[j].kind != tyVoid: t.sons[pos] = t.sons[j] t.n.sons[pos] = t.n.sons[j] inc pos @@ -504,6 +511,6 @@ proc generateTypeInstance*(p: PContext, pt: TIdTable, info: TLineInfo, popInfoContext() template generateTypeInstance*(p: PContext, pt: TIdTable, arg: PNode, - t: PType): expr = + t: PType): untyped = generateTypeInstance(p, pt, arg.info, t) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 82f878932..ec429968c 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -22,7 +22,7 @@ type TCandidateState* = enum csEmpty, csMatch, csNoMatch - CandidateErrors* = seq[PSym] + CandidateErrors* = seq[(PSym,int)] TCandidate* = object c*: PContext exactMatches*: int # also misused to prefer iters over procs @@ -49,6 +49,7 @@ type # a distrinct type typedescMatched*: bool isNoCall*: bool # misused for generic type instantiations C[T] + mutabilityProblem*: uint8 # tyVar mismatch inheritancePenalty: int # to prefer closest father object type errors*: CandidateErrors # additional clarifications to be displayed to the # user if overload resolution fails @@ -96,8 +97,8 @@ proc initCandidate*(ctx: PContext, c: var TCandidate, callee: PType) = c.calleeSym = nil initIdTable(c.bindings) -proc put(t: var TIdTable, key, val: PType) {.inline.} = - idTablePut(t, key, val.skipIntLit) +proc put(c: var TCandidate, key, val: PType) {.inline.} = + idTablePut(c.bindings, key, val.skipIntLit) proc initCandidate*(ctx: PContext, c: var TCandidate, callee: PSym, binding: PNode, calleeScope = -1) = @@ -129,7 +130,7 @@ proc initCandidate*(ctx: PContext, c: var TCandidate, callee: PSym, bound = makeTypeDesc(ctx, bound) else: bound = bound.skipTypes({tyTypeDesc}) - put(c.bindings, formalTypeParam, bound) + put(c, formalTypeParam, bound) proc newCandidate*(ctx: PContext, callee: PSym, binding: PNode, calleeScope = -1): TCandidate = @@ -220,13 +221,14 @@ proc cmpCandidates*(a, b: TCandidate): int = if result != 0: return result = a.convMatches - b.convMatches if result != 0: return - result = a.calleeScope - b.calleeScope - if result != 0: return # the other way round because of other semantics: result = b.inheritancePenalty - a.inheritancePenalty if result != 0: return # prefer more specialized generic over more general generic: result = complexDisambiguation(a.callee, b.callee) + # only as a last resort, consider scoping: + if result != 0: return + result = a.calleeScope - b.calleeScope proc writeMatches*(c: TCandidate) = writeLine(stdout, "exact matches: " & $c.exactMatches) @@ -242,6 +244,8 @@ proc argTypeToString(arg: PNode; prefer: TPreferedDesc): string = for i in 1 .. <arg.len: result.add(" | ") result.add typeToString(arg[i].typ, prefer) + elif arg.typ == nil: + result = "void" else: result = arg.typ.typeToString(prefer) @@ -253,15 +257,16 @@ proc describeArgs*(c: PContext, n: PNode, startIdx = 1; if n.sons[i].kind == nkExprEqExpr: add(result, renderTree(n.sons[i].sons[0])) add(result, ": ") - if arg.typ.isNil: + if arg.typ.isNil and arg.kind notin {nkStmtList, nkDo}: + # XXX we really need to 'tryExpr' here! arg = c.semOperand(c, n.sons[i].sons[1]) n.sons[i].typ = arg.typ n.sons[i].sons[1] = arg else: - if arg.typ.isNil: + if arg.typ.isNil and arg.kind notin {nkStmtList, nkDo}: arg = c.semOperand(c, n.sons[i]) n.sons[i] = arg - if arg.typ.kind == tyError: return + if arg.typ != nil and arg.typ.kind == tyError: return add(result, argTypeToString(arg, prefer)) if i != sonsLen(n) - 1: add(result, ", ") @@ -360,6 +365,50 @@ proc isObjectSubtype(a, f: PType): int = if t != nil: result = depth +type + SkippedPtr = enum skippedNone, skippedRef, skippedPtr + +proc skipToObject(t: PType; skipped: var SkippedPtr): PType = + var r = t + # we're allowed to skip one level of ptr/ref: + var ptrs = 0 + while r != nil: + case r.kind + of tyGenericInvocation: + r = r.sons[0] + of tyRef: + inc ptrs + skipped = skippedRef + r = r.lastSon + of tyPtr: + inc ptrs + skipped = skippedPtr + r = r.lastSon + of tyGenericBody, tyGenericInst: + r = r.lastSon + else: + break + if r.kind == tyObject and ptrs <= 1: result = r + +proc isGenericSubtype(a, f: PType, d: var int): bool = + assert f.kind in {tyGenericInst, tyGenericInvocation, tyGenericBody} + var askip = skippedNone + var fskip = skippedNone + var t = a.skipToObject(askip) + let r = f.skipToObject(fskip) + if r == nil: return false + var depth = 0 + # XXX sameObjectType can return false here. Need to investigate + # why that is but sameObjectType does way too much work here anyway. + while t != nil and r.sym != t.sym and askip == fskip: + t = t.sons[0] + if t != nil: t = t.skipToObject(askip) + else: break + inc depth + if t != nil and askip == fskip: + d = depth + result = true + proc minRel(a, b: TTypeRelation): TTypeRelation = if a <= b: result = a else: result = b @@ -513,7 +562,7 @@ proc typeRangeRel(f, a: PType): TTypeRelation {.noinline.} = proc matchUserTypeClass*(c: PContext, m: var TCandidate, ff, a: PType): TTypeRelation = var body = ff.skipTypes({tyUserTypeClassInst}) - if c.inTypeClass > 20: + if c.inTypeClass > 4: localError(body.n[3].info, $body.n[3] & " too nested for type matching") return isNone @@ -531,7 +580,7 @@ proc matchUserTypeClass*(c: PContext, m: var TCandidate, typ = ff.sons[i] param: PSym - template paramSym(kind): expr = + template paramSym(kind): untyped = newSym(kind, typeParamName, body.sym, body.sym.info) case typ.kind @@ -598,6 +647,10 @@ proc tryResolvingStaticExpr(c: var TCandidate, n: PNode): PNode = let instantiated = replaceTypesInBody(c.c, c.bindings, n, nil) result = c.c.semExpr(c.c, instantiated) +template subtypeCheck() = + if result <= isSubrange and f.lastSon.skipTypes(abstractInst).kind in {tyRef, tyPtr, tyVar}: + result = isNone + proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = # typeRel can be used to establish various relationships between types: # @@ -619,7 +672,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = assert(f != nil) if f.kind == tyExpr: - if aOrig != nil: put(c.bindings, f, aOrig) + if aOrig != nil: put(c, f, aOrig) return isGeneric assert(aOrig != nil) @@ -640,10 +693,10 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = template bindingRet(res) = if doBind: let bound = aOrig.skipTypes({tyRange}).skipIntLit - if doBind: put(c.bindings, f, bound) + put(c, f, bound) return res - template considerPreviousT(body: stmt) {.immediate.} = + template considerPreviousT(body: untyped) = var prev = PType(idTableGet(c.bindings, f)) if prev == nil: body else: return typeRel(c, prev, a) @@ -737,6 +790,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = of tyVar: if aOrig.kind == tyVar: result = typeRel(c, f.base, aOrig.base) else: result = typeRel(c, f.base, aOrig) + subtypeCheck() of tyArray, tyArrayConstr: # tyArrayConstr cannot happen really, but # we wanna be safe here @@ -746,7 +800,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = if fRange.kind == tyGenericParam: var prev = PType(idTableGet(c.bindings, fRange)) if prev == nil: - put(c.bindings, fRange, a.sons[0]) + put(c, fRange, a.sons[0]) fRange = a else: fRange = prev @@ -764,7 +818,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = # we must correct for the off-by-one discrepancy between # ranges and static params: replacementT.n = newIntNode(nkIntLit, inputUpperBound + 1) - put(c.bindings, rangeStaticT, replacementT) + put(c, rangeStaticT, replacementT) return isGeneric let len = tryResolvingStaticExpr(c, fRange.n[1]) @@ -851,6 +905,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = if sameDistinctTypes(f, a): result = isEqual elif f.base.kind == tyAnything: result = isGeneric elif c.coerceDistincts: result = typeRel(c, f.base, a) + elif a.kind == tyNil and f.base.kind in NilableTypes: + result = f.allowsNil elif c.coerceDistincts: result = typeRel(c, f.base, a) of tySet: if a.kind == tySet: @@ -867,6 +923,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = for i in 0..f.len-2: if typeRel(c, f.sons[i], a.sons[i]) == isNone: return isNone result = typeRel(c, f.lastSon, a.lastSon) + subtypeCheck() if result <= isConvertible: result = isNone elif tfNotNil in f.flags and tfNotNil notin a.flags: result = isNilConversion @@ -921,8 +978,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = result = isConvertible else: discard - of tyEmpty: - if a.kind == tyEmpty: result = isEqual + of tyEmpty, tyVoid: + if a.kind == f.kind: result = isEqual of tyGenericInst: result = typeRel(c, lastSon(f), a) @@ -932,21 +989,26 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = if a.kind == tyGenericInst and a.sons[0] == f: bindingRet isGeneric let ff = lastSon(f) - if ff != nil: result = typeRel(c, ff, a) + if ff != nil: + result = typeRel(c, ff, a) of tyGenericInvocation: var x = a.skipGenericAlias + var depth = 0 if x.kind == tyGenericInvocation or f.sons[0].kind != tyGenericBody: #InternalError("typeRel: tyGenericInvocation -> tyGenericInvocation") # simply no match for now: discard elif x.kind == tyGenericInst and - (f.sons[0] == x.sons[0]) and + ((f.sons[0] == x.sons[0]) or isGenericSubtype(x, f, depth)) and (sonsLen(x) - 1 == sonsLen(f)): for i in countup(1, sonsLen(f) - 1): if x.sons[i].kind == tyGenericParam: internalError("wrong instantiated type!") - elif typeRel(c, f.sons[i], x.sons[i]) <= isSubtype: return + elif typeRel(c, f.sons[i], x.sons[i]) <= isSubtype: + # Workaround for regression #4589 + if f.sons[i].kind != tyTypeDesc: return + c.inheritancePenalty += depth result = isGeneric else: let genericBody = f.sons[0] @@ -963,13 +1025,13 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = # # we steal the generic parameters from the tyGenericBody: for i in countup(1, sonsLen(f) - 1): - var x = PType(idTableGet(c.bindings, genericBody.sons[i-1])) + let x = PType(idTableGet(c.bindings, genericBody.sons[i-1])) if x == nil: discard "maybe fine (for eg. a==tyNil)" elif x.kind in {tyGenericInvocation, tyGenericParam}: internalError("wrong instantiated type!") else: - put(c.bindings, f.sons[i], x) + put(c, f.sons[i], x) of tyAnd: considerPreviousT: @@ -979,6 +1041,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = if x < isSubtype: return isNone # 'and' implies minimum matching result: if x < result: result = x + if result > isGeneric: result = isGeneric bindingRet result of tyOr: @@ -989,6 +1052,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = # 'or' implies maximum matching result: if x > result: result = x if result >= isSubtype: + if result > isGeneric: result = isGeneric bindingRet result else: result = isNone @@ -1005,7 +1069,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = considerPreviousT: var concrete = concreteType(c, a) if concrete != nil and doBind: - put(c.bindings, f, concrete) + put(c, f, concrete) return isGeneric of tyBuiltInTypeClass: @@ -1013,7 +1077,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = let targetKind = f.sons[0].kind if targetKind == a.skipTypes({tyRange, tyGenericInst}).kind or (targetKind in {tyProc, tyPointer} and a.kind == tyNil): - put(c.bindings, f, a) + put(c, f, a) return isGeneric else: return isNone @@ -1022,7 +1086,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = considerPreviousT: result = matchUserTypeClass(c.c, c, f, aOrig) if result == isGeneric: - put(c.bindings, f, a) + put(c, f, a) of tyCompositeTypeClass: considerPreviousT: @@ -1038,7 +1102,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = else: result = typeRel(c, rootf.lastSon, a) if result != isNone: - put(c.bindings, f, a) + put(c, f, a) result = isGeneric of tyGenericParam: var x = PType(idTableGet(c.bindings, f)) @@ -1072,7 +1136,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = if doBind and result notin {isNone, isGeneric}: let concrete = concreteType(c, a) if concrete == nil: return isNone - put(c.bindings, f, concrete) + put(c, f, concrete) else: result = isGeneric @@ -1086,7 +1150,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = if concrete == nil: return isNone if doBind: - put(c.bindings, f, concrete) + put(c, f, concrete) elif result > isGeneric: result = isGeneric elif a.kind == tyEmpty: @@ -1105,7 +1169,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = if result != isNone and f.n != nil: if not exprStructuralEquivalent(f.n, aOrig.n): result = isNone - if result != isNone: put(c.bindings, f, aOrig) + if result != isNone: put(c, f, aOrig) else: result = isNone elif prev.kind == tyStatic: @@ -1133,7 +1197,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = result = typeRel(c, f.base, a.base) if result != isNone: - put(c.bindings, f, a) + put(c, f, a) else: if tfUnresolved in f.flags: result = typeRel(c, prev.base, a) @@ -1151,7 +1215,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = of tyStmt: if aOrig != nil and tfOldSchoolExprStmt notin f.flags: - put(c.bindings, f, aOrig) + put(c, f, aOrig) result = isGeneric of tyProxy: @@ -1256,6 +1320,13 @@ proc localConvMatch(c: PContext, m: var TCandidate, f, a: PType, if r == isGeneric: result.typ = getInstantiatedType(c, arg, m, base(f)) m.baseTypeMatch = true + # bug #4545: allow the call to go through a 'var T': + let vt = result.sons[0].typ.sons[1] + if vt.kind == tyVar: + let x = result.sons[1] + let va = newNodeIT(nkHiddenAddr, x.info, vt) + va.add x + result.sons[1] = va proc incMatches(m: var TCandidate; r: TTypeRelation; convMatch = 1) = case r @@ -1399,7 +1470,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType, if skipTypes(f, abstractVar-{tyTypeDesc}).kind in {tyTuple}: result = implicitConv(nkHiddenSubConv, f, arg, m, c) of isNone: - # do not do this in ``typeRel`` as it then can't infere T in ``ref T``: + # do not do this in ``typeRel`` as it then can't infer T in ``ref T``: if a.kind in {tyProxy, tyUnknown}: inc(m.genericMatches) m.fauxMatch = a.kind @@ -1420,6 +1491,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType, m.baseTypeMatch = true else: result = userConvMatch(c, m, base(f), a, arg) + if result != nil: m.baseTypeMatch = true proc paramTypesMatch*(m: var TCandidate, f, a: PType, arg, argOrig: PNode): PNode = @@ -1532,13 +1604,13 @@ proc incrIndexType(t: PType) = assert t.kind == tyArrayConstr inc t.sons[0].n.sons[1].intVal -template isVarargsUntyped(x): expr = +template isVarargsUntyped(x): untyped = x.kind == tyVarargs and x.sons[0].kind == tyExpr and tfOldSchoolExprStmt notin x.sons[0].flags proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var IntSet) = - template checkConstraint(n: expr) {.immediate, dirty.} = + template checkConstraint(n: untyped) {.dirty.} = if not formal.constraint.isNil: if matchNodeKinds(formal.constraint, n): # better match over other routines with no such restriction: @@ -1549,6 +1621,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, if formal.typ.kind == tyVar: if not n.isLValue: m.state = csNoMatch + m.mutabilityProblem = uint8(f-1) return var @@ -1568,6 +1641,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, while a < n.len: if a >= formalLen-1 and formal != nil and formal.typ.isVarargsUntyped: + incl(marker, formal.position) if container.isNil: container = newNodeIT(nkBracket, n.sons[a].info, arrayConstr(c, n.info)) setSon(m.call, formal.position + 1, container) @@ -1630,6 +1704,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, # beware of the side-effects in 'prepareOperand'! So only do it for # varargs matching. See tests/metatype/tstatic_overloading. m.baseTypeMatch = false + incl(marker, formal.position) n.sons[a] = prepareOperand(c, formal.typ, n.sons[a]) var arg = paramTypesMatch(m, formal.typ, n.sons[a].typ, n.sons[a], nOrig.sons[a]) @@ -1653,30 +1728,39 @@ proc matchesAux(c: PContext, n, nOrig: PNode, when false: localError(n.sons[a].info, errCannotBindXTwice, formal.name.s) m.state = csNoMatch return - m.baseTypeMatch = false - n.sons[a] = prepareOperand(c, formal.typ, n.sons[a]) - var arg = paramTypesMatch(m, formal.typ, n.sons[a].typ, - n.sons[a], nOrig.sons[a]) - if arg == nil: - m.state = csNoMatch - return - if m.baseTypeMatch: - #assert(container == nil) + + if formal.typ.isVarargsUntyped: if container.isNil: - container = newNodeIT(nkBracket, n.sons[a].info, arrayConstr(c, arg)) + container = newNodeIT(nkBracket, n.sons[a].info, arrayConstr(c, n.info)) + setSon(m.call, formal.position + 1, container) else: incrIndexType(container.typ) - addSon(container, arg) - setSon(m.call, formal.position + 1, - implicitConv(nkHiddenStdConv, formal.typ, container, m, c)) - #if f != formalLen - 1: container = nil - - # pick the formal from the end, so that 'x, y, varargs, z' works: - f = max(f, formalLen - n.len + a + 1) + addSon(container, n.sons[a]) else: - setSon(m.call, formal.position + 1, arg) - inc(f) - container = nil + m.baseTypeMatch = false + n.sons[a] = prepareOperand(c, formal.typ, n.sons[a]) + var arg = paramTypesMatch(m, formal.typ, n.sons[a].typ, + n.sons[a], nOrig.sons[a]) + if arg == nil: + m.state = csNoMatch + return + if m.baseTypeMatch: + #assert(container == nil) + if container.isNil: + container = newNodeIT(nkBracket, n.sons[a].info, arrayConstr(c, arg)) + else: + incrIndexType(container.typ) + addSon(container, arg) + setSon(m.call, formal.position + 1, + implicitConv(nkHiddenStdConv, formal.typ, container, m, c)) + #if f != formalLen - 1: container = nil + + # pick the formal from the end, so that 'x, y, varargs, z' works: + f = max(f, formalLen - n.len + a + 1) + else: + setSon(m.call, formal.position + 1, arg) + inc(f) + container = nil checkConstraint(n.sons[a]) inc(a) @@ -1707,15 +1791,18 @@ proc matches*(c: PContext, n, nOrig: PNode, m: var TCandidate) = if formal.ast == nil: if formal.typ.kind == tyVarargs: var container = newNodeIT(nkBracket, n.info, arrayConstr(c, n.info)) - addSon(m.call, implicitConv(nkHiddenStdConv, formal.typ, - container, m, c)) + setSon(m.call, formal.position + 1, + implicitConv(nkHiddenStdConv, formal.typ, container, m, c)) else: # no default value m.state = csNoMatch break else: # use default value: - setSon(m.call, formal.position + 1, copyTree(formal.ast)) + var def = copyTree(formal.ast) + if def.kind == nkNilLit: + def = implicitConv(nkHiddenStdConv, formal.typ, def, m, c) + setSon(m.call, formal.position + 1, def) inc(f) proc argtypeMatches*(c: PContext, f, a: PType): bool = @@ -1728,16 +1815,19 @@ proc argtypeMatches*(c: PContext, f, a: PType): bool = result = res != nil proc instTypeBoundOp*(c: PContext; dc: PSym; t: PType; info: TLineInfo; - op: TTypeAttachedOp): PSym {.procvar.} = + op: TTypeAttachedOp; col: int): PSym {.procvar.} = var m: TCandidate initCandidate(c, m, dc.typ) - var f = dc.typ.sons[1] + if col >= dc.typ.len: + localError(info, errGenerated, "cannot instantiate '" & dc.name.s & "'") + return nil + var f = dc.typ.sons[col] if op == attachedDeepCopy: if f.kind in {tyRef, tyPtr}: f = f.lastSon else: if f.kind == tyVar: f = f.lastSon if typeRel(m, f, t) == isNone: - localError(info, errGenerated, "cannot instantiate 'deepCopy'") + localError(info, errGenerated, "cannot instantiate '" & dc.name.s & "'") else: result = c.semGenerateInstance(c, dc, m.bindings, info) assert sfFromGeneric in result.flags @@ -1745,7 +1835,7 @@ proc instTypeBoundOp*(c: PContext; dc: PSym; t: PType; info: TLineInfo; include suggest when not declared(tests): - template tests(s: stmt) {.immediate.} = discard + template tests(s: untyped) = discard tests: var dummyOwner = newSym(skModule, getIdent("test_module"), nil, UnknownLineInfo()) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index bcab6b04a..c127b968c 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -138,7 +138,7 @@ proc suggestField(c: PContext, s: PSym, outputs: var int) = suggestResult(symToSuggest(s, isLocal=true, $ideSug, 100)) inc outputs -template wholeSymTab(cond, section: expr) {.immediate.} = +template wholeSymTab(cond, section: untyped) = var isLocal = true for scope in walkScopes(c.currentScope): if scope == c.topLevelScope: isLocal = false @@ -393,6 +393,8 @@ proc suggestSym*(info: TLineInfo; s: PSym; isDecl=true) {.inline.} = proc markUsed(info: TLineInfo; s: PSym) = incl(s.flags, sfUsed) + if s.kind == skEnumField and s.owner != nil: + incl(s.owner.flags, sfUsed) if {sfDeprecated, sfError} * s.flags != {}: if sfDeprecated in s.flags: message(info, warnDeprecated, s.name.s) if sfError in s.flags: localError(info, errWrongSymbolX, s.name.s) diff --git a/compiler/transf.nim b/compiler/transf.nim index a4a15ea4a..fc400c524 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -95,7 +95,7 @@ proc getCurrOwner(c: PTransf): PSym = proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode = let r = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info) - r.typ = skipTypes(typ, {tyGenericInst}) + r.typ = typ #skipTypes(typ, {tyGenericInst}) incl(r.flags, sfFromGeneric) let owner = getCurrOwner(c) if owner.isIterator and not c.tooEarly: @@ -204,14 +204,8 @@ proc transformConstSection(c: PTransf, v: PNode): PTransNode = if it.kind != nkConstDef: internalError(it.info, "transformConstSection") if it.sons[0].kind != nkSym: internalError(it.info, "transformConstSection") - if sfFakeConst in it[0].sym.flags: - var b = newNodeI(nkConstDef, it.info) - addSon(b, it[0]) - addSon(b, ast.emptyNode) # no type description - addSon(b, transform(c, it[2]).PNode) - result[i] = PTransNode(b) - else: - result[i] = PTransNode(it) + + result[i] = PTransNode(it) proc hasContinue(n: PNode): bool = case n.kind @@ -414,8 +408,8 @@ proc transformConv(c: PTransf, n: PNode): PTransNode = result = newTransNode(nkChckRange, n, 3) dest = skipTypes(n.typ, abstractVar) result[0] = transform(c, n.sons[1]) - result[1] = newIntTypeNode(nkIntLit, firstOrd(dest), source).PTransNode - result[2] = newIntTypeNode(nkIntLit, lastOrd(dest), source).PTransNode + result[1] = newIntTypeNode(nkIntLit, firstOrd(dest), dest).PTransNode + result[2] = newIntTypeNode(nkIntLit, lastOrd(dest), dest).PTransNode of tyFloat..tyFloat128: # XXX int64 -> float conversion? if skipTypes(n.typ, abstractVar).kind == tyRange: @@ -480,12 +474,14 @@ proc transformConv(c: PTransf, n: PNode): PTransNode = type TPutArgInto = enum - paDirectMapping, paFastAsgn, paVarAsgn + paDirectMapping, paFastAsgn, paVarAsgn, paComplexOpenarray proc putArgInto(arg: PNode, formal: PType): TPutArgInto = # This analyses how to treat the mapping "formal <-> arg" in an # inline context. if skipTypes(formal, abstractInst).kind in {tyOpenArray, tyVarargs}: + if arg.kind == nkStmtListExpr: + return paComplexOpenarray return paDirectMapping # XXX really correct? # what if ``arg`` has side-effects? case arg.kind @@ -575,6 +571,14 @@ proc transformFor(c: PTransf, n: PNode): PTransNode = assert(skipTypes(formal.typ, abstractInst).kind == tyVar) idNodeTablePut(newC.mapping, formal, arg) # XXX BUG still not correct if the arg has a side effect! + of paComplexOpenarray: + let typ = newType(tySequence, formal.owner) + addSonSkipIntLit(typ, formal.typ.sons[0]) + var temp = newTemp(c, typ, formal.info) + addVar(v, temp) + add(stmtList, newAsgnStmt(c, temp, arg.PTransNode)) + idNodeTablePut(newC.mapping, formal, temp) + var body = iter.getBody.copyTree pushInfoContext(n.info) # XXX optimize this somehow. But the check "c.inlining" is not correct: diff --git a/compiler/trees.nim b/compiler/trees.nim index 659df334b..fdd88c348 100644 --- a/compiler/trees.nim +++ b/compiler/trees.nim @@ -103,14 +103,16 @@ proc getMagic*(op: PNode): TMagic = else: result = mNone else: result = mNone -proc treeToSym*(t: PNode): PSym = - result = t.sym - proc isConstExpr*(n: PNode): bool = result = (n.kind in {nkCharLit..nkInt64Lit, nkStrLit..nkTripleStrLit, nkFloatLit..nkFloat64Lit, nkNilLit}) or (nfAllConst in n.flags) +proc isCaseObj*(n: PNode): bool = + if n.kind == nkRecCase: return true + for i in 0..<safeLen(n): + if n[i].isCaseObj: return true + proc isDeepConstExpr*(n: PNode): bool = case n.kind of nkCharLit..nkInt64Lit, nkStrLit..nkTripleStrLit, @@ -119,11 +121,14 @@ proc isDeepConstExpr*(n: PNode): bool = of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv: result = isDeepConstExpr(n.sons[1]) of nkCurly, nkBracket, nkPar, nkObjConstr, nkClosure: - for i in 0 .. <n.len: + for i in ord(n.kind == nkObjConstr) .. <n.len: if not isDeepConstExpr(n.sons[i]): return false - # XXX once constant objects are supported by the codegen this needs to be - # weakened: - result = n.typ.isNil or n.typ.skipTypes({tyGenericInst, tyDistinct}).kind != tyObject + if n.typ.isNil: result = true + else: + let t = n.typ.skipTypes({tyGenericInst, tyDistinct}) + if t.kind in {tyRef, tyPtr}: return false + if t.kind != tyObject or not isCaseObj(t.n): + result = true else: discard proc flattenTreeAux(d, a: PNode, op: TMagic) = diff --git a/compiler/types.nim b/compiler/types.nim index 9aa991086..312e36ae5 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -48,6 +48,8 @@ proc isOrdinalType*(t: PType): bool proc enumHasHoles*(t: PType): bool const + # TODO: Remove tyTypeDesc from each abstractX and (where necessary) + # replace with typedescX abstractPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyDistinct, tyOrdinal, tyConst, tyMutable, tyTypeDesc} abstractVar* = {tyVar, tyGenericInst, tyDistinct, tyOrdinal, @@ -61,6 +63,7 @@ const skipPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyConst, tyMutable, tyTypeDesc} + # typedescX is used if we're sure tyTypeDesc should be included (or skipped) typedescPtrs* = abstractPtrs + {tyTypeDesc} typedescInst* = abstractInst + {tyTypeDesc} @@ -148,10 +151,12 @@ proc skipGeneric(t: PType): PType = proc isOrdinalType(t: PType): bool = assert(t != nil) - # caution: uint, uint64 are no ordinal types! - result = t.kind in {tyChar,tyInt..tyInt64,tyUInt8..tyUInt32,tyBool,tyEnum} or - (t.kind in {tyRange, tyOrdinal, tyConst, tyMutable, tyGenericInst}) and - isOrdinalType(t.sons[0]) + const + # caution: uint, uint64 are no ordinal types! + baseKinds = {tyChar,tyInt..tyInt64,tyUInt8..tyUInt32,tyBool,tyEnum} + parentKinds = {tyRange, tyOrdinal, tyConst, tyMutable, tyGenericInst, + tyDistinct} + t.kind in baseKinds or (t.kind in parentKinds and isOrdinalType(t.sons[0])) proc enumHasHoles(t: PType): bool = var b = t @@ -407,7 +412,8 @@ const "!", "varargs[$1]", "iter[$1]", "Error Type", "BuiltInTypeClass", "UserTypeClass", "UserTypeClassInst", "CompositeTypeClass", - "and", "or", "not", "any", "static", "TypeFromExpr", "FieldAccessor"] + "and", "or", "not", "any", "static", "TypeFromExpr", "FieldAccessor", + "void"] const preferToResolveSymbols = {preferName, preferModuleInfo, preferGenericArg} @@ -540,7 +546,9 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = else: result.add typeToString(t.sons[0]) of tyRange: - result = "range " & rangeToStr(t.n) + result = "range " + if t.n != nil and t.n.kind == nkRange: + result.add rangeToStr(t.n) if prefer != preferExported: result.add("(" & typeToString(t.sons[0]) & ")") of tyProc: @@ -727,6 +735,7 @@ proc equalParam(a, b: PSym): TParamsEquality = result = paramsNotEqual proc sameConstraints(a, b: PNode): bool = + if isNil(a) and isNil(b): return true internalAssert a.len == b.len for i in 1 .. <a.len: if not exprStructuralEquivalent(a[i].sym.constraint, @@ -799,8 +808,10 @@ proc sameTuple(a, b: PType, c: var TSameTypeClosure): bool = result = x.name.id == y.name.id if not result: break else: internalError(a.n.info, "sameTuple") + elif a.n != b.n and (a.n == nil or b.n == nil) and IgnoreTupleFields notin c.flags: + result = false -template ifFastObjectTypeCheckFailed(a, b: PType, body: stmt) {.immediate.} = +template ifFastObjectTypeCheckFailed(a, b: PType, body: untyped) = if tfFromGeneric notin a.flags + b.flags: # fast case: id comparison suffices: result = a.id == b.id @@ -908,7 +919,8 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = while a.kind == tyDistinct: a = a.sons[0] if a.kind != b.kind: return false - if x.kind == tyGenericInst: + # this is required by tunique_type but makes no sense really: + if x.kind == tyGenericInst and IgnoreTupleFields notin c.flags: let lhs = x.skipGenericAlias rhs = y.skipGenericAlias @@ -922,7 +934,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = case a.kind of tyEmpty, tyChar, tyBool, tyNil, tyPointer, tyString, tyCString, - tyInt..tyBigNum, tyStmt, tyExpr: + tyInt..tyBigNum, tyStmt, tyExpr, tyVoid: result = sameFlags(a, b) of tyStatic, tyFromExpr: result = exprStructuralEquivalent(a.n, b.n) and sameFlags(a, b) @@ -1108,7 +1120,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = nil of tyExpr, tyStmt, tyStatic: if kind notin {skParam, skResult}: result = t - of tyEmpty: + of tyVoid: if taField notin flags: result = t of tyTypeClasses: if not (tfGenericTypeParam in t.flags or taField notin flags): result = t @@ -1153,7 +1165,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, if result != nil: break if result.isNil and t.n != nil: result = typeAllowedNode(marker, t.n, kind, flags) - of tyProxy: + of tyProxy, tyEmpty: # for now same as error node; we say it's a valid type as it should # prevent cascading errors: result = nil @@ -1313,6 +1325,9 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt = of tyTypeDesc: result = computeSizeAux(typ.base, a) of tyForward: return szIllegalRecursion + of tyStatic: + if typ.n != nil: result = computeSizeAux(lastSon(typ), a) + else: result = szUnknownSize else: #internalError("computeSizeAux()") result = szUnknownSize @@ -1448,6 +1463,18 @@ proc skipConv*(n: PNode): PNode = result = n.sons[1] else: discard +proc skipHidden*(n: PNode): PNode = + result = n + while true: + case result.kind + of nkHiddenStdConv, nkHiddenSubConv: + if result.sons[1].typ.classify == result.typ.classify: + result = result.sons[1] + else: break + of nkHiddenDeref, nkHiddenAddr: + result = result.sons[0] + else: break + proc skipConvTakeType*(n: PNode): PNode = result = n.skipConv result.typ = n.typ @@ -1493,3 +1520,12 @@ proc skipHiddenSubConv*(n: PNode): PNode = result.typ = dest else: result = n + +proc typeMismatch*(n: PNode, formal, actual: PType) = + if formal.kind != tyError and actual.kind != tyError: + let named = typeToString(formal) + let desc = typeToString(formal, preferDesc) + let x = if named == desc: named else: named & " = " & desc + localError(n.info, errGenerated, msgKindToString(errTypeMismatch) & + typeToString(actual) & ") " & + `%`(msgKindToString(errButExpectedX), [x])) diff --git a/compiler/vm.nim b/compiler/vm.nim index 7220e1b8e..5accf628e 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -10,7 +10,9 @@ ## This file implements the new evaluation engine for Nim code. ## An instruction is 1-3 int32s in memory, it is a register based VM. -const debugEchoCode = false +const + debugEchoCode = false + traceCode = debugEchoCode import ast except getstr @@ -90,38 +92,38 @@ when not defined(nimComputedGoto): proc myreset(n: var TFullReg) = reset(n) -template ensureKind(k: expr) {.immediate, dirty.} = +template ensureKind(k: untyped) {.dirty.} = if regs[ra].kind != k: myreset(regs[ra]) regs[ra].kind = k -template decodeB(k: expr) {.immediate, dirty.} = +template decodeB(k: untyped) {.dirty.} = let rb = instr.regB ensureKind(k) -template decodeBC(k: expr) {.immediate, dirty.} = +template decodeBC(k: untyped) {.dirty.} = let rb = instr.regB let rc = instr.regC ensureKind(k) -template declBC() {.immediate, dirty.} = +template declBC() {.dirty.} = let rb = instr.regB let rc = instr.regC -template decodeBImm(k: expr) {.immediate, dirty.} = +template decodeBImm(k: untyped) {.dirty.} = let rb = instr.regB let imm = instr.regC - byteExcess ensureKind(k) -template decodeBx(k: expr) {.immediate, dirty.} = +template decodeBx(k: untyped) {.dirty.} = let rbx = instr.regBx - wordExcess ensureKind(k) -template move(a, b: expr) {.immediate, dirty.} = system.shallowCopy(a, b) +template move(a, b: untyped) {.dirty.} = system.shallowCopy(a, b) # XXX fix minor 'shallowCopy' overloading bug in compiler proc createStrKeepNode(x: var TFullReg; keepNode=true) = - if x.node.isNil: + if x.node.isNil or not keepNode: x.node = newNode(nkStrLit) elif x.node.kind == nkNilLit and keepNode: when defined(useNodeIds): @@ -160,7 +162,7 @@ proc moveConst(x: var TFullReg, y: TFullReg) = # this seems to be the best way to model the reference semantics # of system.NimNode: -template asgnRef(x, y: expr) = moveConst(x, y) +template asgnRef(x, y: untyped) = moveConst(x, y) proc copyValue(src: PNode): PNode = if src == nil or nfIsRef in src.flags: @@ -231,7 +233,7 @@ proc regToNode(x: TFullReg): PNode = of rkRegisterAddr: result = regToNode(x.regAddr[]) of rkNodeAddr: result = x.nodeAddr[] -template getstr(a: expr): expr = +template getstr(a: untyped): untyped = (if a.kind == rkNode: a.node.strVal else: $chr(int(a.intVal))) proc pushSafePoint(f: PStackFrame; pc: int) = @@ -404,7 +406,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let instr = c.code[pc] let ra = instr.regA #if c.traceActive: - #echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra, " rb ", instr.regB, " rc ", instr.regC + when traceCode: + echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra, " rb ", instr.regB, " rc ", instr.regC # message(c.debug[pc], warnUser, "Trace") case instr.opcode @@ -542,7 +545,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = if regs[rb].node.kind == nkRefTy: regs[ra].node = regs[rb].node.sons[0] else: - stackTrace(c, tos, pc, errGenerated, "limited VM support for 'ref'") + stackTrace(c, tos, pc, errGenerated, "limited VM support for pointers") else: stackTrace(c, tos, pc, errNilAccess) of opcWrDeref: @@ -1182,19 +1185,35 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of opcNGetType: let rb = instr.regB let rc = instr.regC - if rc == 0: + case rc: + of 0: + # getType opcode: ensureKind(rkNode) if regs[rb].kind == rkNode and regs[rb].node.typ != nil: regs[ra].node = opMapTypeToAst(regs[rb].node.typ, c.debug[pc]) else: stackTrace(c, tos, pc, errGenerated, "node has no type") - else: + of 1: # typeKind opcode: ensureKind(rkInt) if regs[rb].kind == rkNode and regs[rb].node.typ != nil: regs[ra].intVal = ord(regs[rb].node.typ.kind) #else: # stackTrace(c, tos, pc, errGenerated, "node has no type") + of 2: + # getTypeInst opcode: + ensureKind(rkNode) + if regs[rb].kind == rkNode and regs[rb].node.typ != nil: + regs[ra].node = opMapTypeInstToAst(regs[rb].node.typ, c.debug[pc]) + else: + stackTrace(c, tos, pc, errGenerated, "node has no type") + else: + # getTypeImpl opcode: + ensureKind(rkNode) + if regs[rb].kind == rkNode and regs[rb].node.typ != nil: + regs[ra].node = opMapTypeImplToAst(regs[rb].node.typ, c.debug[pc]) + else: + stackTrace(c, tos, pc, errGenerated, "node has no type") of opcNStrVal: decodeB(rkNode) createStr regs[ra] @@ -1554,7 +1573,7 @@ proc setupMacroParam(x: PNode, typ: PType): TFullReg = var evalMacroCounter: int proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode = - # XXX GlobalError() is ugly here, but I don't know a better solution for now + # XXX globalError() is ugly here, but I don't know a better solution for now inc(evalMacroCounter) if evalMacroCounter > 100: globalError(n.info, errTemplateInstantiationTooNested) @@ -1584,7 +1603,7 @@ proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode = # return value: tos.slots[0].kind = rkNode - tos.slots[0].node = newNodeIT(nkEmpty, n.info, sym.typ.sons[0]) + tos.slots[0].node = newNodeI(nkEmpty, n.info) # setup parameters: for i in 1.. <sym.typ.len: @@ -1604,4 +1623,3 @@ proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode = if cyclicTree(result): globalError(n.info, errCyclicTree) dec(evalMacroCounter) c.callsite = nil - #debug result diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 337e4ec8f..83c1dbf43 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -230,10 +230,10 @@ const slotSomeTemp* = slotTempUnknown relativeJumps* = {opcTJmp, opcFJmp, opcJmp, opcJmpBack} -template opcode*(x: TInstr): TOpcode {.immediate.} = TOpcode(x.uint32 and 0xff'u32) -template regA*(x: TInstr): TRegister {.immediate.} = TRegister(x.uint32 shr 8'u32 and 0xff'u32) -template regB*(x: TInstr): TRegister {.immediate.} = TRegister(x.uint32 shr 16'u32 and 0xff'u32) -template regC*(x: TInstr): TRegister {.immediate.} = TRegister(x.uint32 shr 24'u32) -template regBx*(x: TInstr): int {.immediate.} = (x.uint32 shr 16'u32).int +template opcode*(x: TInstr): TOpcode = TOpcode(x.uint32 and 0xff'u32) +template regA*(x: TInstr): TRegister = TRegister(x.uint32 shr 8'u32 and 0xff'u32) +template regB*(x: TInstr): TRegister = TRegister(x.uint32 shr 16'u32 and 0xff'u32) +template regC*(x: TInstr): TRegister = TRegister(x.uint32 shr 24'u32) +template regBx*(x: TInstr): int = (x.uint32 shr 16'u32).int -template jmpDiff*(x: TInstr): int {.immediate.} = regBx(x) - wordExcess +template jmpDiff*(x: TInstr): int = regBx(x) - wordExcess diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index a4f02092d..7dbf6f801 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -import ast, types, msgs, osproc, streams, options, idents, securehash +import ast, types, msgs, os, osproc, streams, options, idents, securehash proc readOutput(p: Process): string = result = "" @@ -51,7 +51,9 @@ proc opGorge*(cmd, input, cache: string): string = proc opSlurp*(file: string, info: TLineInfo, module: PSym): string = try: - let filename = file.findFile + var filename = parentDir(info.toFullPath) / file + if not fileExists(filename): + filename = file.findFile result = readFile(filename) # we produce a fake include statement for every slurped filename, so that # the module dependencies are accurate: @@ -61,123 +63,249 @@ proc opSlurp*(file: string, info: TLineInfo, module: PSym): string = localError(info, errCannotOpenFile, file) result = "" -proc atomicTypeX(name: string; t: PType; info: TLineInfo): PNode = +proc atomicTypeX(name: string; m: TMagic; t: PType; info: TLineInfo): PNode = let sym = newSym(skType, getIdent(name), t.owner, info) + sym.magic = m sym.typ = t result = newSymNode(sym) result.typ = t -proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode +proc mapTypeToAstX(t: PType; info: TLineInfo; + inst=false; allowRecursionX=false): PNode -proc mapTypeToBracket(name: string; t: PType; info: TLineInfo): PNode = +proc mapTypeToBracketX(name: string; m: TMagic; t: PType; info: TLineInfo; + inst=false): PNode = result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) - result.add atomicTypeX(name, t, info) + result.add atomicTypeX(name, m, t, info) for i in 0 .. < t.len: if t.sons[i] == nil: - let void = atomicTypeX("void", t, info) - void.typ = newType(tyEmpty, t.owner) + let void = atomicTypeX("void", mVoid, t, info) + void.typ = newType(tyVoid, t.owner) result.add void else: - result.add mapTypeToAst(t.sons[i], info) + result.add mapTypeToAstX(t.sons[i], info, inst) + +proc objectNode(n: PNode): PNode = + if n.kind == nkSym: + result = newNodeI(nkIdentDefs, n.info) + result.add n # name + result.add mapTypeToAstX(n.sym.typ, n.info, true, false) # type + result.add ast.emptyNode # no assigned value + else: + result = copyNode(n) + for i in 0 ..< n.safeLen: + result.add objectNode(n[i]) + +proc mapTypeToAstX(t: PType; info: TLineInfo; + inst=false; allowRecursionX=false): PNode = + var allowRecursion = allowRecursionX + template atomicType(name, m): untyped = atomicTypeX(name, m, t, info) + template mapTypeToAst(t,info): untyped = mapTypeToAstX(t, info, inst) + template mapTypeToAstR(t,info): untyped = mapTypeToAstX(t, info, inst, true) + template mapTypeToAst(t,i,info): untyped = + if i<t.len and t.sons[i]!=nil: mapTypeToAstX(t.sons[i], info, inst) + else: ast.emptyNode + template mapTypeToBracket(name, m, t, info): untyped = + mapTypeToBracketX(name, m, t, info, inst) + template newNodeX(kind): untyped = + newNodeIT(kind, if t.n.isNil: info else: t.n.info, t) + template newIdent(s): untyped = + var r = newNodeX(nkIdent) + r.add !s + r + template newIdentDefs(n,t): untyped = + var id = newNodeX(nkIdentDefs) + id.add n # name + id.add mapTypeToAst(t, info) # type + id.add ast.emptyNode # no assigned value + id + template newIdentDefs(s): untyped = newIdentDefs(s, s.typ) -proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = - template atomicType(name): expr = atomicTypeX(name, t, info) + if inst: + if t.sym != nil: # if this node has a symbol + if allowRecursion: # getTypeImpl behavior: turn off recursion + allowRecursion = false + else: # getTypeInst behavior: return symbol + return atomicType(t.sym.name.s, t.sym.magic) case t.kind - of tyNone: result = atomicType("none") - of tyBool: result = atomicType("bool") - of tyChar: result = atomicType("char") - of tyNil: result = atomicType("nil") - of tyExpr: result = atomicType("expr") - of tyStmt: result = atomicType("stmt") - of tyEmpty: result = atomicType"void" + of tyNone: result = atomicType("none", mNone) + of tyBool: result = atomicType("bool", mBool) + of tyChar: result = atomicType("char", mChar) + of tyNil: result = atomicType("nil", mNil) + of tyExpr: result = atomicType("expr", mExpr) + of tyStmt: result = atomicType("stmt", mStmt) + of tyVoid: result = atomicType("void", mVoid) + of tyEmpty: result = atomicType("empty", mNone) of tyArrayConstr, tyArray: result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) - result.add atomicType("array") - result.add mapTypeToAst(t.sons[0], info) + result.add atomicType("array", mArray) + if inst and t.sons[0].kind == tyRange: + var rng = newNodeX(nkInfix) + rng.add newIdentNode(getIdent(".."), info) + rng.add t.sons[0].n.sons[0].copyTree + rng.add t.sons[0].n.sons[1].copyTree + result.add rng + else: + result.add mapTypeToAst(t.sons[0], info) result.add mapTypeToAst(t.sons[1], info) of tyTypeDesc: if t.base != nil: result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) - result.add atomicType("typeDesc") + result.add atomicType("typeDesc", mTypeDesc) result.add mapTypeToAst(t.base, info) else: - result = atomicType"typeDesc" + result = atomicType("typeDesc", mTypeDesc) of tyGenericInvocation: result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) for i in 0 .. < t.len: result.add mapTypeToAst(t.sons[i], info) - of tyGenericInst, tyGenericBody, tyOrdinal, tyUserTypeClassInst: + of tyGenericInst: + if inst: + if allowRecursion: + result = mapTypeToAstR(t.lastSon, info) + else: + result = newNodeX(nkBracketExpr) + result.add mapTypeToAst(t.lastSon, info) + for i in 1 .. < t.len-1: + result.add mapTypeToAst(t.sons[i], info) + else: + result = mapTypeToAst(t.lastSon, info) + of tyGenericBody, tyOrdinal, tyUserTypeClassInst: result = mapTypeToAst(t.lastSon, info) of tyDistinct: - if allowRecursion: - result = mapTypeToBracket("distinct", t, info) + if inst: + result = newNodeX(nkDistinctTy) + result.add mapTypeToAst(t.sons[0], info) else: - result = atomicType(t.sym.name.s) - of tyGenericParam, tyForward: result = atomicType(t.sym.name.s) + if allowRecursion or t.sym == nil: + result = mapTypeToBracket("distinct", mDistinct, t, info) + else: + result = atomicType(t.sym.name.s, t.sym.magic) + of tyGenericParam, tyForward: + result = atomicType(t.sym.name.s, t.sym.magic) of tyObject: - if allowRecursion: - result = newNodeIT(nkObjectTy, if t.n.isNil: info else: t.n.info, t) - if t.sons[0] == nil: - result.add ast.emptyNode + if inst: + result = newNodeX(nkObjectTy) + result.add ast.emptyNode # pragmas not reconstructed yet + if t.sons[0] == nil: result.add ast.emptyNode # handle parent object else: - result.add mapTypeToAst(t.sons[0], info) - result.add copyTree(t.n) + var nn = newNodeX(nkOfInherit) + nn.add mapTypeToAst(t.sons[0], info) + result.add nn + if t.n.len > 0: + result.add objectNode(t.n) + else: + result.add ast.emptyNode else: - result = atomicType(t.sym.name.s) + if allowRecursion or t.sym == nil: + result = newNodeIT(nkObjectTy, if t.n.isNil: info else: t.n.info, t) + result.add ast.emptyNode + if t.sons[0] == nil: + result.add ast.emptyNode + else: + result.add mapTypeToAst(t.sons[0], info) + result.add copyTree(t.n) + else: + result = atomicType(t.sym.name.s, t.sym.magic) of tyEnum: result = newNodeIT(nkEnumTy, if t.n.isNil: info else: t.n.info, t) result.add copyTree(t.n) - of tyTuple: result = mapTypeToBracket("tuple", t, info) - of tySet: result = mapTypeToBracket("set", t, info) - of tyPtr: result = mapTypeToBracket("ptr", t, info) - of tyRef: result = mapTypeToBracket("ref", t, info) - of tyVar: result = mapTypeToBracket("var", t, info) - of tySequence: result = mapTypeToBracket("seq", t, info) - of tyProc: result = mapTypeToBracket("proc", t, info) - of tyOpenArray: result = mapTypeToBracket("openArray", t, info) + of tyTuple: + if inst: + result = newNodeX(nkTupleTy) + for s in t.n.sons: + result.add newIdentDefs(s) + else: + result = mapTypeToBracket("tuple", mTuple, t, info) + of tySet: result = mapTypeToBracket("set", mSet, t, info) + of tyPtr: + if inst: + result = newNodeX(nkPtrTy) + result.add mapTypeToAst(t.sons[0], info) + else: + result = mapTypeToBracket("ptr", mPtr, t, info) + of tyRef: + if inst: + result = newNodeX(nkRefTy) + result.add mapTypeToAst(t.sons[0], info) + else: + result = mapTypeToBracket("ref", mRef, t, info) + of tyVar: result = mapTypeToBracket("var", mVar, t, info) + of tySequence: result = mapTypeToBracket("seq", mSeq, t, info) + of tyProc: + if inst: + result = newNodeX(nkProcTy) + var fp = newNodeX(nkFormalParams) + if t.sons[0] == nil: + fp.add ast.emptyNode + else: + fp.add mapTypeToAst(t.sons[0], t.n[0].info) + for i in 1..<t.sons.len: + fp.add newIdentDefs(t.n[i], t.sons[i]) + result.add fp + result.add ast.emptyNode # pragmas aren't reconstructed yet + else: + result = mapTypeToBracket("proc", mNone, t, info) + of tyOpenArray: result = mapTypeToBracket("openArray", mOpenArray, t, info) of tyRange: result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) - result.add atomicType("range") + result.add atomicType("range", mRange) result.add t.n.sons[0].copyTree result.add t.n.sons[1].copyTree - of tyPointer: result = atomicType"pointer" - of tyString: result = atomicType"string" - of tyCString: result = atomicType"cstring" - of tyInt: result = atomicType"int" - of tyInt8: result = atomicType"int8" - of tyInt16: result = atomicType"int16" - of tyInt32: result = atomicType"int32" - of tyInt64: result = atomicType"int64" - of tyFloat: result = atomicType"float" - of tyFloat32: result = atomicType"float32" - of tyFloat64: result = atomicType"float64" - of tyFloat128: result = atomicType"float128" - of tyUInt: result = atomicType"uint" - of tyUInt8: result = atomicType"uint8" - of tyUInt16: result = atomicType"uint16" - of tyUInt32: result = atomicType"uint32" - of tyUInt64: result = atomicType"uint64" - of tyBigNum: result = atomicType"bignum" - of tyConst: result = mapTypeToBracket("const", t, info) - of tyMutable: result = mapTypeToBracket("mutable", t, info) - of tyVarargs: result = mapTypeToBracket("varargs", t, info) - of tyIter: result = mapTypeToBracket("iter", t, info) - of tyProxy: result = atomicType"error" - of tyBuiltInTypeClass: result = mapTypeToBracket("builtinTypeClass", t, info) + of tyPointer: result = atomicType("pointer", mPointer) + of tyString: result = atomicType("string", mString) + of tyCString: result = atomicType("cstring", mCString) + of tyInt: result = atomicType("int", mInt) + of tyInt8: result = atomicType("int8", mInt8) + of tyInt16: result = atomicType("int16", mInt16) + of tyInt32: result = atomicType("int32", mInt32) + of tyInt64: result = atomicType("int64", mInt64) + of tyFloat: result = atomicType("float", mFloat) + of tyFloat32: result = atomicType("float32", mFloat32) + of tyFloat64: result = atomicType("float64", mFloat64) + of tyFloat128: result = atomicType("float128", mFloat128) + of tyUInt: result = atomicType("uint", mUint) + of tyUInt8: result = atomicType("uint8", mUint8) + of tyUInt16: result = atomicType("uint16", mUint16) + of tyUInt32: result = atomicType("uint32", mUint32) + of tyUInt64: result = atomicType("uint64", mUint64) + of tyBigNum: result = atomicType("bignum", mNone) + of tyConst: result = mapTypeToBracket("const", mNone, t, info) + of tyMutable: result = mapTypeToBracket("mutable", mNone, t, info) + of tyVarargs: result = mapTypeToBracket("varargs", mVarargs, t, info) + of tyIter: result = mapTypeToBracket("iter", mNone, t, info) + of tyProxy: result = atomicType("error", mNone) + of tyBuiltInTypeClass: + result = mapTypeToBracket("builtinTypeClass", mNone, t, info) of tyUserTypeClass: - result = mapTypeToBracket("concept", t, info) + result = mapTypeToBracket("concept", mNone, t, info) result.add t.n.copyTree - of tyCompositeTypeClass: result = mapTypeToBracket("compositeTypeClass", t, info) - of tyAnd: result = mapTypeToBracket("and", t, info) - of tyOr: result = mapTypeToBracket("or", t, info) - of tyNot: result = mapTypeToBracket("not", t, info) - of tyAnything: result = atomicType"anything" + of tyCompositeTypeClass: + result = mapTypeToBracket("compositeTypeClass", mNone, t, info) + of tyAnd: result = mapTypeToBracket("and", mAnd, t, info) + of tyOr: result = mapTypeToBracket("or", mOr, t, info) + of tyNot: result = mapTypeToBracket("not", mNot, t, info) + of tyAnything: result = atomicType("anything", mNone) of tyStatic, tyFromExpr, tyFieldAccessor: - result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) - result.add atomicType("static") - if t.n != nil: - result.add t.n.copyTree + if inst: + if t.n != nil: result = t.n.copyTree + else: result = atomicType("void", mVoid) + else: + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) + result.add atomicType("static", mNone) + if t.n != nil: + result.add t.n.copyTree proc opMapTypeToAst*(t: PType; info: TLineInfo): PNode = - result = mapTypeToAst(t, info, true) + result = mapTypeToAstX(t, info, false, true) + +# the "Inst" version includes generic parameters in the resulting type tree +# and also tries to look like the corresponding Nim type declaration +proc opMapTypeInstToAst*(t: PType; info: TLineInfo): PNode = + result = mapTypeToAstX(t, info, true, false) + +# the "Impl" version includes generic parameters in the resulting type tree +# and also tries to look like the corresponding Nim type implementation +proc opMapTypeImplToAst*(t: PType; info: TLineInfo): PNode = + result = mapTypeToAstX(t, info, true, true) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 47f16a013..61ab65360 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -102,6 +102,10 @@ proc gABC(ctx: PCtx; n: PNode; opc: TOpcode; a, b, c: TRegister = 0) = let ins = (opc.uint32 or (a.uint32 shl 8'u32) or (b.uint32 shl 16'u32) or (c.uint32 shl 24'u32)).TInstr + when false: + if ctx.code.len == 43: + writeStackTrace() + echo "generating ", opc ctx.code.add(ins) ctx.debug.add(n.info) @@ -122,6 +126,11 @@ proc gABI(c: PCtx; n: PNode; opc: TOpcode; a, b: TRegister; imm: BiggestInt) = proc gABx(c: PCtx; n: PNode; opc: TOpcode; a: TRegister = 0; bx: int) = # Applies `opc` to `bx` and stores it into register `a` # `bx` must be signed and in the range [-32768, 32767] + when false: + if c.code.len == 43: + writeStackTrace() + echo "generating ", opc + if bx >= -32768 and bx <= 32767: let ins = (opc.uint32 or a.uint32 shl 8'u32 or (bx+wordExcess).uint32 shl 16'u32).TInstr @@ -222,7 +231,7 @@ proc getTempRange(cc: PCtx; n: int; kind: TSlotKind): TRegister = proc freeTempRange(c: PCtx; start: TRegister, n: int) = for i in start .. start+n-1: c.freeTemp(TRegister(i)) -template withTemp(tmp, typ: expr, body: stmt) {.immediate, dirty.} = +template withTemp(tmp, typ, body: untyped) {.dirty.} = var tmp = getTemp(c, typ) body c.freeTemp(tmp) @@ -232,7 +241,7 @@ proc popBlock(c: PCtx; oldLen: int) = c.patch(f) c.prc.blocks.setLen(oldLen) -template withBlock(labl: PSym; body: stmt) {.immediate, dirty.} = +template withBlock(labl: PSym; body: untyped) {.dirty.} = var oldLen {.gensym.} = c.prc.blocks.len c.prc.blocks.add TBlock(label: labl, fixups: @[]) body @@ -258,7 +267,7 @@ proc genx(c: PCtx; n: PNode; flags: TGenFlags = {}): TRegister = proc clearDest(c: PCtx; n: PNode; dest: var TDest) {.inline.} = # stmt is different from 'void' in meta programming contexts. # So we only set dest to -1 if 'void': - if dest >= 0 and (n.typ.isNil or n.typ.kind == tyEmpty): + if dest >= 0 and (n.typ.isNil or n.typ.kind == tyVoid): c.freeTemp(dest) dest = -1 @@ -365,7 +374,7 @@ proc canonValue*(n: PNode): PNode = proc rawGenLiteral(c: PCtx; n: PNode): int = result = c.constants.len - assert(n.kind != nkCall) + #assert(n.kind != nkCall) n.flags.incl nfAllConst c.constants.add n.canonValue internalAssert result < 0x7fff @@ -488,12 +497,30 @@ proc genReturn(c: PCtx; n: PNode) = gen(c, n.sons[0]) c.gABC(n, opcRet) + +proc genLit(c: PCtx; n: PNode; dest: var TDest) = + # opcLdConst is now always valid. We produce the necessary copy in the + # assignments now: + #var opc = opcLdConst + if dest < 0: dest = c.getTemp(n.typ) + #elif c.prc.slots[dest].kind == slotFixedVar: opc = opcAsgnConst + let lit = genLiteral(c, n) + c.gABx(n, opcLdConst, dest, lit) + proc genCall(c: PCtx; n: PNode; dest: var TDest) = + # it can happen that due to inlining we have a 'n' that should be + # treated as a constant (see issue #537). + #if n.typ != nil and n.typ.sym != nil and n.typ.sym.magic == mPNimrodNode: + # genLit(c, n, dest) + # return if dest < 0 and not isEmptyType(n.typ): dest = getTemp(c, n.typ) let x = c.getTempRange(n.len, slotTempUnknown) # varargs need 'opcSetType' for the FFI support: - let fntyp = n.sons[0].typ + let fntyp = skipTypes(n.sons[0].typ, abstractInst) for i in 0.. <n.len: + #if i > 0 and i < sonsLen(fntyp): + # let paramType = fntyp.n.sons[i] + # if paramType.typ.isCompileTimeOnly: continue var r: TRegister = x+i c.gen(n.sons[i], r) if i >= fntyp.len: @@ -570,16 +597,27 @@ proc genNew(c: PCtx; n: PNode) = c.freeTemp(dest) proc genNewSeq(c: PCtx; n: PNode) = - let dest = if needsAsgnPatch(n.sons[1]): c.getTemp(n.sons[1].typ) + let t = n.sons[1].typ + let dest = if needsAsgnPatch(n.sons[1]): c.getTemp(t) else: c.genx(n.sons[1]) let tmp = c.genx(n.sons[2]) - c.gABx(n, opcNewSeq, dest, c.genType(n.sons[1].typ.skipTypes( + c.gABx(n, opcNewSeq, dest, c.genType(t.skipTypes( abstractVar-{tyTypeDesc}))) c.gABx(n, opcNewSeq, tmp, 0) c.freeTemp(tmp) c.genAsgnPatch(n.sons[1], dest) c.freeTemp(dest) +proc genNewSeqOfCap(c: PCtx; n: PNode; dest: var TDest) = + let t = n.typ + let tmp = c.getTemp(n.sons[1].typ) + c.gABx(n, opcLdNull, dest, c.genType(t)) + c.gABx(n, opcLdImmInt, tmp, 0) + c.gABx(n, opcNewSeq, dest, c.genType(t.skipTypes( + abstractVar-{tyTypeDesc}))) + c.gABx(n, opcNewSeq, tmp, 0) + c.freeTemp(tmp) + proc genUnaryABC(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = let tmp = c.genx(n.sons[1]) if dest < 0: dest = c.getTemp(n.typ) @@ -626,8 +664,7 @@ proc genNarrowU(c: PCtx; n: PNode; dest: TDest) = let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) # uint is uint64 in the VM, we we only need to mask the result for # other unsigned types: - if t.kind in {tyUInt8..tyUInt32, tyInt8..tyInt32} or - (t.kind == tyInt and t.size == 4): + if t.kind in {tyUInt8..tyUInt32, tyInt8..tyInt32}: c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) proc genBinaryABCnarrow(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = @@ -704,6 +741,10 @@ proc genAddSubInt(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = c.genNarrow(n, dest) proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) = + if n.typ.kind == arg.typ.kind and arg.typ.kind == tyProc: + # don't do anything for lambda lifting conversions: + gen(c, arg, dest) + return let tmp = c.genx(arg) if dest < 0: dest = c.getTemp(n.typ) c.gABC(n, opc, dest, tmp) @@ -751,6 +792,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mNewSeq: unused(n, dest) c.genNewSeq(n) + of mNewSeqOfCap: c.genNewSeqOfCap(n, dest) of mNewString: genUnaryABC(c, n, dest, opcNewStr) # XXX buggy @@ -804,7 +846,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mLtF64: genBinaryABC(c, n, dest, opcLtFloat) of mLePtr, mLeU, mLeU64: genBinaryABC(c, n, dest, opcLeu) of mLtPtr, mLtU, mLtU64: genBinaryABC(c, n, dest, opcLtu) - of mEqProc, mEqRef, mEqUntracedRef, mEqCString: + of mEqProc, mEqRef, mEqUntracedRef: genBinaryABC(c, n, dest, opcEqRef) of mXor: genBinaryABCnarrowU(c, n, dest, opcXor) of mNot: genUnaryABC(c, n, dest, opcNot) @@ -821,7 +863,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = mToBiggestInt, mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr: genConv(c, n, n.sons[1], dest) - of mEqStr: genBinaryABC(c, n, dest, opcEqStr) + of mEqStr, mEqCString: genBinaryABC(c, n, dest, opcEqStr) of mLeStr: genBinaryABC(c, n, dest, opcLeStr) of mLtStr: genBinaryABC(c, n, dest, opcLtStr) of mEqSet: genBinarySet(c, n, dest, opcEqSet) @@ -970,7 +1012,12 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mNGetType: let tmp = c.genx(n.sons[1]) if dest < 0: dest = c.getTemp(n.typ) - c.gABC(n, opcNGetType, dest, tmp, if n[0].sym.name.s == "typeKind": 1 else: 0) + let rc = case n[0].sym.name.s: + of "getType": 0 + of "typeKind": 1 + of "getTypeInst": 2 + else: 3 # "getTypeImpl" + c.gABC(n, opcNGetType, dest, tmp, rc) c.freeTemp(tmp) #genUnaryABC(c, n, dest, opcNGetType) of mNStrVal: genUnaryABC(c, n, dest, opcNStrVal) @@ -1117,7 +1164,10 @@ proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; if isAddr and (let m = canElimAddr(n); m != nil): gen(c, m, dest, flags) return - let newflags = if isAddr: flags+{gfAddrOf} else: flags + + let af = if n[0].kind in {nkBracketExpr, nkDotExpr, nkCheckedFieldExpr}: {gfAddrOf, gfFieldAccess} + else: {gfAddrOf} + let newflags = if isAddr: flags+af else: flags # consider: # proc foo(f: var ref int) = # f = new(int) @@ -1132,7 +1182,7 @@ proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; if gfAddrOf notin flags and fitsRegister(n.typ): c.gABC(n, opcNodeToReg, dest, dest) elif isAddr and isGlobal(n.sons[0]): - gen(c, n.sons[0], dest, flags+{gfAddrOf}) + gen(c, n.sons[0], dest, flags+af) else: let tmp = c.genx(n.sons[0], newflags) if dest < 0: dest = c.getTemp(n.typ) @@ -1215,7 +1265,7 @@ proc checkCanEval(c: PCtx; n: PNode) = proc isTemp(c: PCtx; dest: TDest): bool = result = dest >= 0 and c.prc.slots[dest].kind >= slotTempUnknown -template needsAdditionalCopy(n): expr = +template needsAdditionalCopy(n): untyped = not c.isTemp(dest) and not fitsRegister(n.typ) proc skipDeref(n: PNode): PNode = @@ -1286,15 +1336,6 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) = let dest = c.genx(le, {gfAddrOf}) genAsgn(c, dest, ri, requiresCopy) -proc genLit(c: PCtx; n: PNode; dest: var TDest) = - # opcLdConst is now always valid. We produce the necessary copy in the - # assignments now: - #var opc = opcLdConst - if dest < 0: dest = c.getTemp(n.typ) - #elif c.prc.slots[dest].kind == slotFixedVar: opc = opcAsgnConst - let lit = genLiteral(c, n) - c.gABx(n, opcLdConst, dest, lit) - proc genTypeLit(c: PCtx; t: PType; dest: var TDest) = var n = newNode(nkType) n.typ = t @@ -1321,10 +1362,11 @@ proc genGlobalInit(c: PCtx; n: PNode; s: PSym) = # var decls{.compileTime.}: seq[NimNode] = @[] let dest = c.getTemp(s.typ) c.gABx(n, opcLdGlobal, dest, s.position) - let tmp = c.genx(s.ast) - c.preventFalseAlias(n, opcWrDeref, dest, 0, tmp) - c.freeTemp(dest) - c.freeTemp(tmp) + if s.ast != nil: + let tmp = c.genx(s.ast) + c.preventFalseAlias(n, opcWrDeref, dest, 0, tmp) + c.freeTemp(dest) + c.freeTemp(tmp) proc genRdVar(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = let s = n.sym @@ -1361,7 +1403,7 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = # see tests/t99bott for an example that triggers it: cannotEval(n) -template needsRegLoad(): expr = +template needsRegLoad(): untyped = gfAddrOf notin flags and fitsRegister(n.typ.skipTypes({tyVar})) proc genArrAccess2(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; @@ -1447,12 +1489,12 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode = of tyObject: result = newNodeIT(nkObjConstr, info, t) result.add(newNodeIT(nkEmpty, info, t)) - getNullValueAux(t.n, result) # initialize inherited fields: var base = t.sons[0] while base != nil: getNullValueAux(skipTypes(base, skipPtrs).n, result) base = base.sons[0] + getNullValueAux(t.n, result) of tyArray, tyArrayConstr: result = newNodeIT(nkBracket, info, t) for i in countup(0, int(lengthOrd(t)) - 1): @@ -1655,10 +1697,10 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = of skType: genTypeLit(c, s.typ, dest) of skGenericParam: - if c.prc.sym.kind == skMacro: + if c.prc.sym != nil and c.prc.sym.kind == skMacro: genRdVar(c, n, dest, flags) else: - internalError(n.info, "cannot generate code for: " & s.name.s) + globalError(n.info, errGenerated, "cannot generate code for: " & s.name.s) else: globalError(n.info, errGenerated, "cannot generate code for: " & s.name.s) of nkCallKinds: @@ -1736,9 +1778,9 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = of declarativeDefs: unused(n, dest) of nkLambdaKinds: - let s = n.sons[namePos].sym - discard genProc(c, s) - genLit(c, n.sons[namePos], dest) + #let s = n.sons[namePos].sym + #discard genProc(c, s) + genLit(c, newSymNode(n.sons[namePos].sym), dest) of nkChckRangeF, nkChckRange64, nkChckRange: let tmp0 = c.genx(n.sons[0]) @@ -1766,6 +1808,8 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = genConv(c, n, n.sons[1], dest, opcCast) else: globalError(n.info, errGenerated, "VM is not allowed to 'cast'") + of nkTypeOfExpr: + genTypeLit(c, n.typ, dest) else: globalError(n.info, errGenerated, "cannot generate VM code for " & $n) @@ -1865,8 +1909,8 @@ proc optimizeJumps(c: PCtx; start: int) = else: discard proc genProc(c: PCtx; s: PSym): int = - let x = s.ast.sons[optimizedCodePos] - if x.kind == nkEmpty: + var x = s.ast.sons[miscPos] + if x.kind == nkEmpty or x[0].kind == nkEmpty: #if s.name.s == "outterMacro" or s.name.s == "innerProc": # echo "GENERATING CODE FOR ", s.name.s let last = c.code.len-1 @@ -1877,7 +1921,11 @@ proc genProc(c: PCtx; s: PSym): int = c.debug.setLen(last) #c.removeLastEof result = c.code.len+1 # skip the jump instruction - s.ast.sons[optimizedCodePos] = newIntNode(nkIntLit, result) + if x.kind == nkEmpty: + x = newTree(nkBracket, newIntNode(nkIntLit, result), ast.emptyNode) + else: + x.sons[0] = newIntNode(nkIntLit, result) + s.ast.sons[miscPos] = x # thanks to the jmp we can add top level statements easily and also nest # procs easily: let body = s.getBody @@ -1912,4 +1960,4 @@ proc genProc(c: PCtx; s: PSym): int = c.prc = oldPrc else: c.prc.maxSlots = s.offset - result = x.intVal.int + result = x[0].intVal.int diff --git a/compiler/vmhooks.nim b/compiler/vmhooks.nim index 3456e893b..548a3af97 100644 --- a/compiler/vmhooks.nim +++ b/compiler/vmhooks.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -template setX(k, field) {.immediate, dirty.} = +template setX(k, field) {.dirty.} = var s: seq[TFullReg] move(s, cast[seq[TFullReg]](a.slots)) if s[a.ra].kind != k: @@ -48,7 +48,7 @@ proc setResult*(a: VmArgs; v: seq[string]) = for x in v: n.add newStrNode(nkStrLit, x) s[a.ra].node = n -template getX(k, field) {.immediate, dirty.} = +template getX(k, field) {.dirty.} = doAssert i < a.rc-1 let s = cast[seq[TFullReg]](a.slots) doAssert s[i+a.rb+1].kind == k diff --git a/compiler/vmops.nim b/compiler/vmops.nim index e40e05eff..1c2725a98 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -15,31 +15,36 @@ from math import sqrt, ln, log10, log2, exp, round, arccos, arcsin, from os import getEnv, existsEnv, dirExists, fileExists, walkDir -template mathop(op) {.immediate, dirty.} = +template mathop(op) {.dirty.} = registerCallback(c, "stdlib.math." & astToStr(op), `op Wrapper`) -template osop(op) {.immediate, dirty.} = +template osop(op) {.dirty.} = registerCallback(c, "stdlib.os." & astToStr(op), `op Wrapper`) -template systemop(op) {.immediate, dirty.} = +template systemop(op) {.dirty.} = registerCallback(c, "stdlib.system." & astToStr(op), `op Wrapper`) -template wrap1f(op) {.immediate, dirty.} = +template wrap1f_math(op) {.dirty.} = proc `op Wrapper`(a: VmArgs) {.nimcall.} = setResult(a, op(getFloat(a, 0))) mathop op -template wrap2f(op) {.immediate, dirty.} = +template wrap2f_math(op) {.dirty.} = proc `op Wrapper`(a: VmArgs) {.nimcall.} = setResult(a, op(getFloat(a, 0), getFloat(a, 1))) mathop op -template wrap1s(op) {.immediate, dirty.} = +template wrap1s_os(op) {.dirty.} = proc `op Wrapper`(a: VmArgs) {.nimcall.} = setResult(a, op(getString(a, 0))) osop op -template wrap2svoid(op) {.immediate, dirty.} = +template wrap1s_system(op) {.dirty.} = + proc `op Wrapper`(a: VmArgs) {.nimcall.} = + setResult(a, op(getString(a, 0))) + systemop op + +template wrap2svoid_system(op) {.dirty.} = proc `op Wrapper`(a: VmArgs) {.nimcall.} = op(getString(a, 0), getString(a, 1)) systemop op @@ -55,34 +60,35 @@ proc staticWalkDirImpl(path: string, relative: bool): PNode = newStrNode(nkStrLit, f)) proc registerAdditionalOps*(c: PCtx) = - wrap1f(sqrt) - wrap1f(ln) - wrap1f(log10) - wrap1f(log2) - wrap1f(exp) - wrap1f(round) - wrap1f(arccos) - wrap1f(arcsin) - wrap1f(arctan) - wrap2f(arctan2) - wrap1f(cos) - wrap1f(cosh) - wrap2f(hypot) - wrap1f(sinh) - wrap1f(sin) - wrap1f(tan) - wrap1f(tanh) - wrap2f(pow) - wrap1f(trunc) - wrap1f(floor) - wrap1f(ceil) - wrap2f(fmod) + wrap1f_math(sqrt) + wrap1f_math(ln) + wrap1f_math(log10) + wrap1f_math(log2) + wrap1f_math(exp) + wrap1f_math(round) + wrap1f_math(arccos) + wrap1f_math(arcsin) + wrap1f_math(arctan) + wrap2f_math(arctan2) + wrap1f_math(cos) + wrap1f_math(cosh) + wrap2f_math(hypot) + wrap1f_math(sinh) + wrap1f_math(sin) + wrap1f_math(tan) + wrap1f_math(tanh) + wrap2f_math(pow) + wrap1f_math(trunc) + wrap1f_math(floor) + wrap1f_math(ceil) + wrap2f_math(fmod) - wrap1s(getEnv) - wrap1s(existsEnv) - wrap1s(dirExists) - wrap1s(fileExists) - wrap2svoid(writeFile) + wrap1s_os(getEnv) + wrap1s_os(existsEnv) + wrap1s_os(dirExists) + wrap1s_os(fileExists) + wrap2svoid_system(writeFile) + wrap1s_system(readFile) systemop getCurrentExceptionMsg registerCallback c, "stdlib.*.staticWalkDir", proc (a: VmArgs) {.nimcall.} = setResult(a, staticWalkDirImpl(getString(a, 0), getBool(a, 1))) diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 3e0e05a94..b5ffd51c2 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -36,6 +36,7 @@ type wColon, wColonColon, wEquals, wDot, wDotDot, wStar, wMinus, wMagic, wThread, wFinal, wProfiler, wObjChecks, + wIntDefine, wStrDefine, wDestroy, @@ -121,7 +122,7 @@ const ":", "::", "=", ".", "..", "*", "-", - "magic", "thread", "final", "profiler", "objchecks", + "magic", "thread", "final", "profiler", "objchecks", "intdefine", "strdefine", "destroy", diff --git a/config/nim.cfg b/config/nim.cfg index b8608b3ce..0cc014a93 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -23,10 +23,6 @@ arm.linux.gcc.linkerexe = "arm-linux-gcc" mips.linux.gcc.exe = "mips-openwrt-linux-gcc" mips.linux.gcc.linkerexe = "mips-openwrt-linux-gcc" -@if not nimfix: - cs:partial -@end - path="$lib/deprecated/core" path="$lib/deprecated/pure" path="$lib/pure/collections" @@ -44,6 +40,11 @@ path="$lib/pure" @if nimbabel: nimblepath="$home/.nimble/pkgs/" + @if not windows: + nimblepath="/opt/nimble/pkgs/" + @else: + # TODO: + @end @end @if release or quick: @@ -163,10 +164,5 @@ vcc.options.always = "/nologo" vcc.options.speed = "/O2 /arch:SSE2" vcc.options.size = "/O1" -# Configuration for the Digital Mars C/C++ compiler: -@if windows: - dmc.path = r"$nimrod\dist\dm\bin" -@end - # Configuration for the Tiny C Compiler: tcc.options.always = "-w" diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index d973b922a..112833e58 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -1,5 +1,5 @@ # This is the config file for the documentation generator. -# (c) 2012 Andreas Rumpf +# (c) 2016 Andreas Rumpf # Feel free to edit the templates as you need. If you modify this file, it # might be worth updating the hardcoded values in packages/docutils/rstgen.nim @@ -1235,10 +1235,46 @@ dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator background-repeat: no-repeat; background-image: url("data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="); margin-bottom: -5px; } + div.pragma { + display: none; + } + span.pragmabegin { + cursor: pointer; + } + span.pragmaend { + cursor: pointer; + } </style> + +<script type="text/javascript"> +function togglepragma(d) { + if (d.style.display != 'inline') + d.style.display = 'inline'; + else + d.style.display = 'none'; +} + +function main() { + var elements = document.getElementsByClassName("pragmabegin"); + for (var i = 0; i < elements.length; ++i) { + var e = elements[i]; + e.onclick = function(event) { + togglepragma(event.target.nextSibling); + }; + } + var elements = document.getElementsByClassName("pragmaend"); + for (var i = 0; i < elements.length; ++i) { + var e = elements[i]; + e.onclick = function(event) { + togglepragma(event.target.previousSibling); + }; + } +} +</script> + </head> -<body> +<body onload="main()"> <div class="document" id="documentId"> <div class="container"> <h1 class="title">$title</h1> diff --git a/doc/advopt.txt b/doc/advopt.txt index 02aada4fb..02e69c5b8 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -6,6 +6,7 @@ Advanced commands: //rst2html convert a reStructuredText file to HTML //rst2tex convert a reStructuredText file to TeX //jsondoc extract the documentation to a json file + //jsondoc2 extract documentation to a json file (uses doc2) //buildIndex build an index for the whole documentation //run run the project (with Tiny C backend; buggy!) //genDepend generate a DOT file containing the diff --git a/doc/astspec.txt b/doc/astspec.txt index f235e2984..35eb1727b 100644 --- a/doc/astspec.txt +++ b/doc/astspec.txt @@ -33,7 +33,7 @@ contains: of nnkIdent: ident: NimIdent ## the identifier of nnkSym: - symbol: NimSymbol ## the symbol (after symbol lookup phase) + symbol: NimSym ## the symbol (after symbol lookup phase) else: sons: seq[NimNode] ## the node's sons (or children) diff --git a/doc/backends.txt b/doc/backends.txt index c7939baec..5846cce9b 100644 --- a/doc/backends.txt +++ b/doc/backends.txt @@ -110,8 +110,8 @@ interface <manual.html#foreign-function-interface>`_ mainly through the `importc pragma <manual.html#importc-pragma>`_. The ``importc`` pragma is the *generic* way of making backend symbols available in Nim and is available in all the target backends (JavaScript too). The C++ or Objective-C backends -have their respective `ImportCpp <nimc.html#importcpp-pragma>`_ and -`ImportObjC <nimc.html#importobjc-pragma>`_ pragmas to call methods from +have their respective `ImportCpp <manual.html#implementation-specific-pragmas-importcpp-pragma>`_ and +`ImportObjC <manual.html#implementation-specific-pragmas-importobjc-pragma>`_ pragmas to call methods from classes. Whenever you use any of these pragmas you need to integrate native code into @@ -420,7 +420,7 @@ function directly will be able to use it since Nim's garbage collector has not had a chance to run *yet*. This gives you enough time to make a copy for the C side of the program, as calling any further Nim procs *might* trigger garbage collection making the previously returned string garbage. Or maybe you -are `triggering yourself the collection <gc.html>`_. +are `yourself triggering the collection <gc.html>`_. Custom data types diff --git a/doc/basicopt.txt b/doc/basicopt.txt index 6a905bd53..9a1cfd956 100644 --- a/doc/basicopt.txt +++ b/doc/basicopt.txt @@ -11,7 +11,9 @@ Arguments: arguments are passed to the program being run (if --run option is selected) Options: -p, --path:PATH add path to search paths - -d, --define:SYMBOL define a conditional symbol + -d, --define:SYMBOL(:VAL) + define a conditional symbol + (Optionally: Define the value for that symbol) -u, --undef:SYMBOL undefine a conditional symbol -f, --forceBuild force rebuilding of all modules --stackTrace:on|off turn stack tracing on|off diff --git a/doc/docgen.txt b/doc/docgen.rst index 40c464ebd..40c464ebd 100644 --- a/doc/docgen.txt +++ b/doc/docgen.rst diff --git a/doc/endb.txt b/doc/endb.rst index 6757d98e3..6757d98e3 100644 --- a/doc/endb.txt +++ b/doc/endb.rst diff --git a/doc/estp.txt b/doc/estp.rst index 805a84eb7..805a84eb7 100644 --- a/doc/estp.txt +++ b/doc/estp.rst diff --git a/doc/filters.txt b/doc/filters.rst index 46bc6c3e5..1937b187c 100644 --- a/doc/filters.txt +++ b/doc/filters.rst @@ -23,6 +23,10 @@ just like an ordinary procedure call with named or positional arguments. The available parameters depend on the invoked filter. Before version 0.12.0 of the language ``#!`` was used instead of ``#?``. +**Hint:** With ``--hint[codeBegin]:on```or ``--verbosity:2`` +(or higher) Nim lists the processed code after each filter +application. + Pipe operator ============= @@ -41,9 +45,6 @@ Filters can be combined with the ``|`` pipe operator:: Available filters ================= -**Hint:** With ``--verbosity:2`` (or higher) Nim lists the processed code -after each filter application. - Replace filter -------------- diff --git a/doc/gc.txt b/doc/gc.rst index 4ada88d2f..1c8cb9122 100644 --- a/doc/gc.txt +++ b/doc/gc.rst @@ -56,7 +56,7 @@ file as well). With this switch the GC supports the following operations: .. code-block:: nim proc GC_setMaxPause*(MaxPauseInUs: int) - proc GC_step*(us: int, strongAdvice = false) + proc GC_step*(us: int, strongAdvice = false, stackSize = -1) The unit of the parameters ``MaxPauseInUs`` and ``us`` is microseconds. @@ -75,7 +75,13 @@ These two procs are the two modus operandi of the realtime GC: This allows the GC to perform some work for up to ``us`` time. This is useful to call in a main loop to ensure the GC can do its work. To bind all GC activity to a ``GC_step`` call, deactivate the GC with - ``GC_disable`` at program startup. + ``GC_disable`` at program startup. If ``strongAdvice`` is set to ``true``, + GC will be forced to perform collection cycle. Otherwise, GC may decide not + to do anything, if there is not much garbage to collect. + You may also specify the current stack size via ``stackSize`` parameter. + It can improve performance, when you know that there are no unique Nim + references below certain point on the stack. Make sure the size you specify + is greater than the potential worst case size. These procs provide a "best effort" realtime guarantee; in particular the cycle collector is not aware of deadlines yet. Deactivate it to get more diff --git a/doc/idetools.txt b/doc/idetools.rst index 2ffe46d4b..2ffe46d4b 100644 --- a/doc/idetools.txt +++ b/doc/idetools.rst diff --git a/doc/koch.txt b/doc/koch.rst index a58d386ea..a58d386ea 100644 --- a/doc/koch.txt +++ b/doc/koch.rst diff --git a/doc/lib.txt b/doc/lib.rst index 5ff6de7fd..66928055a 100644 --- a/doc/lib.txt +++ b/doc/lib.rst @@ -87,7 +87,7 @@ Collections and algorithms * `sequtils <sequtils.html>`_ This module implements operations for the built-in seq type which were inspired by functional programming languages. - + String handling --------------- @@ -97,9 +97,16 @@ String handling case of a string, splitting a string into substrings, searching for substrings, replacing substrings. +* `strmisc <strmisc.html>`_ + This module contains uncommon string handling operations that do not + fit with the commonly used operations in strutils. + * `parseutils <parseutils.html>`_ This module contains helpers for parsing tokens, numbers, identifiers, etc. +* `strscans <strscans.html>`_ + This module contains a ``scanf`` macro for convenient parsing of mini languages. + * `strtabs <strtabs.html>`_ The ``strtabs`` module implements an efficient hash table that is a mapping from strings to strings. Supports a case-sensitive, case-insensitive and @@ -199,6 +206,9 @@ Math libraries * `mersenne <mersenne.html>`_ Mersenne twister random number generator. +* `random <random.html>`_ + Fast and tiny random number generator. + * `stats <stats.html>`_ Statistical analysis @@ -440,6 +450,10 @@ Regular expressions This module contains procedures and operators for handling regular expressions. The current implementation uses PCRE. +* `nre <nre.html>`_ + Another implementation of procedures for using regular expressions. Also uses + PCRE. + Database support ---------------- diff --git a/doc/manual.txt b/doc/manual.rst index 2dbbf0447..2dbbf0447 100644 --- a/doc/manual.txt +++ b/doc/manual.rst diff --git a/doc/manual/exceptions.txt b/doc/manual/exceptions.txt index e7af65386..d06c13df4 100644 --- a/doc/manual/exceptions.txt +++ b/doc/manual/exceptions.txt @@ -155,4 +155,4 @@ Exception hierarchy The exception tree is defined in the `system <system.html>`_ module: -.. include:: exception_hierarchy_fragment.txt +.. include:: ../exception_hierarchy_fragment.txt diff --git a/doc/manual/ffi.txt b/doc/manual/ffi.txt index f08be6ad3..d7d9596d2 100644 --- a/doc/manual/ffi.txt +++ b/doc/manual/ffi.txt @@ -16,11 +16,19 @@ spelled*: .. code-block:: proc printf(formatstr: cstring) {.header: "<stdio.h>", importc: "printf", varargs.} -Note that this pragma is somewhat of a misnomer: Other backends will provide +Note that this pragma is somewhat of a misnomer: Other backends do provide the same feature under the same name. Also, if one is interfacing with C++ -the `ImportCpp pragma <nimc.html#importcpp-pragma>`_ and +the `ImportCpp pragma <manual.html#implementation-specific-pragmas-importcpp-pragma>`_ and interfacing with Objective-C the `ImportObjC pragma -<nimc.html#importobjc-pragma>`_ can be used. +<manual.html#implementation-specific-pragmas-importobjc-pragma>`_ can be used. + +The string literal passed to ``importc`` can be a format string: + +.. code-block:: Nim + proc p(s: cstring) {.importc: "prefix$1".} + +In the example the external name of ``p`` is set to ``prefixp``. Only ``$1`` +is available and a literal dollar sign must be written as ``$$``. Exportc pragma @@ -33,9 +41,19 @@ name is the Nim identifier *exactly as spelled*: .. code-block:: Nim proc callme(formatstr: cstring) {.exportc: "callMe", varargs.} -Note that this pragma is somewhat of a misnomer: Other backends will provide +Note that this pragma is somewhat of a misnomer: Other backends do provide the same feature under the same name. +The string literal passed to ``exportc`` can be a format string: + +.. code-block:: Nim + proc p(s: string) {.exportc: "prefix$1".} = + echo s + +In the example the external name of ``p`` is set to ``prefixp``. Only ``$1`` +is available and a literal dollar sign must be written as ``$$``. + + Extern pragma ------------- @@ -46,7 +64,9 @@ mangling. The string literal passed to ``extern`` can be a format string: proc p(s: string) {.extern: "prefix$1".} = echo s -In the example the external name of ``p`` is set to ``prefixp``. +In the example the external name of ``p`` is set to ``prefixp``. Only ``$1`` +is available and a literal dollar sign must be written as ``$$``. + Bycopy pragma diff --git a/doc/manual/generics.txt b/doc/manual/generics.txt index 07d98b289..c1c6467e7 100644 --- a/doc/manual/generics.txt +++ b/doc/manual/generics.txt @@ -232,16 +232,6 @@ type signatures of the required operations, but since type inference and default parameters are still applied in the provided block, it's also possible to encode usage protocols that do not reveal implementation details. -As a special rule providing further convenience when writing concepts, any -type value appearing in a callable expression will be treated as a variable of -the designated type for overload resolution purposes, unless the type value was -passed in its explicit ``typedesc[T]`` form: - -.. code-block:: nim - type - OutputStream = concept s - write(var s, string) - Much like generics, concepts are instantiated exactly once for each tested type and any static code included within them is also executed once. @@ -303,7 +293,7 @@ definition): var lastId = 0 - template genId*: expr = + template genId*: untyped = bind lastId inc(lastId) lastId diff --git a/doc/manual/lexing.txt b/doc/manual/lexing.txt index 4187a60a4..4d03023c3 100644 --- a/doc/manual/lexing.txt +++ b/doc/manual/lexing.txt @@ -116,7 +116,7 @@ operator characters instead. The following keywords are reserved and cannot be used as identifiers: .. code-block:: nim - :file: keywords.txt + :file: ../keywords.txt Some keywords are unused; they are reserved for future developments of the language. diff --git a/doc/manual/locking.txt b/doc/manual/locking.txt index b14c98636..c00efdd91 100644 --- a/doc/manual/locking.txt +++ b/doc/manual/locking.txt @@ -48,7 +48,7 @@ semantics and should not be used directly! It should only be used in templates that also implement some form of locking at runtime: .. code-block:: nim - template lock(a: TLock; body: stmt) = + template lock(a: TLock; body: untyped) = pthread_mutex_lock(a) {.locks: [a].}: try: @@ -64,7 +64,7 @@ model low level lockfree mechanisms: var dummyLock {.compileTime.}: int var atomicCounter {.guard: dummyLock.}: int - template atomicRead(x): expr = + template atomicRead(x): untyped = {.locks: [dummyLock].}: memoryReadBarrier() x @@ -167,7 +167,7 @@ the runtime check is required to ensure a global ordering for two locks ``a`` and ``b`` of the same lock level: .. code-block:: nim - template multilock(a, b: ptr TLock; body: stmt) = + template multilock(a, b: ptr TLock; body: untyped) = if cast[ByteAddress](a) < cast[ByteAddress](b): pthread_mutex_lock(a) pthread_mutex_lock(b) diff --git a/doc/manual/pragmas.txt b/doc/manual/pragmas.txt index f89194c9a..70fc4a914 100644 --- a/doc/manual/pragmas.txt +++ b/doc/manual/pragmas.txt @@ -55,7 +55,7 @@ destructor pragma ----------------- The ``destructor`` pragma is used to mark a proc to act as a type destructor. -Its usage is deprecated, See `type bound operations`_ instead. +Its usage is deprecated, see `type bound operations`_ instead. override pragma --------------- @@ -213,7 +213,7 @@ statement as seen in stack backtraces: .. code-block:: nim - template myassert*(cond: expr, msg = "") = + template myassert*(cond: untyped, msg = "") = if not cond: # change run-time line information of the 'raise' statement: {.line: InstantiationInfo().}: @@ -518,11 +518,16 @@ Example: .. code-block:: nim {.experimental.} + type + FooId = distinct int + BarId = distinct int + using + foo: FooId + bar: BarId - proc useUsing(dest: var string) = - using dest - add "foo" - add "bar" + proc useUsing(bar, foo) = + echo "bar is of type BarId" + echo "foo is of type FooId" Implementation Specific Pragmas @@ -1006,3 +1011,30 @@ debugging: # ... complex code here that produces crashes ... +compile time define pragmas +--------------------------- + +The pragmas listed here can be used to optionally accept values from +the -d/--define option at compile time. + +The implementation currently provides the following possible options (various +others may be added later). + +=============== ============================================ +pragma description +=============== ============================================ +intdefine Reads in a build-time define as an integer +strdefine Reads in a build-time define as a string +=============== ============================================ + +.. code-block:: nim + const FooBar {.intdefine.}: int = 5 + echo FooBar + +.. code-block:: bash + nim c -d:FooBar=42 foobar.c + +In the above example, providing the -d flag causes the symbol +``FooBar`` to be overwritten at compile time, printing out 42. If the +``-d:FooBar=42`` were to be omitted, the default value of 5 would be +used. diff --git a/doc/manual/procs.txt b/doc/manual/procs.txt index 9ce92de0d..ea6866845 100644 --- a/doc/manual/procs.txt +++ b/doc/manual/procs.txt @@ -215,6 +215,12 @@ the closure and its enclosing scope (i.e. any modifications made to them are visible in both places). The closure environment may be allocated on the heap or on the stack if the compiler determines that this would be safe. +Creating closures in loops +~~~~~~~~~~~~~~~~ + +Since closures capture local variables by reference it is often not wanted +behavior inside loop bodies. See `closureScope <system.html#closureScope>`_ +for details on how to change this behavior. Anonymous Procs --------------- @@ -223,7 +229,7 @@ Procs can also be treated as expressions, in which case it's allowed to omit the proc's name. .. code-block:: nim - var cities = @["Frankfurt", "Tokyo", "New York"] + var cities = @["Frankfurt", "Tokyo", "New York", "Kyiv"] cities.sort(proc (x,y: string): int = cmp(x.len, y.len)) @@ -619,7 +625,7 @@ Note that ``system.finished`` is error prone to use because it only returns 3 0 -Instead this code has be used: +Instead this code has to be used: .. code-block:: nim var c = mycount # instantiate the iterator diff --git a/doc/manual/special_ops.txt b/doc/manual/special_ops.txt index 702693423..1c7136bec 100644 --- a/doc/manual/special_ops.txt +++ b/doc/manual/special_ops.txt @@ -4,6 +4,9 @@ Special Operators dot operators ------------- +**Note**: Dot operators are still experimental and so need to be enabled +via ``{.experimental.}``. + Nim offers a special family of dot operators that can be used to intercept and rewrite proc call and field access attempts, referring to previously undeclared symbol names. They can be used to provide a diff --git a/doc/manual/stmts.txt b/doc/manual/stmts.txt index 65c810cf7..318738063 100644 --- a/doc/manual/stmts.txt +++ b/doc/manual/stmts.txt @@ -577,6 +577,9 @@ name ``c`` should default to type ``Context``, ``n`` should default to The ``using`` section uses the same indentation based grouping syntax as a ``var`` or ``let`` section. +Note that ``using`` is not applied for ``template`` since untyped template +parameters default to the type ``system.untyped``. + If expression ------------- diff --git a/doc/manual/syntax.txt b/doc/manual/syntax.txt index ca3b582ca..89f8ca707 100644 --- a/doc/manual/syntax.txt +++ b/doc/manual/syntax.txt @@ -19,10 +19,8 @@ other binary operators are left-associative. proc `^/`(x, y: float): float = # a right-associative division operator result = x / y - echo 12 ^/ 4 ^/ 8 # 24.0 (4 / 8 = 0.5, then - 12 / 0.5 = 24.0) - echo 12 / 4 / 8 # 0.375 (12 / 4 = 3.0, then - 3 / 8 = 0.375) + echo 12 ^/ 4 ^/ 8 # 24.0 (4 / 8 = 0.5, then 12 / 0.5 = 24.0) + echo 12 / 4 / 8 # 0.375 (12 / 4 = 3.0, then 3 / 8 = 0.375) Precedence ---------- @@ -72,53 +70,11 @@ Whether an operator is used a prefix operator is also affected by preceeding whi echo($foo) -Strong spaces -------------- - -The number of spaces preceding a non-keyword operator affects precedence -if the experimental parser directive ``#?strongSpaces`` is used. Indentation -is not used to determine the number of spaces. If 2 or more operators have the -same number of preceding spaces the precedence table applies, so ``1 + 3 * 4`` -is still parsed as ``1 + (3 * 4)``, but ``1+3 * 4`` is parsed as ``(1+3) * 4``: - -.. code-block:: nim - #? strongSpaces - if foo+4 * 4 == 8 and b&c | 9 ++ - bar: - echo "" - # is parsed as - if ((foo+4)*4 == 8) and (((b&c) | 9) ++ bar): echo "" - - -Furthermore whether an operator is used a prefix operator is affected by the -number of spaces: - -.. code-block:: nim - #? strongSpaces - echo $foo - # is parsed as - echo($foo) - -This also affects whether ``[]``, ``{}``, ``()`` are parsed as constructors -or as accessors: - -.. code-block:: nim - #? strongSpaces - echo (1,2) - # is parsed as - echo((1,2)) - -Only 0, 1, 2, 4 or 8 spaces are allowed to specify precedence and it is -enforced that infix operators have the same amount of spaces before and after -them. This rules does not apply when a newline follows after the operator, -then only the preceding spaces are considered. - - Grammar ------- The grammar's start symbol is ``module``. -.. include:: grammar.txt +.. include:: ../grammar.txt :literal: diff --git a/doc/manual/templates.txt b/doc/manual/templates.txt index b60fe632e..be5c6fa18 100644 --- a/doc/manual/templates.txt +++ b/doc/manual/templates.txt @@ -57,8 +57,7 @@ A template where every parameter is ``untyped`` is called an `immediate`:idx: template. For historical reasons templates can be explicitly annotated with an ``immediate`` pragma and then these templates do not take part in overloading resolution and the parameters' types are *ignored* by the -compiler. Explicit immediate templates are about to be deprecated in later -versions of the compiler. +compiler. Explicit immediate templates are now deprecated. **Note**: For historical reasons ``stmt`` is an alias for ``typed`` and ``expr`` an alias for ``untyped``, but new code should use the newer, @@ -159,7 +158,7 @@ bound from the definition scope of the template: var lastId = 0 - template genId*: expr = + template genId*: untyped = inc(lastId) lastId @@ -181,7 +180,7 @@ In templates identifiers can be constructed with the backticks notation: .. code-block:: nim - template typedef(name: expr, typ: typedesc) {.immediate.} = + template typedef(name: untyped, typ: typedesc) = type `T name`* {.inject.} = typ `P name`* {.inject.} = ref `T name` @@ -242,7 +241,7 @@ template cannot be accessed in the instantiation context: .. code-block:: nim - template newException*(exceptn: typedesc, message: string): expr = + template newException*(exceptn: typedesc, message: string): untyped = var e: ref exceptn # e is implicitly gensym'ed here new(e) @@ -264,7 +263,7 @@ is ``gensym`` and for ``proc``, ``iterator``, ``converter``, ``template``, template parameter, it is an inject'ed symbol: .. code-block:: nim - template withFile(f, fn, mode: expr, actions: stmt): stmt {.immediate.} = + template withFile(f, fn, mode: untyped, actions: untyped): untyped = block: var f: File # since 'f' is a template param, it's injected implicitly ... @@ -298,7 +297,7 @@ rewritten to ``f(x)``. Therefore the dot syntax has some limiations when it is used to invoke templates/macros: .. code-block:: nim - template declareVar(name: expr): stmt = + template declareVar(name: untyped) = const name {.inject.} = 45 # Doesn't compile: @@ -325,8 +324,7 @@ Macros ====== A macro is a special kind of low level template. Macros can be used -to implement `domain specific languages`:idx:. Like templates, macros come in -the 2 flavors *immediate* and *ordinary*. +to implement `domain specific languages`:idx:. While macros enable advanced compile-time code transformations, they cannot change Nim's syntax. However, this is no real restriction because @@ -351,7 +349,7 @@ variable number of arguments: # ``macros`` module: import macros - macro debug(n: varargs[expr]): stmt = + macro debug(n: varargs[untyped]): untyped = # `n` is a Nim AST that contains the whole macro invocation # this macro returns a list of statements: result = newNimNode(nnkStmtList, n) @@ -406,7 +404,7 @@ builtin can be used for that: .. code-block:: nim import macros - macro debug(n: varargs[expr]): stmt = + macro debug(n: varargs[typed]): untyped = result = newNimNode(nnkStmtList, n) for i in 0..n.len-1: # we can bind symbols in scope via 'bindSym': @@ -454,7 +452,7 @@ regular expressions: .. code-block:: nim import macros - macro case_token(n: stmt): stmt = + macro case_token(n: untyped): untyped = # creates a lexical analyzer from regular expressions # ... (implementation is an exercise for the reader :-) discard @@ -486,14 +484,14 @@ Whole routines (procs, iterators etc.) can also be passed to a template or a macro via the pragma notation: .. code-block:: nim - template m(s: stmt) = discard + template m(s: untyped) = discard proc p() {.m.} = discard This is a simple syntactic transformation into: .. code-block:: nim - template m(s: stmt) = discard + template m(s: untyped) = discard m: proc p() = discard diff --git a/doc/manual/trmacros.txt b/doc/manual/trmacros.txt index 446896e28..0845cebf5 100644 --- a/doc/manual/trmacros.txt +++ b/doc/manual/trmacros.txt @@ -135,7 +135,7 @@ The ``|`` operator The ``|`` operator if used as infix operator creates an ordered choice: .. code-block:: nim - template t{0|1}(): expr = 3 + template t{0|1}(): untyped = 3 let a = 1 # outputs 3: echo a @@ -144,7 +144,7 @@ The matching is performed after the compiler performed some optimizations like constant folding, so the following does not work: .. code-block:: nim - template t{0|1}(): expr = 3 + template t{0|1}(): untyped = 3 # outputs 1: echo 1 @@ -161,7 +161,7 @@ A pattern expression can be bound to a pattern parameter via the ``expr{param}`` notation: .. code-block:: nim - template t{(0|1|2){x}}(x: expr): expr = x+1 + template t{(0|1|2){x}}(x: untyped): untyped = x+1 let a = 1 # outputs 2: echo a @@ -173,7 +173,7 @@ The ``~`` operator The ``~`` operator is the **not** operator in patterns: .. code-block:: nim - template t{x = (~x){y} and (~x){z}}(x, y, z: bool): stmt = + template t{x = (~x){y} and (~x){z}}(x, y, z: bool) = x = y if x: x = z @@ -200,7 +200,7 @@ to ``&(a, b, c)``: for i in 1..len(s)-1: result.add s[i] inc calls - template optConc{ `&&` * a }(a: string): expr = &&a + template optConc{ `&&` * a }(a: string): untyped = &&a let space = " " echo "my" && (space & "awe" && "some " ) && "concat" @@ -239,7 +239,7 @@ all the arguments, but also the matched operators in reverse polish notation: proc mat21(): Matrix = result.dummy = 21 - macro optM{ (`+`|`-`|`*`) ** a }(a: Matrix): expr = + macro optM{ (`+`|`-`|`*`) ** a }(a: Matrix): untyped = echo treeRepr(a) result = newCall(bindSym"mat21") @@ -273,7 +273,7 @@ parameter is of the type ``varargs`` it is treated specially and it can match template optWrite{ write(f, x) ((write|writeLine){w})(f, y) - }(x, y: varargs[expr], f: File, w: expr) = + }(x, y: varargs[untyped], f: File, w: untyped) = w(f, x, y) @@ -288,8 +288,8 @@ implemented with term rewriting: proc p(x, y: int; cond: bool): int = result = if cond: x + y else: x - y - template optP1{p(x, y, true)}(x, y: expr): expr = x + y - template optP2{p(x, y, false)}(x, y: expr): expr = x - y + template optP1{p(x, y, true)}(x, y: untyped): untyped = x + y + template optP2{p(x, y, false)}(x, y: untyped): untyped = x - y Example: Hoisting diff --git a/doc/manual/type_rel.txt b/doc/manual/type_rel.txt index d62cf65c3..5b68f73aa 100644 --- a/doc/manual/type_rel.txt +++ b/doc/manual/type_rel.txt @@ -313,42 +313,78 @@ matches better than just ``T`` then. Automatic dereferencing ----------------------- -If the `experimental mode <experimental pragma>`_ is active and no other match +If the `experimental mode <#pragmas-experimental-pragma>`_ is active and no other match is found, the first argument ``a`` is dereferenced automatically if it's a pointer type and overloading resolution is tried with ``a[]`` instead. +Automatic self insertions +------------------------- -Lazy type resolution for expr ------------------------------ +Starting with version 0.14 of the language, Nim supports ``field`` as a +shortcut for ``self.field`` comparable to the `this`:idx: keyword in Java +or C++. This feature has to be explicitly enabled via a ``{.this: self.}`` +statement pragma. This pragma is active for the rest of the module: + +.. code-block:: nim + type + Parent = object of RootObj + parentField: int + Child = object of Parent + childField: int + + {.this: self.} + proc sumFields(self: Child): int = + result = parentField + childField + # is rewritten to: + # result = self.parentField + self.childField + +Instead of ``self`` any other identifier can be used too, but +``{.this: self.}`` will become the default directive for the whole language +eventually. + +In addition to fields, routine applications are also rewritten, but only +if no other interpretation of the call is possible: + +.. code-block:: nim + proc test(self: Child) = + echo childField, " ", sumFields() + # is rewritten to: + echo self.childField, " ", sumFields(self) + # but NOT rewritten to: + echo self, self.childField, " ", sumFields(self) + + +Lazy type resolution for untyped +-------------------------------- **Note**: An `unresolved`:idx: expression is an expression for which no symbol lookups and no type checking have been performed. Since templates and macros that are not declared as ``immediate`` participate in overloading resolution it's essential to have a way to pass unresolved -expressions to a template or macro. This is what the meta-type ``expr`` +expressions to a template or macro. This is what the meta-type ``untyped`` accomplishes: .. code-block:: nim - template rem(x: expr) = discard + template rem(x: untyped) = discard rem unresolvedExpression(undeclaredIdentifier) -A parameter of type ``expr`` always matches any argument (as long as there is +A parameter of type ``untyped`` always matches any argument (as long as there is any argument passed to it). But one has to watch out because other overloads might trigger the argument's resolution: .. code-block:: nim - template rem(x: expr) = discard + template rem(x: untyped) = discard proc rem[T](x: T) = discard # undeclared identifier: 'unresolvedExpression' rem unresolvedExpression(undeclaredIdentifier) -``expr`` is the only metatype that is lazy in this sense, the other -metatypes ``stmt`` and ``typedesc`` are not lazy. +``untyped`` and ``varargs[untyped]`` are the only metatype that are lazy in this sense, the other +metatypes ``typed`` and ``typedesc`` are not lazy. Varargs matching diff --git a/doc/manual/typedesc.txt b/doc/manual/typedesc.txt index de1d84d7d..6922d77e4 100644 --- a/doc/manual/typedesc.txt +++ b/doc/manual/typedesc.txt @@ -77,38 +77,6 @@ Once bound, typedesc params can appear in the rest of the proc signature: declareVariableWithType int, 42 -When used with macros and .compileTime. procs on the other hand, the compiler -does not need to instantiate the code multiple times, because types then can be -manipulated using the unified internal symbol representation. In such context -typedesc acts as any other type. One can create variables, store typedesc -values inside containers and so on. For example, here is how one can create -a type-safe wrapper for the unsafe `printf` function from C: - -.. code-block:: nim - macro safePrintF(formatString: string{lit}, args: varargs[expr]): expr = - var i = 0 - for c in formatChars(formatString): - var expectedType = case c - of 'c': char - of 'd', 'i', 'x', 'X': int - of 'f', 'e', 'E', 'g', 'G': float - of 's': string - of 'p': pointer - else: EOutOfRange - - var actualType = args[i].getType - inc i - - if expectedType == EOutOfRange: - error c & " is not a valid format character" - elif expectedType != actualType: - error "type mismatch for argument ", i, ". expected type: ", - expectedType.name, ", actual type: ", actualType.name - - # keep the original callsite, but use cprintf instead - result = callsite() - result[0] = newIdentNode(!"cprintf") - Overload resolution can be further influenced by constraining the set of types that will match the typedesc param: diff --git a/doc/manual/types.txt b/doc/manual/types.txt index a1596bcea..1e2dc857f 100644 --- a/doc/manual/types.txt +++ b/doc/manual/types.txt @@ -435,7 +435,9 @@ has the same type. Arrays always have a fixed length which is specified at compile time (except for open arrays). They can be indexed by any ordinal type. A parameter ``A`` may be an *open array*, in which case it is indexed by integers from 0 to ``len(A)-1``. An array expression may be constructed by the -array constructor ``[]``. +array constructor ``[]``. The element type of this array expression is +inferred from the type of the first element. All other elements need to be +implicitly convertable to this type. Sequences are similar to arrays but of dynamic length which may change during runtime (like strings). Sequences are implemented as growable arrays, @@ -460,6 +462,8 @@ Example: x = [1, 2, 3, 4, 5, 6] # [] is the array constructor y = @[1, 2, 3, 4, 5, 6] # the @ turns the array into a sequence + let z = [1.0, 2, 3, 4] # the type of z is array[0..3, float] + The lower bound of an array or sequence may be received by the built-in proc ``low()``, the higher bound by ``high()``. The length may be received by ``len()``. ``low()`` for a sequence or an open array always returns @@ -539,12 +543,12 @@ not wrapped in another implicit array construction: takeV([123, 2, 1]) # takeV's T is "int", not "array of int" -``varargs[expr]`` is treated specially: It matches a variable list of arguments +``varargs[typed]`` is treated specially: It matches a variable list of arguments of arbitrary type but *always* constructs an implicit array. This is required so that the builtin ``echo`` proc does what is expected: .. code-block:: nim - proc echo*(x: varargs[expr, `$`]) {...} + proc echo*(x: varargs[typed, `$`]) {...} echo @[1, 2, 3] # prints "@[1, 2, 3]" and not "123" @@ -694,7 +698,7 @@ branch switch ``system.reset`` has to be used. Set type -------- -.. include:: sets_fragment.txt +.. include:: ../sets_fragment.txt Reference and pointer types --------------------------- @@ -1082,7 +1086,7 @@ But it seems all this boilerplate code needs to be repeated for the ``Euro`` currency. This can be solved with templates_. .. code-block:: nim - template additive(typ: typedesc): stmt = + template additive(typ: typedesc) = proc `+` *(x, y: typ): typ {.borrow.} proc `-` *(x, y: typ): typ {.borrow.} @@ -1090,18 +1094,18 @@ currency. This can be solved with templates_. proc `+` *(x: typ): typ {.borrow.} proc `-` *(x: typ): typ {.borrow.} - template multiplicative(typ, base: typedesc): stmt = + template multiplicative(typ, base: typedesc) = proc `*` *(x: typ, y: base): typ {.borrow.} proc `*` *(x: base, y: typ): typ {.borrow.} proc `div` *(x: typ, y: base): typ {.borrow.} proc `mod` *(x: typ, y: base): typ {.borrow.} - template comparable(typ: typedesc): stmt = + template comparable(typ: typedesc) = proc `<` * (x, y: typ): bool {.borrow.} proc `<=` * (x, y: typ): bool {.borrow.} proc `==` * (x, y: typ): bool {.borrow.} - template defineCurrency(typ, base: expr): stmt = + template defineCurrency(typ, base: untyped) = type typ* = distinct base additive(typ) diff --git a/doc/nep1.txt b/doc/nep1.rst index b4bd6309c..b5991ba9e 100644 --- a/doc/nep1.txt +++ b/doc/nep1.rst @@ -125,8 +125,8 @@ changed in the future. Coding Conventions ------------------ -- The 'return' statement should only be used when it's control-flow properties - are required. Use a procedures implicit 'result' variable instead. This +- The 'return' statement should only be used when its control-flow properties + are required. Use a procedure's implicit 'result' variable instead. This improves readability. - Prefer to return `[]` and `""` instead of `nil`, or throw an exception if @@ -150,7 +150,7 @@ Conventions for multi-line statements and expressions - Any tuple type declarations that are longer than one line should use the regular object type layout instead. This enhances the readability of the - tuple declaration by splitting its members information across multiple lines. + tuple declaration by splitting its members' information across multiple lines. .. code-block:: nim type diff --git a/doc/nimc.txt b/doc/nimc.rst index e7cb57037..eb1beb549 100644 --- a/doc/nimc.txt +++ b/doc/nimc.rst @@ -98,6 +98,11 @@ enable builds in release mode (``-d:release``) where certain safety checks are omitted for better performance. Another common use is the ``-d:ssl`` switch to activate `SSL sockets <sockets.html>`_. +Additionally, you may pass a value along with the symbol: ``-d:x=y`` +which may be used in conjunction with the `compile time define +pragmas<manual.html#implementation-specific-pragmas-compile-time-define-pragmas>`_ +to override symbols during build time. + Configuration files ------------------- @@ -243,7 +248,9 @@ Define Effect ``useFork`` Makes ``osproc`` use ``fork`` instead of ``posix_spawn``. ``useNimRtl`` Compile and link against ``nimrtl.dll``. ``useMalloc`` Makes Nim use C's `malloc`:idx: instead of Nim's - own memory manager. This only works with ``gc:none``. + own memory manager, ableit prefixing each allocation with + its size to support clearing memory on reallocation. + This only works with ``gc:none``. ``useRealtimeGC`` Enables support of Nim's GC for *soft* realtime systems. See the documentation of the `gc <gc.html>`_ for further information. @@ -368,7 +375,10 @@ For example, to generate code for an `AVR`:idx: processor use this command:: For the ``standalone`` target one needs to provide a file ``panicoverride.nim``. See ``tests/manyloc/standalone/panicoverride.nim`` for an example -implementation. +implementation. Additionally, users should specify the +amount of heap space to use with the ``-d:StandaloneHeapSize=<size>`` +command line switch. Note that the total heap size will be +``<size> * sizeof(float64)``. Nim for realtime systems diff --git a/doc/nimfix.txt b/doc/nimfix.rst index 62064fe69..62064fe69 100644 --- a/doc/nimfix.txt +++ b/doc/nimfix.rst diff --git a/doc/nimgrep.txt b/doc/nimgrep.rst index 791ead162..791ead162 100644 --- a/doc/nimgrep.txt +++ b/doc/nimgrep.rst diff --git a/doc/niminst.txt b/doc/niminst.rst index 7bd0f719e..bf5cb0f50 100644 --- a/doc/niminst.txt +++ b/doc/niminst.rst @@ -27,7 +27,7 @@ Configuration file niminst uses the Nim `parsecfg <parsecfg.html>`_ module to parse the configuration file. Here's an example of how the syntax looks like: -.. include:: doc/mytest.cfg +.. include:: mytest.cfg :literal: The value of a key-value pair can reference user-defined variables via @@ -190,6 +190,6 @@ Real world example The installers for the Nim compiler itself are generated by niminst. Have a look at its configuration file: -.. include:: compiler/installer.ini +.. include:: ../compiler/installer.ini :literal: diff --git a/doc/nims.txt b/doc/nims.rst index 7c76efe42..7c76efe42 100644 --- a/doc/nims.txt +++ b/doc/nims.rst diff --git a/doc/nimsuggest.txt b/doc/nimsuggest.rst index 2b52196b9..2b52196b9 100644 --- a/doc/nimsuggest.txt +++ b/doc/nimsuggest.rst diff --git a/doc/tut1.txt b/doc/tut1.rst index d896a7044..d896a7044 100644 --- a/doc/tut1.txt +++ b/doc/tut1.rst diff --git a/doc/tut2.txt b/doc/tut2.rst index 3f94325ff..3f94325ff 100644 --- a/doc/tut2.txt +++ b/doc/tut2.rst diff --git a/examples/ssl/extradata.nim b/examples/ssl/extradata.nim new file mode 100644 index 000000000..1e3b89b02 --- /dev/null +++ b/examples/ssl/extradata.nim @@ -0,0 +1,26 @@ +# Stores extra data inside the SSL context. +import net + +let ctx = newContext() + +# Our unique index for storing foos +let fooIndex = ctx.getExtraDataIndex() +# And another unique index for storing foos +let barIndex = ctx.getExtraDataIndex() +echo "got indexes ", fooIndex, " ", barIndex + +try: + discard ctx.getExtraData(fooIndex) + assert false +except IndexError: + echo("Success") + +type + FooRef = ref object of RootRef + foo: int + +let foo = FooRef(foo: 5) +ctx.setExtraData(fooIndex, foo) +doAssert ctx.getExtraData(fooIndex).FooRef == foo + +ctx.destroyContext() diff --git a/examples/ssl/pskclient.nim b/examples/ssl/pskclient.nim new file mode 100644 index 000000000..c83f27fbc --- /dev/null +++ b/examples/ssl/pskclient.nim @@ -0,0 +1,16 @@ +# Create connection encrypted using preshared key (TLS-PSK). +import net + +static: assert defined(ssl) + +let sock = newSocket() +sock.connect("localhost", Port(8800)) + +proc clientFunc(identityHint: string): tuple[identity: string, psk: string] = + echo "identity hint ", identityHint.repr + return ("foo", "psk-of-foo") + +let context = newContext(cipherList="PSK-AES256-CBC-SHA") +context.clientGetPskFunc = clientFunc +context.wrapConnectedSocket(sock, handshakeAsClient) +context.destroyContext() diff --git a/examples/ssl/pskserver.nim b/examples/ssl/pskserver.nim new file mode 100644 index 000000000..859eaa875 --- /dev/null +++ b/examples/ssl/pskserver.nim @@ -0,0 +1,20 @@ +# Accept connection encrypted using preshared key (TLS-PSK). +import net + +static: assert defined(ssl) + +let sock = newSocket() +sock.bindAddr(Port(8800)) +sock.listen() + +let context = newContext(cipherList="PSK-AES256-CBC-SHA") +context.pskIdentityHint = "hello" +context.serverGetPskFunc = proc(identity: string): string = "psk-of-" & identity + +while true: + var client = new(Socket) + sock.accept(client) + sock.setSockOpt(OptReuseAddr, true) + echo "accepted connection" + context.wrapConnectedSocket(client, handshakeAsServer) + echo "got connection with identity ", client.getPskIdentity() diff --git a/examples/unix_socket/client.nim b/examples/unix_socket/client.nim new file mode 100644 index 000000000..f4283d64d --- /dev/null +++ b/examples/unix_socket/client.nim @@ -0,0 +1,6 @@ +import net + +let sock = newSocket(AF_UNIX, SOCK_STREAM, IPPROTO_IP) + +sock.connectUnix("sock") +sock.send("hello\n") diff --git a/examples/unix_socket/server.nim b/examples/unix_socket/server.nim new file mode 100644 index 000000000..e798bbb48 --- /dev/null +++ b/examples/unix_socket/server.nim @@ -0,0 +1,14 @@ +import net + +let sock = newSocket(AF_UNIX, SOCK_STREAM, IPPROTO_IP) +sock.bindUnix("sock") +sock.listen() + +while true: + var client = new(Socket) + sock.accept(client) + var output = "" + output.setLen 32 + client.readLine(output) + echo "got ", output + client.close() diff --git a/koch.nim b/koch.nim index f7c0ae204..04f6a4e4e 100644 --- a/koch.nim +++ b/koch.nim @@ -1,7 +1,7 @@ # # # Maintenance program for Nim -# (c) Copyright 2015 Andreas Rumpf +# (c) Copyright 2016 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -30,7 +30,7 @@ const +-----------------------------------------------------------------+ | Maintenance program for Nim | | Version $1| -| (c) 2015 Andreas Rumpf | +| (c) 2016 Andreas Rumpf | +-----------------------------------------------------------------+ Build time: $2, $3 @@ -42,6 +42,7 @@ Possible Commands: boot [options] bootstraps with given command line options install [bindir] installs to given directory; Unix only! geninstall generate ./install.sh; Unix only! + testinstall test tar.xz package; Unix only! Only for devs! clean cleans Nim project; removes generated files web [options] generates the website and the full documentation website [options] generates only the website @@ -79,9 +80,55 @@ proc findNim(): string = # assume there is a symlink to the exe or something: return nim -proc exec(cmd: string, errorcode: int = QuitFailure) = +proc exec(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = + let prevPath = getEnv("PATH") + if additionalPath.len > 0: + var absolute = additionalPATH + if not absolute.isAbsolute: + absolute = getCurrentDir() / absolute + echo("Adding to $PATH: ", absolute) + putEnv("PATH", prevPath & PathSep & absolute) echo(cmd) if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) + putEnv("PATH", prevPath) + +proc execCleanPath(cmd: string, + additionalPath = ""; errorcode: int = QuitFailure) = + # simulate a poor man's virtual environment + let prevPath = getEnv("PATH") + when defined(windows): + let CleanPath = r"$1\system32;$1;$1\System32\Wbem" % getEnv"SYSTEMROOT" + else: + const CleanPath = r"/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin" + putEnv("PATH", CleanPath & PathSep & additionalPath) + echo(cmd) + if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) + putEnv("PATH", prevPath) + +proc testUnixInstall() = + let oldCurrentDir = getCurrentDir() + try: + let destDir = getTempDir() + copyFile("build/nim-$1.tar.xz" % VersionAsString, + destDir / "nim-$1.tar.xz" % VersionAsString) + setCurrentDir(destDir) + execCleanPath("tar -xJf nim-$1.tar.xz" % VersionAsString) + setCurrentDir("nim-$1" % VersionAsString) + execCleanPath("sh build.sh") + # first test: try if './bin/nim --version' outputs something sane: + let output = execProcess("./bin/nim --version").splitLines + if output.len > 0 and output[0].contains(VersionAsString): + echo "Version check: success" + execCleanPath("./bin/nim c koch.nim") + execCleanPath("./koch boot -d:release", destDir / "bin") + # check the docs build: + execCleanPath("./koch web", destDir / "bin") + # check the tests work: + execCleanPath("./koch tests", destDir / "bin") + else: + echo "Version check: failure" + finally: + setCurrentDir oldCurrentDir proc tryExec(cmd: string): bool = echo(cmd) @@ -102,13 +149,28 @@ proc csource(args: string) = exec("$4 cc $1 -r $3 --var:version=$2 --var:mingw=none csource --main:compiler/nim.nim compiler/installer.ini $1" % [args, VersionAsString, compileNimInst, findNim()]) +proc bundleNimble() = + if dirExists("dist/nimble/.git"): + exec("git --git-dir dist/nimble/.git pull") + else: + exec("git clone https://github.com/nim-lang/nimble.git dist/nimble") + let tags = execProcess("git --git-dir dist/nimble/.git tag -l v*").splitLines + let tag = tags[^1] + exec("git --git-dir dist/nimble/.git checkout " & tag) + # now compile Nimble and copy it to $nim/bin for the installer.ini + # to pick it up: + exec(findNim() & " c dist/nimble/src/nimble.nim") + copyExe("dist/nimble/src/nimble".exe, "bin/nimble".exe) + proc zip(args: string) = + bundleNimble() exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % [VersionAsString, compileNimInst, findNim()]) exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim zip compiler/installer.ini" % ["tools/niminst/niminst".exe, VersionAsString]) proc xz(args: string) = + bundleNimble() exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % [VersionAsString, compileNimInst, findNim()]) exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim xz compiler/installer.ini" % @@ -119,6 +181,7 @@ proc buildTool(toolname, args: string) = copyFile(dest="bin"/ splitFile(toolname).name.exe, source=toolname.exe) proc nsis(args: string) = + bundleNimble() # make sure we have generated the niminst executables: buildTool("tools/niminst/niminst", args) #buildTool("tools/nimgrep", args) @@ -146,7 +209,7 @@ proc website(args: string) = proc pdf(args="") = exec("$# cc -r tools/nimweb.nim $# --pdf web/website.ini --putenv:nimversion=$#" % - [findNim(), args, VersionAsString]) + [findNim(), args, VersionAsString], additionalPATH=findNim().splitFile.dir) # -------------- boot --------------------------------------------------------- @@ -336,10 +399,11 @@ template `|`(a, b): expr = (if a.len > 0: a else: b) proc tests(args: string) = # we compile the tester with taintMode:on to have a basic # taint mode test :-) - exec "nim cc --taintMode:on tests/testament/tester" + let nimexe = findNim() + exec nimexe & " cc --taintMode:on tests/testament/tester" # Since tests take a long time (on my machine), and we want to defy Murhpys # law - lets make sure the compiler really is freshly compiled! - exec "nim c --lib:lib -d:release --opt:speed compiler/nim.nim" + exec nimexe & " c --lib:lib -d:release --opt:speed compiler/nim.nim" let tester = quoteShell(getCurrentDir() / "tests/testament/tester".exe) let success = tryExec tester & " " & (args|"all") if not existsEnv("TRAVIS") and not existsEnv("APPVEYOR"): @@ -369,6 +433,7 @@ of cmdArgument: of "boot": boot(op.cmdLineRest) of "clean": clean(op.cmdLineRest) of "web": web(op.cmdLineRest) + of "json2": web("--json2 " & op.cmdLineRest) of "website": website(op.cmdLineRest & " --googleAnalytics:UA-48159761-1") of "web0": # undocumented command for Araq-the-merciful: @@ -380,6 +445,7 @@ of cmdArgument: of "nsis": nsis(op.cmdLineRest) of "geninstall": geninstall(op.cmdLineRest) of "install": install(op.cmdLineRest) + of "testinstall": testUnixInstall() of "test", "tests": tests(op.cmdLineRest) of "update": when defined(withUpdate): diff --git a/lib/core/locks.nim b/lib/core/locks.nim index 66e0ab520..fbe9c8acf 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -9,6 +9,7 @@ ## This module contains Nim's support for locks and condition vars. +const insideRLocksModule = false include "system/syslocks" type @@ -63,4 +64,4 @@ template withLock*(a: Lock, body: untyped) = try: body finally: - a.release() \ No newline at end of file + a.release() diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 4522e0fc6..4296cb0ae 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -12,7 +12,7 @@ include "system/inclrtl" ## This module contains the interface to the compiler's abstract syntax ## tree (`AST`:idx:). Macros operate on this tree. -## .. include:: ../doc/astspec.txt +## .. include:: ../../doc/astspec.txt type NimNodeKind* = enum @@ -197,6 +197,18 @@ proc typeKind*(n: NimNode): NimTypeKind {.magic: "NGetType", noSideEffect.} ## Returns the type kind of the node 'n' that should represent a type, that ## means the node should have been obtained via `getType`. +proc getTypeInst*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} + ## Like getType except it includes generic parameters for a specific instance + +proc getTypeInst*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} + ## Like getType except it includes generic parameters for a specific instance + +proc getTypeImpl*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} + ## Like getType except it includes generic parameters for the implementation + +proc getTypeImpl*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} + ## Like getType except it includes generic parameters for the implementation + proc strVal*(n: NimNode): string {.magic: "NStrVal", noSideEffect.} proc `intVal=`*(n: NimNode, val: BiggestInt) {.magic: "NSetIntVal", noSideEffect.} @@ -496,7 +508,7 @@ proc lispRepr*(n: NimNode): string {.compileTime, benign.} = add(result, ")") -macro dumpTree*(s: stmt): stmt {.immediate.} = echo s.treeRepr +macro dumpTree*(s: untyped): untyped = echo s.treeRepr ## Accepts a block of nim code and prints the parsed abstract syntax ## tree using the `toTree` function. Printing is done *at compile time*. ## @@ -504,17 +516,17 @@ macro dumpTree*(s: stmt): stmt {.immediate.} = echo s.treeRepr ## tree and to discover what kind of nodes must be created to represent ## a certain expression/statement. -macro dumpLisp*(s: stmt): stmt {.immediate.} = echo s.lispRepr +macro dumpLisp*(s: untyped): untyped = echo s.lispRepr ## Accepts a block of nim code and prints the parsed abstract syntax ## tree using the `toLisp` function. Printing is done *at compile time*. ## ## See `dumpTree`. -macro dumpTreeImm*(s: stmt): stmt {.immediate, deprecated.} = echo s.treeRepr - ## The ``immediate`` version of `dumpTree`. +macro dumpTreeImm*(s: untyped): untyped {.deprecated.} = echo s.treeRepr + ## Deprecated. -macro dumpLispImm*(s: stmt): stmt {.immediate, deprecated.} = echo s.lispRepr - ## The ``immediate`` version of `dumpLisp`. +macro dumpLispImm*(s: untyped): untyped {.deprecated.} = echo s.lispRepr + ## Deprecated. proc newEmptyNode*(): NimNode {.compileTime, noSideEffect.} = @@ -680,8 +692,16 @@ proc `pragma=`*(someProc: NimNode; val: NimNode){.compileTime.}= assert val.kind in {nnkEmpty, nnkPragma} someProc[4] = val +proc addPragma*(someProc, pragma: NimNode) {.compileTime.} = + ## Adds pragma to routine definition + someProc.expectRoutine + var pragmaNode = someProc.pragma + if pragmaNode.isNil or pragmaNode.kind == nnkEmpty: + pragmaNode = newNimNode(nnkPragma) + someProc.pragma = pragmaNode + pragmaNode.add(pragma) -template badNodeKind(k; f): stmt{.immediate.} = +template badNodeKind(k, f) = assert false, "Invalid node kind " & $k & " for macros.`" & $f & "`" proc body*(someProc: NimNode): NimNode {.compileTime.} = @@ -738,20 +758,19 @@ iterator children*(n: NimNode): NimNode {.inline.} = for i in 0 ..< n.len: yield n[i] -template findChild*(n: NimNode; cond: expr): NimNode {. - immediate, dirty.} = +template findChild*(n: NimNode; cond: untyped): NimNode {.dirty.} = ## Find the first child node matching condition (or nil). ## ## .. code-block:: nim ## var res = findChild(n, it.kind == nnkPostfix and ## it.basename.ident == !"foo") block: - var result: NimNode + var res: NimNode for it in n.children: if cond: - result = it + res = it break - result + res proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} = ## Insert node B into A at pos @@ -796,17 +815,17 @@ proc infix*(a: NimNode; op: string; proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] {. compileTime.} = node.expectKind nnkPostfix - result = (node[0], $node[1]) + result = (node[1], $node[0]) proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] {. compileTime.} = node.expectKind nnkPrefix - result = (node[0], $node[1]) + result = (node[1], $node[0]) proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; right: NimNode] {.compileTime.} = assert node.kind == nnkInfix - result = (node[0], $node[1], node[2]) + result = (node[1], $node[0], node[2]) proc copy*(node: NimNode): NimNode {.compileTime.} = ## An alias for copyNimTree(). @@ -818,6 +837,8 @@ proc cmpIgnoreStyle(a, b: cstring): int {.noSideEffect.} = else: result = c var i = 0 var j = 0 + # first char is case sensitive + if a[0] != b[0]: return 1 while true: while a[i] == '_': inc(i) while b[j] == '_': inc(j) # BUGFIX: typo @@ -828,9 +849,23 @@ proc cmpIgnoreStyle(a, b: cstring): int {.noSideEffect.} = inc(i) inc(j) -proc eqIdent* (a, b: string): bool = cmpIgnoreStyle(a, b) == 0 +proc eqIdent*(a, b: string): bool = cmpIgnoreStyle(a, b) == 0 ## Check if two idents are identical. +proc eqIdent*(node: NimNode; s: string): bool {.compileTime.} = + ## Check if node is some identifier node (``nnkIdent``, ``nnkSym``, etc.) + ## is the same as ``s``. Note that this is the preferred way to check! Most + ## other ways like ``node.ident`` are much more error-prone, unfortunately. + case node.kind + of nnkIdent: + result = node.ident == !s + of nnkSym: + result = eqIdent($node.symbol, s) + of nnkOpenSymChoice, nnkClosedSymChoice: + result = eqIdent($node[0], s) + else: + result = false + proc hasArgOfName* (params: NimNode; name: string): bool {.compiletime.}= ## Search nnkFormalParams for an argument. assert params.kind == nnkFormalParams @@ -856,7 +891,7 @@ proc boolVal*(n: NimNode): bool {.compileTime, noSideEffect.} = else: n == bindSym"true" # hacky solution for now when not defined(booting): - template emit*(e: static[string]): stmt = + template emit*(e: static[string]): stmt {.deprecated.} = ## accepts a single string argument and treats it as nim code ## that should be inserted verbatim in the program ## Example: @@ -864,6 +899,7 @@ when not defined(booting): ## .. code-block:: nim ## emit("echo " & '"' & "hello world".toUpper & '"') ## + ## Deprecated since version 0.15 since it's so rarely useful. macro payload: stmt {.gensym.} = result = parseStmt(e) payload() diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim index 14f04592b..4710d6cf1 100644 --- a/lib/core/rlocks.nim +++ b/lib/core/rlocks.nim @@ -9,6 +9,7 @@ ## This module contains Nim's support for reentrant locks. +const insideRLocksModule = true include "system/syslocks" type diff --git a/lib/deprecated/pure/ftpclient.nim b/lib/deprecated/pure/ftpclient.nim index 1188c0795..ed2f14450 100644 --- a/lib/deprecated/pure/ftpclient.nim +++ b/lib/deprecated/pure/ftpclient.nim @@ -129,10 +129,10 @@ proc ftpClient*(address: string, port = Port(21), result.csock = socket() if result.csock == invalidSocket: raiseOSError(osLastError()) -template blockingOperation(sock: Socket, body: stmt) {.immediate.} = +template blockingOperation(sock: Socket, body: untyped) = body -template blockingOperation(sock: asyncio.AsyncSocket, body: stmt) {.immediate.} = +template blockingOperation(sock: asyncio.AsyncSocket, body: untyped) = sock.setBlocking(true) body sock.setBlocking(false) diff --git a/lib/deprecated/pure/sockets.nim b/lib/deprecated/pure/sockets.nim index 20e6d9364..34c2e7a7d 100644 --- a/lib/deprecated/pure/sockets.nim +++ b/lib/deprecated/pure/sockets.nim @@ -711,8 +711,13 @@ proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = cint(sockets.AF_INET)) if s == nil: raiseOSError(osLastError()) else: - var s = posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, - cint(posix.AF_INET)) + var s = + when defined(android4): + posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, + cint(posix.AF_INET)) + else: + posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, + cint(posix.AF_INET)) if s == nil: raiseOSError(osLastError(), $hstrerror(h_errno)) diff --git a/lib/impure/db_odbc.nim b/lib/impure/db_odbc.nim index 3a14e6304..d6343acc7 100644 --- a/lib/impure/db_odbc.nim +++ b/lib/impure/db_odbc.nim @@ -210,7 +210,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string {. add(result, c) proc prepareFetch(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]) : TSqlSmallInt {. + args: varargs[string, `$`]): TSqlSmallInt {. tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = # Prepare a statement, execute it and fetch the data to the driver # ready for retrieval of the data @@ -222,9 +222,8 @@ proc prepareFetch(db: var DbConn, query: SqlQuery, var q = dbFormat(query, args) db.sqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)) db.sqlCheck(SQLExecute(db.stmt)) - var retcode = SQLFetch(db.stmt) - db.sqlCheck(retcode) - result=retcode + result = SQLFetch(db.stmt) + db.sqlCheck(result) proc prepareFetchDirect(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. @@ -250,8 +249,8 @@ proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool var rCnt = -1 res = SQLRowCount(db.stmt, rCnt) - if res != SQL_SUCCESS: dbError(db) properFreeResult(SQL_HANDLE_STMT, db.stmt) + if res != SQL_SUCCESS: dbError(db) except: discard return res == SQL_SUCCESS @@ -286,10 +285,10 @@ iterator fastRows*(db: var DbConn, query: SqlQuery, sz: TSqlSmallInt = 0 cCnt: TSqlSmallInt = 0.TSqlSmallInt res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt:TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - # tempcCnt,A field to store the number of temporary variables, for unknown reasons, + tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. + # tempcCnt,A field to store the number of temporary variables, for unknown reasons, # after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - # so the values of the temporary variable to store the cCnt. + # so the values of the temporary variable to store the cCnt. # After every cycle and specified to cCnt. To ensure the traversal of all fields. res = db.prepareFetch(query, args) if res == SQL_NO_DATA: @@ -308,8 +307,8 @@ iterator fastRows*(db: var DbConn, query: SqlQuery, cCnt = tempcCnt yield rowRes res = SQLFetch(db.stmt) - db.sqlCheck(res) properFreeResult(SQL_HANDLE_STMT, db.stmt) + db.sqlCheck(res) iterator instantRows*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow @@ -317,14 +316,14 @@ iterator instantRows*(db: var DbConn, query: SqlQuery, ## Same as fastRows but returns a handle that can be used to get column text ## on demand using []. Returned handle is valid only within the interator body. var - rowRes: Row + rowRes: Row = @[] sz: TSqlSmallInt = 0 cCnt: TSqlSmallInt = 0.TSqlSmallInt res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt:TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - # tempcCnt,A field to store the number of temporary variables, for unknown reasons, + tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. + # tempcCnt,A field to store the number of temporary variables, for unknown reasons, # after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - # so the values of the temporary variable to store the cCnt. + # so the values of the temporary variable to store the cCnt. # After every cycle and specified to cCnt. To ensure the traversal of all fields. res = db.prepareFetch(query, args) if res == SQL_NO_DATA: @@ -343,8 +342,8 @@ iterator instantRows*(db: var DbConn, query: SqlQuery, cCnt = tempcCnt yield (row: rowRes, len: cCnt.int) res = SQLFetch(db.stmt) - db.sqlCheck(res) properFreeResult(SQL_HANDLE_STMT, db.stmt) + db.sqlCheck(res) proc `[]`*(row: InstantRow, col: int): string {.inline.} = ## Returns text for given column of the row @@ -364,10 +363,10 @@ proc getRow*(db: var DbConn, query: SqlQuery, sz: TSqlSmallInt = 0.TSqlSmallInt cCnt: TSqlSmallInt = 0.TSqlSmallInt res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt:TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - ## tempcCnt,A field to store the number of temporary variables, for unknown reasons, + tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. + ## tempcCnt,A field to store the number of temporary variables, for unknown reasons, ## after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - ## so the values of the temporary variable to store the cCnt. + ## so the values of the temporary variable to store the cCnt. ## After every cycle and specified to cCnt. To ensure the traversal of all fields. res = db.prepareFetch(query, args) if res == SQL_NO_DATA: @@ -385,8 +384,8 @@ proc getRow*(db: var DbConn, query: SqlQuery, cCnt = tempcCnt res = SQLFetch(db.stmt) result = rowRes - db.sqlCheck(res) properFreeResult(SQL_HANDLE_STMT, db.stmt) + db.sqlCheck(res) proc getAllRows*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): seq[Row] {. @@ -398,10 +397,10 @@ proc getAllRows*(db: var DbConn, query: SqlQuery, sz: TSqlSmallInt = 0 cCnt: TSqlSmallInt = 0.TSqlSmallInt res: TSqlSmallInt = 0.TSqlSmallInt - tempcCnt:TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. - ## tempcCnt,A field to store the number of temporary variables, for unknown reasons, + tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled. + ## tempcCnt,A field to store the number of temporary variables, for unknown reasons, ## after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0, - ## so the values of the temporary variable to store the cCnt. + ## so the values of the temporary variable to store the cCnt. ## After every cycle and specified to cCnt. To ensure the traversal of all fields. res = db.prepareFetch(query, args) if res == SQL_NO_DATA: @@ -421,8 +420,8 @@ proc getAllRows*(db: var DbConn, query: SqlQuery, rows.add(rowRes) res = SQLFetch(db.stmt) result = rows - db.sqlCheck(res) properFreeResult(SQL_HANDLE_STMT, db.stmt) + db.sqlCheck(res) iterator rows*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {. @@ -544,4 +543,4 @@ proc setEncoding*(connection: DbConn, encoding: string): bool {. ## Sets the encoding of a database connection, returns true for ## success, false for failure. ##result = set_character_set(connection, encoding) == 0 - dbError("setEncoding() is currently not implemented by the db_odbc module") \ No newline at end of file + dbError("setEncoding() is currently not implemented by the db_odbc module") diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index c8f690461..557bb0549 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -15,6 +15,8 @@ from math import ceil import options from unicode import runeLenAt +export options + ## What is NRE? ## ============ @@ -24,46 +26,34 @@ from unicode import runeLenAt ## Licencing ## --------- ## -## PCRE has some additional terms that you must comply with if you use this module.:: +## PCRE has `some additional terms`_ that you must agree to in order to use +## this module. +## +## .. _`some additional terms`: http://pcre.sourceforge.net/license.txt +## +## Example +## ------- +## +## .. code-block:: nim +## +## import nre ## -## > Copyright (c) 1997-2001 University of Cambridge -## > -## > Permission is granted to anyone to use this software for any purpose on any -## > computer system, and to redistribute it freely, subject to the following -## > restrictions: -## > -## > 1. This software is distributed in the hope that it will be useful, -## > but WITHOUT ANY WARRANTY; without even the implied warranty of -## > MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -## > -## > 2. The origin of this software must not be misrepresented, either by -## > explicit claim or by omission. In practice, this means that if you use -## > PCRE in software that you distribute to others, commercially or -## > otherwise, you must put a sentence like this -## > -## > Regular expression support is provided by the PCRE library package, -## > which is open source software, written by Philip Hazel, and copyright -## > by the University of Cambridge, England. -## > -## > somewhere reasonably visible in your documentation and in any relevant -## > files or online help data or similar. A reference to the ftp site for -## > the source, that is, to -## > -## > ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/ -## > -## > should also be given in the documentation. However, this condition is not -## > intended to apply to whole chains of software. If package A includes PCRE, -## > it must acknowledge it, but if package B is software that includes package -## > A, the condition is not imposed on package B (unless it uses PCRE -## > independently). -## > -## > 3. Altered versions must be plainly marked as such, and must not be -## > misrepresented as being the original software. -## > -## > 4. If PCRE is embedded in any software that is released under the GNU -## > General Purpose Licence (GPL), or Lesser General Purpose Licence (LGPL), -## > then the terms of that licence shall supersede any condition above with -## > which it is incompatible. +## let vowels = re"[aeoui]" +## +## for match in "moigagoo".findIter(vowels): +## echo match.matchBounds +## # (a: 1, b: 1) +## # (a: 2, b: 2) +## # (a: 4, b: 4) +## # (a: 6, b: 6) +## # (a: 7, b: 7) +## +## let firstVowel = "foo".find(vowels) +## let hasVowel = firstVowel.isSome() +## if hasVowel: +## let matchBounds = firstVowel.get().captureBounds[-1] +## echo "first vowel @", matchBounds.get().a +## # first vowel @1 # Type definitions {{{ @@ -125,11 +115,11 @@ type ## - ``(*NO_STUDY)`` - turn off studying; study is enabled by default ## ## For more details on the leading option groups, see the `Option - ## Setting <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#OPTION_SETTING>`__ + ## Setting <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#OPTION_SETTING>`_ ## and the `Newline - ## Convention <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#NEWLINE_CONVENTION>`__ + ## Convention <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#NEWLINE_CONVENTION>`_ ## sections of the `PCRE syntax - ## manual <http://man7.org/linux/man-pages/man3/pcresyntax.3.html>`__. + ## manual <http://man7.org/linux/man-pages/man3/pcresyntax.3.html>`_. pattern*: string ## not nil pcreObj: ptr pcre.Pcre ## not nil pcreExtra: ptr pcre.ExtraData ## nil @@ -284,7 +274,7 @@ proc `[]`*(pattern: Captures, name: string): string = let pattern = RegexMatch(pattern) return pattern.captures[pattern.pattern.captureNameToId.fget(name)] -template toTableImpl(cond: bool): stmt {.immediate, dirty.} = +template toTableImpl(cond: untyped) {.dirty.} = for key in RegexMatch(pattern).pattern.captureNameId.keys: let nextVal = pattern[key] if cond: @@ -301,7 +291,7 @@ proc toTable*(pattern: CaptureBounds, default = none(Slice[int])): result = initTable[string, Option[Slice[int]]]() toTableImpl(nextVal.isNone) -template itemsImpl(cond: bool): stmt {.immediate, dirty.} = +template itemsImpl(cond: untyped) {.dirty.} = for i in 0 .. <RegexMatch(pattern).pattern.captureCount: let nextVal = pattern[i] # done in this roundabout way to avoid multiple yields (potential code @@ -493,17 +483,17 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt raise RegexInternalError(msg : "Unknown internal error: " & $execRet) proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] = - ## Like ```find(...)`` <#proc-find>`__, but anchored to the start of the + ## Like ```find(...)`` <#proc-find>`_, but anchored to the start of the ## string. This means that ``"foo".match(re"f") == true``, but ## ``"foo".match(re"o") == false``. return str.matchImpl(pattern, start, endpos, pcre.ANCHORED) iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): RegexMatch = - ## Works the same as ```find(...)`` <#proc-find>`__, but finds every + ## Works the same as ```find(...)`` <#proc-find>`_, but finds every ## non-overlapping match. ``"2222".find(re"22")`` is ``"22", "22"``, not ## ``"22", "22", "22"``. ## - ## Arguments are the same as ```find(...)`` <#proc-find>`__ + ## Arguments are the same as ```find(...)`` <#proc-find>`_ ## ## Variants: ## @@ -566,6 +556,16 @@ proc findAll*(str: string, pattern: Regex, start = 0, endpos = int.high): seq[st for match in str.findIter(pattern, start, endpos): result.add(match.match) +proc contains*(str: string, pattern: Regex, start = 0, endpos = int.high): bool = + ## Determine if the string contains the given pattern between the end and + ## start positions: + ## - "abc".contains(re"bc") == true + ## - "abc".contains(re"cd") == false + ## - "abc".contains(re"a", start = 1) == false + ## + ## Same as ``isSome(str.find(pattern, start, endpos))``. + return isSome(str.find(pattern, start, endpos)) + proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] = ## Splits the string with the given regex. This works according to the ## rules that Perl and Javascript use: @@ -581,7 +581,7 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] ## strings in the output seq. ## ``"1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"]`` ## - ## ``start`` behaves the same as in ```find(...)`` <#proc-find>`__. + ## ``start`` behaves the same as in ```find(...)`` <#proc-find>`_. result = @[] var lastIdx = start var splits = 0 @@ -625,7 +625,7 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] result.add(str.substr(bounds.b + 1, str.high)) template replaceImpl(str: string, pattern: Regex, - replacement: expr): stmt {.immediate, dirty.} = + replacement: untyped) {.dirty.} = # XXX seems very similar to split, maybe I can reduce code duplication # somehow? result = "" diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index 469bb69c5..f722a6b39 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -105,7 +105,8 @@ else: proc readLineFromStdin*(prompt: string): TaintedString {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) + if isNil(buffer): + raise newException(IOError, "Linenoise returned nil") result = TaintedString($buffer) if result.string.len > 0: historyAdd(buffer) @@ -114,12 +115,12 @@ else: proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) + if isNil(buffer): + raise newException(IOError, "Linenoise returned nil") line = TaintedString($buffer) if line.string.len > 0: historyAdd(buffer) linenoise.free(buffer) - # XXX how to determine CTRL+D? result = true proc readPasswordFromStdin*(prompt: string, password: var TaintedString): diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 60bb6c77f..bf397550a 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -7,8 +7,11 @@ # distribution, for details about the copyright. # -## Regular expression support for Nim. Deprecated. Consider using the ``nre`` -## or ``pegs`` modules instead. +## Regular expression support for Nim. This module still has some +## obscure bugs and limitations, +## consider using the ``nre`` or ``pegs`` modules instead. +## We had to de-deprecate this module since too much code relies on it +## and many people prefer its API over ``nre``'s. ## ## **Note:** The 're' proc defaults to the **extended regular expression ## syntax** which lets you use whitespace freely to make your regexes readable. @@ -22,14 +25,12 @@ ## though. ## PRCE's licence follows: ## -## .. include:: ../doc/regexprs.txt +## .. include:: ../../doc/regexprs.txt ## import pcre, strutils, rtarrays -{.deprecated.} - const MaxSubpatterns* = 20 ## defines the maximum number of subpatterns that can be captured. @@ -78,7 +79,7 @@ proc finalizeRegEx(x: Regex) = if not isNil(x.e): pcre.free_substring(cast[cstring](x.e)) -proc re*(s: string, flags = {reExtended, reStudy}): Regex {.deprecated.} = +proc re*(s: string, flags = {reExtended, reStudy}): Regex = ## Constructor of regular expressions. Note that Nim's ## extended raw string literals support this syntax ``re"[abc]"`` as ## a short form for ``re(r"[abc]")``. diff --git a/lib/impure/ssl.nim b/lib/impure/ssl.nim index 721e5ce51..e3312d792 100644 --- a/lib/impure/ssl.nim +++ b/lib/impure/ssl.nim @@ -9,6 +9,9 @@ ## This module provides an easy to use sockets-style ## nim interface to the OpenSSL library. +## +## **Warning:** This module is deprecated, use the SSL procedures defined in +## the ``net`` module instead. {.deprecated.} diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 11df959d7..5104712e8 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -402,7 +402,9 @@ proc routeEvent*(w: Window, event: Event) proc scrollBy*(w: Window, x, y: int) proc scrollTo*(w: Window, x, y: int) proc setInterval*(w: Window, code: cstring, pause: int): ref TInterval +proc setInterval*(w: Window, function: proc (), pause: int): ref TInterval proc setTimeout*(w: Window, code: cstring, pause: int): ref TTimeOut +proc setTimeout*(w: Window, function: proc (), pause: int): ref TInterval proc stop*(w: Window) # Node "methods" @@ -481,6 +483,9 @@ proc getAttribute*(s: Style, attr: cstring, caseSensitive=false): cstring proc removeAttribute*(s: Style, attr: cstring, caseSensitive=false) proc setAttribute*(s: Style, attr, value: cstring, caseSensitive=false) +# Event "methods" +proc preventDefault*(ev: Event) + {.pop.} var diff --git a/lib/nimbase.h b/lib/nimbase.h index 5a4f403b6..37f8f5c3d 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -44,7 +44,7 @@ __clang__ #if defined(_MSC_VER) # pragma warning(disable: 4005 4100 4101 4189 4191 4200 4244 4293 4296 4309) # pragma warning(disable: 4310 4365 4456 4477 4514 4574 4611 4668 4702 4706) -# pragma warning(disable: 4710 4711 4774 4800 4820 4996) +# pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090) #endif /* ------------------------------------------------------------------------- */ @@ -102,7 +102,7 @@ __clang__ defined __ICL || \ defined __DMC__ || \ defined __BORLANDC__ ) -# define NIM_THREADVAR __declspec(thread) +# define NIM_THREADVAR __declspec(thread) /* note that ICC (linux) and Clang are covered by __GNUC__ */ #elif defined __GNUC__ || \ defined __SUNPRO_C || \ @@ -222,6 +222,8 @@ __clang__ /* ----------------------------------------------------------------------- */ +#define COMMA , + #include <limits.h> #include <stddef.h> @@ -345,9 +347,6 @@ static N_INLINE(NI32, float32ToInt32)(float x) { #define float64ToInt64(x) ((NI64) (x)) -#define zeroMem(a, size) memset(a, 0, size) -#define equalMem(a, b, size) (memcmp(a, b, size) == 0) - #define STRING_LITERAL(name, str, length) \ static const struct { \ TGenericSeq Sup; \ @@ -450,8 +449,8 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); } /* Test to see if Nim and the C compiler agree on the size of a pointer. On disagreement, your C compiler will say something like: - "error: 'assert_numbits' declared as an array with a negative size" */ -typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; + "error: 'Nim_and_C_compiler_disagree_on_target_architecture' declared as an array with a negative size" */ +typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; #endif #ifdef __cplusplus diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 1bc0af1b6..9de25f82b 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -31,13 +31,14 @@ type state: TokenClass SourceLanguage* = enum - langNone, langNim, langNimrod, langCpp, langCsharp, langC, langJava + langNone, langNim, langNimrod, langCpp, langCsharp, langC, langJava, + langYaml {.deprecated: [TSourceLanguage: SourceLanguage, TTokenClass: TokenClass, TGeneralTokenizer: GeneralTokenizer].} const sourceLanguageToStr*: array[SourceLanguage, string] = ["none", - "Nim", "Nimrod", "C++", "C#", "C", "Java"] + "Nim", "Nimrod", "C++", "C#", "C", "Java", "Yaml"] tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace", "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber", "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit", @@ -578,6 +579,309 @@ proc javaNextToken(g: var GeneralTokenizer) = "try", "void", "volatile", "while"] clikeNextToken(g, keywords, {}) +proc yamlPlainStrLit(g: var GeneralTokenizer, pos: var int) = + g.kind = gtStringLit + while g.buf[pos] notin {'\0', '\x09'..'\x0D', ',', ']', '}'}: + if g.buf[pos] == ':' and + g.buf[pos + 1] in {'\0', '\x09'..'\x0D', ' '}: + break + inc(pos) + +proc yamlPossibleNumber(g: var GeneralTokenizer, pos: var int) = + g.kind = gtNone + if g.buf[pos] == '-': inc(pos) + if g.buf[pos] == '0': inc(pos) + elif g.buf[pos] in '1'..'9': + inc(pos) + while g.buf[pos] in {'0'..'9'}: inc(pos) + else: yamlPlainStrLit(g, pos) + if g.kind == gtNone: + if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}: + g.kind = gtDecNumber + elif g.buf[pos] == '.': + inc(pos) + if g.buf[pos] notin {'0'..'9'}: yamlPlainStrLit(g, pos) + else: + while g.buf[pos] in {'0'..'9'}: inc(pos) + if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}: + g.kind = gtFloatNumber + if g.kind == gtNone: + if g.buf[pos] in {'e', 'E'}: + inc(pos) + if g.buf[pos] in {'-', '+'}: inc(pos) + if g.buf[pos] notin {'0'..'9'}: yamlPlainStrLit(g, pos) + else: + while g.buf[pos] in {'0'..'9'}: inc(pos) + if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}: + g.kind = gtFloatNumber + else: yamlPlainStrLit(g, pos) + else: yamlPlainStrLit(g, pos) + while g.buf[pos] notin {'\0', ',', ']', '}', '\x0A', '\x0D'}: + inc(pos) + if g.buf[pos] notin {'\x09'..'\x0D', ' ', ',', ']', '}'}: + yamlPlainStrLit(g, pos) + break + # theoretically, we would need to parse indentation (like with block scalars) + # because of possible multiline flow scalars that start with number-like + # content, but that is far too troublesome. I think it is fine that the + # highlighter is sloppy here. + +proc yamlNextToken(g: var GeneralTokenizer) = + const + hexChars = {'0'..'9', 'A'..'F', 'a'..'f'} + var pos = g.pos + g.start = g.pos + if g.state == gtStringLit: + g.kind = gtStringLit + while true: + case g.buf[pos] + of '\\': + if pos != g.pos: break + g.kind = gtEscapeSequence + inc(pos) + case g.buf[pos] + of 'x': + inc(pos) + for i in 1..2: + {.unroll.} + if g.buf[pos] in hexChars: inc(pos) + break + of 'u': + inc(pos) + for i in 1..4: + {.unroll.} + if g.buf[pos] in hexChars: inc(pos) + break + of 'U': + inc(pos) + for i in 1..8: + {.unroll.} + if g.buf[pos] in hexChars: inc(pos) + break + else: inc(pos) + break + of '\0': + g.state = gtOther + break + of '\"': + inc(pos) + g.state = gtOther + break + else: inc(pos) + elif g.state == gtCharLit: + # abusing gtCharLit as single-quoted string lit + g.kind = gtStringLit + inc(pos) # skip the starting ' + while true: + case g.buf[pos] + of '\'': + inc(pos) + if g.buf[pos] == '\'': + inc(pos) + g.kind = gtEscapeSequence + else: g.state = gtOther + break + else: inc(pos) + elif g.state == gtCommand: + # gtCommand means 'block scalar header' + case g.buf[pos] + of ' ', '\t': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\t'}: inc(pos) + of '#': + g.kind = gtComment + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + of '\x0A', '\x0D': discard + else: + # illegal here. just don't parse a block scalar + g.kind = gtNone + g.state = gtOther + if g.buf[pos] in {'\x0A', '\x0D'} and g.state == gtCommand: + g.state = gtLongStringLit + elif g.state == gtLongStringLit: + # beware, this is the only token where we actually have to parse + # indentation. + + g.kind = gtLongStringLit + # first, we have to find the parent indentation of the block scalar, so that + # we know when to stop + assert g.buf[pos] in {'\x0A', '\x0D'} + var lookbehind = pos - 1 + var headerStart = -1 + while lookbehind >= 0 and g.buf[lookbehind] notin {'\x0A', '\x0D'}: + if headerStart == -1 and g.buf[lookbehind] in {'|', '>'}: + headerStart = lookbehind + dec(lookbehind) + assert headerStart != -1 + var indentation = 1 + while g.buf[lookbehind + indentation] == ' ': inc(indentation) + if g.buf[lookbehind + indentation] in {'|', '>'}: + # when the header is alone in a line, this line does not show the parent's + # indentation, so we must go further. search the first previous line with + # non-whitespace content. + while lookbehind >= 0 and g.buf[lookbehind] in {'\x0A', '\x0D'}: + dec(lookbehind) + while lookbehind >= 0 and + g.buf[lookbehind] in {' ', '\t'}: dec(lookbehind) + # now, find the beginning of the line... + while lookbehind >= 0 and g.buf[lookbehind] notin {'\x0A', '\x0D'}: + dec(lookbehind) + # ... and its indentation + indentation = 1 + while g.buf[lookbehind + indentation] == ' ': inc(indentation) + if lookbehind == -1: indentation = 0 # top level + elif g.buf[lookbehind + 1] == '-' and g.buf[lookbehind + 2] == '-' and + g.buf[lookbehind + 3] == '-' and + g.buf[lookbehind + 4] in {'\x09'..'\x0D', ' '}: + # this is a document start, therefore, we are at top level + indentation = 0 + # because lookbehind was at newline char when calculating indentation, we're + # off by one. fix that. top level's parent will have indentation of -1. + let parentIndentation = indentation - 1 + + # find first content + while g.buf[pos] in {' ', '\x0A', '\x0D'}: + if g.buf[pos] == ' ': inc(indentation) + else: indentation = 0 + inc(pos) + var minIndentation = indentation + + # for stupid edge cases, we must check whether an explicit indentation depth + # is given at the header. + while g.buf[headerStart] in {'>', '|', '+', '-'}: inc(headerStart) + if g.buf[headerStart] in {'0'..'9'}: + minIndentation = min(minIndentation, ord(g.buf[headerStart]) - ord('0')) + + # process content lines + while indentation > parentIndentation and g.buf[pos] != '\0': + if (indentation < minIndentation and g.buf[pos] == '#') or + (indentation == 0 and g.buf[pos] == '.' and g.buf[pos + 1] == '.' and + g.buf[pos + 2] == '.' and + g.buf[pos + 3] in {'\0', '\x09'..'\x0D', ' '}): + # comment after end of block scalar, or end of document + break + minIndentation = min(indentation, minIndentation) + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + while g.buf[pos] in {' ', '\x0A', '\x0D'}: + if g.buf[pos] == ' ': inc(indentation) + else: indentation = 0 + inc(pos) + + g.state = gtOther + elif g.state == gtOther: + # gtOther means 'inside YAML document' + case g.buf[pos] + of ' ', '\x09'..'\x0D': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) + of '#': + g.kind = gtComment + inc(pos) + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + of '-': + inc(pos) + if g.buf[pos] in {'\0', ' ', '\x09'..'\x0D'}: + g.kind = gtPunctuation + elif g.buf[pos] == '-' and + (pos == 1 or g.buf[pos - 2] in {'\x0A', '\x0D'}): # start of line + inc(pos) + if g.buf[pos] == '-' and g.buf[pos + 1] in {'\0', '\x09'..'\x0D', ' '}: + inc(pos) + g.kind = gtKeyword + else: yamlPossibleNumber(g, pos) + else: yamlPossibleNumber(g, pos) + of '.': + if pos == 0 or g.buf[pos - 1] in {'\x0A', '\x0D'}: + inc(pos) + for i in 1..2: + {.unroll.} + if g.buf[pos] != '.': break + inc(pos) + if pos == g.start + 3: + g.kind = gtKeyword + g.state = gtNone + else: yamlPlainStrLit(g, pos) + else: yamlPlainStrLit(g, pos) + of '?': + inc(pos) + if g.buf[pos] in {'\0', ' ', '\x09'..'\x0D'}: + g.kind = gtPunctuation + else: yamlPlainStrLit(g, pos) + of ':': + inc(pos) + if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', '\'', '\"'} or + (pos > 0 and g.buf[pos - 2] in {'}', ']', '\"', '\''}): + g.kind = gtPunctuation + else: yamlPlainStrLit(g, pos) + of '[', ']', '{', '}', ',': + inc(pos) + g.kind = gtPunctuation + of '\"': + inc(pos) + g.state = gtStringLit + g.kind = gtStringLit + of '\'': + g.state = gtCharLit + g.kind = gtNone + of '!': + g.kind = gtTagStart + inc(pos) + if g.buf[pos] == '<': + # literal tag (e.g. `!<tag:yaml.org,2002:str>`) + while g.buf[pos] notin {'\0', '>', '\x09'..'\x0D', ' '}: inc(pos) + if g.buf[pos] == '>': inc(pos) + else: + while g.buf[pos] in {'A'..'Z', 'a'..'z', '0'..'9', '-'}: inc(pos) + case g.buf[pos] + of '!': + # prefixed tag (e.g. `!!str`) + inc(pos) + while g.buf[pos] notin + {'\0', '\x09'..'\x0D', ' ', ',', '[', ']', '{', '}'}: inc(pos) + of '\0', '\x09'..'\x0D', ' ': discard + else: + # local tag (e.g. `!nim:system:int`) + while g.buf[pos] notin {'\0', '\x09'..'\x0D', ' '}: inc(pos) + of '&': + g.kind = gtLabel + while g.buf[pos] notin {'\0', '\x09'..'\x0D', ' '}: inc(pos) + of '*': + g.kind = gtReference + while g.buf[pos] notin {'\0', '\x09'..'\x0D', ' '}: inc(pos) + of '|', '>': + # this can lead to incorrect tokenization when | or > appear inside flow + # content. checking whether we're inside flow content is not + # chomsky type-3, so we won't do that here. + g.kind = gtCommand + g.state = gtCommand + inc(pos) + while g.buf[pos] in {'0'..'9', '+', '-'}: inc(pos) + of '0'..'9': yamlPossibleNumber(g, pos) + of '\0': g.kind = gtEOF + else: yamlPlainStrLit(g, pos) + else: + # outside document + case g.buf[pos] + of '%': + if pos == 0 or g.buf[pos - 1] in {'\x0A', '\x0D'}: + g.kind = gtDirective + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + else: + g.state = gtOther + yamlPlainStrLit(g, pos) + of ' ', '\x09'..'\x0D': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) + of '#': + g.kind = gtComment + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + of '\0': g.kind = gtEOF + else: + g.kind = gtNone + g.state = gtOther + g.length = pos - g.pos + g.pos = pos + proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) = case lang of langNone: assert false @@ -586,6 +890,7 @@ proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) = of langCsharp: csharpNextToken(g) of langC: cNextToken(g) of langJava: javaNextToken(g) + of langYaml: yamlNextToken(g) when isMainModule: var keywords: seq[string] diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index e1d5f902e..53699166f 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -49,7 +49,7 @@ type TMsgKind: MsgKind].} const - messages: array [MsgKind, string] = [ + messages: array[MsgKind, string] = [ meCannotOpenFile: "cannot open '$1'", meExpected: "'$1' expected", meGridTableNotImplemented: "grid table is not implemented", @@ -323,6 +323,11 @@ proc newSharedState(options: RstParseOptions, result.msgHandler = if not isNil(msgHandler): msgHandler else: defaultMsgHandler result.findFile = if not isNil(findFile): findFile else: defaultFindFile +proc findRelativeFile(p: RstParser; filename: string): string = + result = p.filename.splitFile.dir / filename + if not existsFile(result): + result = p.s.findFile(filename) + proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = p.s.msgHandler(p.filename, p.line + p.tok[p.idx].line, p.col + p.tok[p.idx].col, msgKind, arg) @@ -1500,7 +1505,7 @@ proc dirInclude(p: var RstParser): PRstNode = result = nil var n = parseDirective(p, {hasArg, argIsFile, hasOptions}, nil) var filename = strip(addNodes(n.sons[0])) - var path = p.s.findFile(filename) + var path = p.findRelativeFile(filename) if path == "": rstMessage(p, meCannotOpenFile, filename) else: @@ -1511,7 +1516,7 @@ proc dirInclude(p: var RstParser): PRstNode = else: var q: RstParser initParser(q, p.s) - q.filename = filename + q.filename = path q.col += getTokens(readFile(path), false, q.tok) # workaround a GCC bug; more like the interior pointer bug? #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0: @@ -1538,7 +1543,7 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = result = parseDirective(p, {hasArg, hasOptions}, parseLiteralBlock) var filename = strip(getFieldValue(result, "file")) if filename != "": - var path = p.s.findFile(filename) + var path = p.findRelativeFile(filename) if path == "": rstMessage(p, meCannotOpenFile, filename) var n = newRstNode(rnLiteralBlock) add(n, newRstNode(rnLeaf, readFile(path))) @@ -1590,7 +1595,7 @@ proc dirRawAux(p: var RstParser, result: var PRstNode, kind: RstNodeKind, contentParser: SectionParser) = var filename = getFieldValue(result, "file") if filename.len > 0: - var path = p.s.findFile(filename) + var path = p.findRelativeFile(filename) if path.len == 0: rstMessage(p, meCannotOpenFile, filename) else: diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 4c7b817cb..bd69b2328 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -35,6 +35,15 @@ const hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays hasAioH = defined(linux) +when defined(linux): + # On Linux: + # timer_{create,delete,settime,gettime}, + # clock_{getcpuclockid, getres, gettime, nanosleep, settime} lives in librt + {.passL: "-lrt".} +when defined(solaris): + # On Solaris hstrerror lives in libresolv + {.passL: "-lresolv".} + when false: const C_IRUSR = 0c000400 ## Read by owner. @@ -101,7 +110,7 @@ type ## (not POSIX) when defined(linux) or defined(bsd): d_off*: Off ## Not an offset. Value that ``telldir()`` would return. - d_name*: array [0..255, char] ## Name of entry. + d_name*: array[0..255, char] ## Name of entry. Tflock* {.importc: "struct flock", final, pure, header: "<fcntl.h>".} = object ## flock type @@ -233,7 +242,7 @@ type ## network to which this node is attached, if any. release*, ## Current release level of this implementation. version*, ## Current version level of this release. - machine*: array [0..255, char] ## Name of the hardware type on which the + machine*: array[0..255, char] ## Name of the hardware type on which the ## system is running. Sem* {.importc: "sem_t", header: "<semaphore.h>", final, pure.} = object @@ -439,6 +448,14 @@ when hasSpawnH: Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", header: "<spawn.h>", final, pure.} = object +when defined(linux): + # from sys/un.h + const Sockaddr_un_path_length* = 108 +else: + # according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html + # this is >=92 + const Sockaddr_un_path_length* = 92 + type Socklen* {.importc: "socklen_t", header: "<sys/socket.h>".} = cuint TSa_Family* {.importc: "sa_family_t", header: "<sys/socket.h>".} = cint @@ -446,7 +463,12 @@ type SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", pure, final.} = object ## struct sockaddr sa_family*: TSa_Family ## Address family. - sa_data*: array [0..255, char] ## Socket address (variable-length data). + sa_data*: array[0..255, char] ## Socket address (variable-length data). + + Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>", + pure, final.} = object ## struct sockaddr_un + sun_family*: TSa_Family ## Address family. + sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path Sockaddr_storage* {.importc: "struct sockaddr_storage", header: "<sys/socket.h>", @@ -504,7 +526,7 @@ type In6Addr* {.importc: "struct in6_addr", pure, final, header: "<netinet/in.h>".} = object ## struct in6_addr - s6_addr*: array [0..15, char] + s6_addr*: array[0..15, char] Sockaddr_in6* {.importc: "struct sockaddr_in6", pure, final, header: "<netinet/in.h>".} = object ## struct sockaddr_in6 @@ -1604,6 +1626,16 @@ else: var MAP_POPULATE*: cint = 0 +when defined(linux) or defined(nimdoc): + when defined(alpha) or defined(mips) or defined(parisc) or + defined(sparc) or defined(nimdoc): + const SO_REUSEPORT* = cint(0x0200) + ## Multiple binding: load balancing on incoming TCP connections + ## or UDP packets. (Requires Linux kernel > 3.9) + else: + const SO_REUSEPORT* = cint(15) +else: + var SO_REUSEPORT* {.importc, header: "<sys/socket.h>".}: cint when defined(macosx): # We can't use the NOSIGNAL flag in the ``send`` function, it has no effect @@ -1612,6 +1644,10 @@ when defined(macosx): MSG_NOSIGNAL* = 0'i32 var SO_NOSIGPIPE* {.importc, header: "<sys/socket.h>".}: cint +elif defined(solaris): + # Solaris dont have MSG_NOSIGNAL + const + MSG_NOSIGNAL* = 0'i32 else: var MSG_NOSIGNAL* {.importc, header: "<sys/socket.h>".}: cint @@ -2309,9 +2345,9 @@ proc strftime*(a1: cstring, a2: int, a3: cstring, a4: var Tm): int {.importc, header: "<time.h>".} proc strptime*(a1, a2: cstring, a3: var Tm): cstring {.importc, header: "<time.h>".} proc time*(a1: var Time): Time {.importc, header: "<time.h>".} -proc timer_create*(a1: var ClockId, a2: var SigEvent, +proc timer_create*(a1: ClockId, a2: var SigEvent, a3: var Timer): cint {.importc, header: "<time.h>".} -proc timer_delete*(a1: var Timer): cint {.importc, header: "<time.h>".} +proc timer_delete*(a1: Timer): cint {.importc, header: "<time.h>".} proc timer_gettime*(a1: Timer, a2: var Itimerspec): cint {. importc, header: "<time.h>".} proc timer_getoverrun*(a1: Timer): cint {.importc, header: "<time.h>".} @@ -2357,8 +2393,14 @@ proc sigrelse*(a1: cint): cint {.importc, header: "<signal.h>".} proc sigset*(a1: int, a2: proc (x: cint) {.noconv.}) {. importc, header: "<signal.h>".} proc sigsuspend*(a1: var Sigset): cint {.importc, header: "<signal.h>".} -proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, + +when defined(android): + proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, + a3: var Timespec, sigsetsize: csize = sizeof(culong)*2): cint {.importc: "__rt_sigtimedwait", header:"<signal.h>".} +else: + proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, a3: var Timespec): cint {.importc, header: "<signal.h>".} + proc sigwait*(a1: var Sigset, a2: var cint): cint {. importc, header: "<signal.h>".} proc sigwaitinfo*(a1: var Sigset, a2: var SigInfo): cint {. @@ -2574,8 +2616,12 @@ proc gai_strerror*(a1: cint): cstring {.importc:"(char *)$1", header: "<netdb.h> proc getaddrinfo*(a1, a2: cstring, a3: ptr AddrInfo, a4: var ptr AddrInfo): cint {.importc, header: "<netdb.h>".} -proc gethostbyaddr*(a1: pointer, a2: Socklen, a3: cint): ptr Hostent {. - importc, header: "<netdb.h>".} +when not defined(android4): + proc gethostbyaddr*(a1: pointer, a2: Socklen, a3: cint): ptr Hostent {. + importc, header: "<netdb.h>".} +else: + proc gethostbyaddr*(a1: cstring, a2: cint, a3: cint): ptr Hostent {. + importc, header: "<netdb.h>".} proc gethostbyname*(a1: cstring): ptr Hostent {.importc, header: "<netdb.h>".} proc gethostent*(): ptr Hostent {.importc, header: "<netdb.h>".} @@ -2607,7 +2653,7 @@ proc poll*(a1: ptr TPollfd, a2: Tnfds, a3: int): cint {. proc realpath*(name, resolved: cstring): cstring {. importc: "realpath", header: "<stdlib.h>".} -proc utimes*(path: cstring, times: ptr array [2, Timeval]): int {. +proc utimes*(path: cstring, times: ptr array[2, Timeval]): int {. importc: "utimes", header: "<sys/time.h>".} ## Sets file access and modification times. ## @@ -2618,3 +2664,17 @@ proc utimes*(path: cstring, times: ptr array [2, Timeval]): int {. ## Returns zero on success. ## ## For more information read http://www.unix.com/man-page/posix/3/utimes/. + +proc handle_signal(sig: cint, handler: proc (a: cint) {.noconv.}) {.importc: "signal", header: "<signal.h>".} + +template onSignal*(signals: varargs[cint], body: untyped) = + ## Setup code to be executed when Unix signals are received. Example: + ## from posix import SIGINT, SIGTERM + ## onSignal(SIGINT, SIGTERM): + ## echo "bye" + + for s in signals: + handle_signal(s, + proc (sig: cint) {.noconv.} = + body + ) diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index c0acae138..b83daf245 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -114,7 +114,7 @@ proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int {.closure.}) proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) proc merge[T](a, b: var openArray[T], lo, m, hi: int, cmp: proc (x, y: T): int {.closure.}, order: SortOrder) = - template `<-` (a, b: expr) = + template `<-` (a, b) = when false: a = b elif onlySafeCode: @@ -206,7 +206,7 @@ proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, result[i] = a[i] sort(result, cmp, order) -template sortedByIt*(seq1, op: expr): expr = +template sortedByIt*(seq1, op: untyped): untyped = ## Convenience template around the ``sorted`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an @@ -231,7 +231,7 @@ template sortedByIt*(seq1, op: expr): expr = ## ## echo people.sortedByIt((it.age, it.name)) ## - var result {.gensym.} = sorted(seq1, proc(x, y: type(seq1[0])): int = + var result = sorted(seq1, proc(x, y: type(seq1[0])): int = var it {.inject.} = x let a = op it = y diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index cc337452f..79bc1b96d 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -9,9 +9,9 @@ include "system/inclrtl" -import os, oids, tables, strutils, macros, times +import os, oids, tables, strutils, macros, times, heapqueue -import nativesockets, net +import nativesockets, net, queues export Port, SocketFlag @@ -155,6 +155,9 @@ type when not defined(release): var currentID = 0 + +proc callSoon*(cbproc: proc ()) {.gcsafe.} + proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = ## Creates a new future. ## @@ -257,7 +260,7 @@ proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = ## passes ``future`` as a param to the callback. future.cb = cb if future.finished: - future.cb() + callSoon(future.cb) proc `callback=`*[T](future: Future[T], cb: proc (future: Future[T]) {.closure,gcsafe.}) = @@ -352,28 +355,88 @@ proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = fut2.callback = cb return retFuture +proc all*[T](futs: varargs[Future[T]]): auto = + ## Returns a future which will complete once + ## all futures in ``futs`` complete. + ## + ## If the awaited futures are not ``Future[void]``, the returned future + ## will hold the values of all awaited futures in a sequence. + ## + ## If the awaited futures *are* ``Future[void]``, + ## this proc returns ``Future[void]``. + + when T is void: + var + retFuture = newFuture[void]("asyncdispatch.all") + completedFutures = 0 + + let totalFutures = len(futs) + + for fut in futs: + fut.callback = proc(f: Future[T]) = + inc(completedFutures) + + if completedFutures == totalFutures: + retFuture.complete() + + return retFuture + + else: + var + retFuture = newFuture[seq[T]]("asyncdispatch.all") + retValues = newSeq[T](len(futs)) + completedFutures = 0 + + for i, fut in futs: + proc setCallback(i: int) = + fut.callback = proc(f: Future[T]) = + retValues[i] = f.read() + inc(completedFutures) + + if completedFutures == len(retValues): + retFuture.complete(retValues) + + setCallback(i) + + return retFuture + type PDispatcherBase = ref object of RootRef - timers: seq[tuple[finishAt: float, fut: Future[void]]] - -proc processTimers(p: PDispatcherBase) = - var oldTimers = p.timers - p.timers = @[] - for t in oldTimers: - if epochTime() >= t.finishAt: - t.fut.complete() - else: - p.timers.add(t) + timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]] + callbacks: Queue[proc ()] + +proc processTimers(p: PDispatcherBase) {.inline.} = + while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt: + p.timers.pop().fut.complete() + +proc processPendingCallbacks(p: PDispatcherBase) = + while p.callbacks.len > 0: + var cb = p.callbacks.dequeue() + cb() + +proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = + # If dispatcher has active timers this proc returns the timeout + # of the nearest timer. Returns `timeout` otherwise. + result = timeout + if p.timers.len > 0: + let timerTimeout = p.timers[0].finishAt + let curTime = epochTime() + if timeout == -1 or (curTime + (timeout / 1000)) > timerTimeout: + result = int((timerTimeout - curTime) * 1000) + if result < 0: result = 0 when defined(windows) or defined(nimdoc): import winlean, sets, hashes type - CompletionKey = Dword + CompletionKey = ULONG_PTR CompletionData* = object fd*: AsyncFD # TODO: Rename this. cb*: proc (fd: AsyncFD, bytesTransferred: Dword, errcode: OSErrorCode) {.closure,gcsafe.} + cell*: ForeignCell # we need this `cell` to protect our `cb` environment, + # when using RegisterWaitForSingleObject, because + # waiting is done in different thread. PDispatcher* = ref object of PDispatcherBase ioPort: Handle @@ -385,6 +448,15 @@ when defined(windows) or defined(nimdoc): PCustomOverlapped* = ref CustomOverlapped AsyncFD* = distinct int + + PostCallbackData = object + ioPort: Handle + handleFd: AsyncFD + waitFd: Handle + ovl: PCustomOverlapped + PostCallbackDataPtr = ptr PostCallbackData + + Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} {.deprecated: [TCompletionKey: CompletionKey, TAsyncFD: AsyncFD, TCustomOverlapped: CustomOverlapped, TCompletionData: CompletionData].} @@ -396,7 +468,8 @@ when defined(windows) or defined(nimdoc): new result result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) result.handles = initSet[AsyncFD]() - result.timers = @[] + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = @@ -423,15 +496,17 @@ when defined(windows) or defined(nimdoc): proc poll*(timeout = 500) = ## Waits for completion events and processes them. let p = getGlobalDispatcher() - if p.handles.len == 0 and p.timers.len == 0: + if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: raise newException(ValueError, "No handles or timers registered in dispatcher.") - let llTimeout = - if timeout == -1: winlean.INFINITE - else: timeout.int32 + let at = p.adjustedTimeout(timeout) + var llTimeout = + if at == -1: winlean.INFINITE + else: at.int32 + var lpNumberOfBytesTransferred: Dword - var lpCompletionKey: ULONG + var lpCompletionKey: ULONG_PTR var customOverlapped: PCustomOverlapped let res = getQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred, addr lpCompletionKey, @@ -445,6 +520,13 @@ when defined(windows) or defined(nimdoc): customOverlapped.data.cb(customOverlapped.data.fd, lpNumberOfBytesTransferred, OSErrorCode(-1)) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) else: let errCode = osLastError() @@ -452,6 +534,8 @@ when defined(windows) or defined(nimdoc): assert customOverlapped.data.fd == lpCompletionKey.AsyncFD customOverlapped.data.cb(customOverlapped.data.fd, lpNumberOfBytesTransferred, errCode) + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) GC_unref(customOverlapped) else: if errCode.int32 == WAIT_TIMEOUT: @@ -461,6 +545,8 @@ when defined(windows) or defined(nimdoc): # Timer processing. processTimers(p) + # Callback queue processing + processPendingCallbacks(p) var connectExPtr: pointer = nil var acceptExPtr: pointer = nil @@ -651,34 +737,16 @@ when defined(windows) or defined(nimdoc): retFuture.complete("") else: retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0': - # We have to ensure that the buffer is empty because WSARecv will tell - # us immediately when it was disconnected, even when there is still - # data in the buffer. - # We want to give the user as much data as we can. So we only return - # the empty string (which signals a disconnection) when there is - # nothing left to read. - retFuture.complete("") - # TODO: "For message-oriented sockets, where a zero byte message is often - # allowable, a failure with an error code of WSAEDISCON is used to - # indicate graceful closure." - # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx - else: - # Request to read completed immediately. - # From my tests bytesReceived isn't reliable. - let realSize = - if bytesReceived == 0: - size - else: - bytesReceived - var data = newString(realSize) - assert realSize <= size - copyMem(addr data[0], addr dataBuf.buf[0], realSize) - #dealloc dataBuf.buf - retFuture.complete($data) - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + var data = newString(bytesReceived) + assert bytesReceived <= size + copyMem(addr data[0], addr dataBuf.buf[0], bytesReceived) + retFuture.complete($data) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete("") return retFuture proc recvInto*(socket: AsyncFD, buf: cstring, size: int, @@ -741,31 +809,14 @@ when defined(windows) or defined(nimdoc): retFuture.complete(0) else: retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0': - # We have to ensure that the buffer is empty because WSARecv will tell - # us immediately when it was disconnected, even when there is still - # data in the buffer. - # We want to give the user as much data as we can. So we only return - # the empty string (which signals a disconnection) when there is - # nothing left to read. - retFuture.complete(0) - # TODO: "For message-oriented sockets, where a zero byte message is often - # allowable, a failure with an error code of WSAEDISCON is used to - # indicate graceful closure." - # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx - else: - # Request to read completed immediately. - # From my tests bytesReceived isn't reliable. - let realSize = - if bytesReceived == 0: - size - else: - bytesReceived - assert realSize <= size - retFuture.complete(realSize) - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) return retFuture proc send*(socket: AsyncFD, data: string, @@ -811,6 +862,101 @@ when defined(windows) or defined(nimdoc): # free ``ol``. return retFuture + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: Socklen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to specified destination ``saddr``, using + ## socket ``socket``. The returned future will complete once all data + ## has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("sendTo") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](data) + dataBuf.len = size.ULONG + var bytesSent = 0.Dword + var lowFlags = 0.Dword + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen: cint = cint(saddrLen) + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, + lowFlags, cast[ptr SockAddr](addr(staddr[0])), + stalen, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``buf``, which must + ## be at least of size ``size``, address of datagram's sender will be + ## stored into ``saddr`` and ``saddrLen``. Returned future will complete + ## once one datagram has been received, and will return size of packet + ## received. + verifyPresence(socket) + var retFuture = newFuture[int]("recvFromInto") + + var dataBuf = TWSABuf(buf: cast[cstring](data), len: size.ULONG) + + var bytesReceived = 0.Dword + var lowFlags = 0.Dword + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount <= size + retFuture.complete(bytesCount) + else: + # datagram sockets don't have disconnection, + # so we can just raise an exception + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, + addr bytesReceived, addr lowFlags, + saddr, cast[ptr cint](saddrLen), + cast[POVERLAPPED](ol), nil) + if res == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) + return retFuture + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): Future[tuple[address: string, client: AsyncFD]] = ## Accepts a new connection. Returns a future containing the client socket @@ -837,7 +983,7 @@ when defined(windows) or defined(nimdoc): let dwLocalAddressLength = Dword(sizeof (Sockaddr_in) + 16) let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16) - template completeAccept(): stmt {.immediate, dirty.} = + template completeAccept() {.dirty.} = var listenSock = socket let setoptRet = setsockopt(clientSock, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, @@ -857,7 +1003,7 @@ when defined(windows) or defined(nimdoc): client: clientSock.AsyncFD) ) - template failAccept(errcode): stmt = + template failAccept(errcode) = if flags.isDisconnectionError(errcode): var newAcceptFut = acceptAddr(socket, flags) newAcceptFut.callback = @@ -923,6 +1069,126 @@ when defined(windows) or defined(nimdoc): ## Unregisters ``fd``. getGlobalDispatcher().handles.excl(fd) + {.push stackTrace:off.} + proc waitableCallback(param: pointer, + timerOrWaitFired: WINBOOL): void {.stdcall.} = + var p = cast[PostCallbackDataPtr](param) + discard postQueuedCompletionStatus(p.ioPort, timerOrWaitFired.Dword, + ULONG_PTR(p.handleFd), + cast[pointer](p.ovl)) + {.pop.} + + template registerWaitableEvent(mask) = + let p = getGlobalDispatcher() + var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword + var hEvent = wsaCreateEvent() + if hEvent == 0: + raiseOSError(osLastError()) + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + pcd.ioPort = p.ioPort + pcd.handleFd = fd + var ol = PCustomOverlapped() + GC_ref(ol) + + ol.data = CompletionData(fd: fd, cb: + proc(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + # we excluding our `fd` because cb(fd) can register own handler + # for this `fd` + p.handles.excl(fd) + # unregisterWait() is called before callback, because appropriate + # winsockets function can re-enable event. + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms741576(v=vs.85).aspx + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + if cb(fd): + # callback returned `true`, so we free all allocated resources + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + # pcd.ovl will be unrefed in poll(). + else: + # callback returned `false` we need to continue + if p.handles.contains(fd): + # new callback was already registered with `fd`, so we free all + # allocated resources. This happens because in callback `cb` + # addRead/addWrite was called with same `fd`. + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + else: + # we need to include `fd` again + p.handles.incl(fd) + # and register WaitForSingleObject again + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + # pcd.ovl will be unrefed in poll() + discard wsaCloseEvent(hEvent) + deallocShared(cast[pointer](pcd)) + raiseOSError(osLastError()) + else: + # we ref pcd.ovl one more time, because it will be unrefed in + # poll() + GC_ref(pcd.ovl) + ) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + # This is main part of `hacky way` is using WSAEventSelect, so `hEvent` + # will be signaled when appropriate `mask` events will be triggered. + if wsaEventSelect(fd.SocketHandle, hEvent, mask) != 0: + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + p.handles.incl(fd) + + proc addRead*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for read availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addRead` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.recv() + ## or asyncdispatch.accept(), because they are using IOCP, please use + ## nativesockets.recv() and nativesockets.accept() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `read` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) + + proc addWrite*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for write availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addWrite` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.send() + ## or asyncdispatch.connect(), because they are using IOCP, please use + ## nativesockets.send() and nativesockets.connect() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `write` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_WRITE or FD_CONNECT or FD_CLOSE) + initAll() else: import selectors @@ -956,7 +1222,8 @@ else: proc newDispatcher*(): PDispatcher = new result result.selector = newSelector() - result.timers = @[] + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = @@ -1014,7 +1281,7 @@ else: proc poll*(timeout = 500) = let p = getGlobalDispatcher() - for info in p.selector.select(timeout): + for info in p.selector.select(p.adjustedTimeout(timeout)): let data = PData(info.key.data) assert data.fd == info.key.fd.AsyncFD #echo("In poll ", data.fd.cint) @@ -1052,7 +1319,10 @@ else: # (e.g. socket disconnected). discard + # Timer processing. processTimers(p) + # Callback queue processing + processPendingCallbacks(p) proc connect*(socket: AsyncFD, address: string, port: Port, domain = AF_INET): Future[void] = @@ -1184,6 +1454,60 @@ else: addWrite(socket, cb) return retFuture + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: SockLen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` of size ``size`` in bytes to specified destination + ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``. + ## The returned future will complete once all data has been sent. + var retFuture = newFuture[void]("sendTo") + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen = saddrLen + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + proc cb(sock: AsyncFD): bool = + result = true + let res = sendto(sock.SocketHandle, data, size, MSG_NOSIGNAL, + cast[ptr SockAddr](addr(staddr[0])), stalen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete() + + addWrite(socket, cb) + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``data``, which must + ## be at least of size ``size`` in bytes, address of datagram's sender + ## will be stored into ``saddr`` and ``saddrLen``. Returned future will + ## complete once one datagram has been received, and will return size + ## of packet received. + var retFuture = newFuture[int]("recvFromInto") + proc cb(sock: AsyncFD): bool = + result = true + let res = recvfrom(sock.SocketHandle, data, size.cint, flags.toOSFlags(), + saddr, saddrLen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false + else: + retFuture.complete(res) + addRead(socket, cb) + return retFuture + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): Future[tuple[address: string, client: AsyncFD]] = var retFuture = newFuture[tuple[address: string, @@ -1215,7 +1539,25 @@ proc sleepAsync*(ms: int): Future[void] = ## ``ms`` milliseconds. var retFuture = newFuture[void]("sleepAsync") let p = getGlobalDispatcher() - p.timers.add((epochTime() + (ms / 1000), retFuture)) + p.timers.push((epochTime() + (ms / 1000), retFuture)) + return retFuture + +proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] = + ## Returns a future which will complete once ``fut`` completes or after + ## ``timeout`` milliseconds has elapsed. + ## + ## If ``fut`` completes first the returned future will hold true, + ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned + ## future will hold false. + + var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") + var timeoutFuture = sleepAsync(timeout) + fut.callback = + proc () = + if not retFuture.finished: retFuture.complete(true) + timeoutFuture.callback = + proc () = + if not retFuture.finished: retFuture.complete(false) return retFuture proc accept*(socket: AsyncFD, @@ -1248,7 +1590,7 @@ proc skipStmtList(node: NimNode): NimNode {.compileTime.} = result = node[0] template createCb(retFutureSym, iteratorNameSym, - name: expr): stmt {.immediate.} = + name: untyped) = var nameIterVar = iteratorNameSym #{.push stackTrace: off.} proc cb {.closure,gcsafe.} = @@ -1316,6 +1658,23 @@ proc generateExceptionCheck(futSym, ) result.add elseNode +template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, + rootReceiver: expr, fromNode: NimNode) = + ## Params: + ## futureVarNode: The NimNode which is a symbol identifying the Future[T] + ## variable to yield. + ## fromNode: Used for better debug information (to give context). + ## valueReceiver: The node which defines an expression that retrieves the + ## future's value. + ## + ## rootReceiver: ??? TODO + # -> yield future<x> + result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) + # -> future<x>.read + valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) + result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, + fromNode) + template createVar(result: var NimNode, futSymName: string, asyncProc: NimNode, valueReceiver, rootReceiver: expr, @@ -1323,9 +1682,7 @@ template createVar(result: var NimNode, futSymName: string, result = newNimNode(nnkStmtList, fromNode) var futSym = genSym(nskVar, "future") result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y - result.add newNimNode(nnkYieldStmt, fromNode).add(futSym) # -> yield future<x> - valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future<x>.read - result.add generateExceptionCheck(futSym, tryStmt, rootReceiver, fromNode) + useVar(result, futSym, valueReceiver, rootReceiver, fromNode) proc processBody(node, retFutureSym: NimNode, subTypeIsVoid: bool, @@ -1342,19 +1699,23 @@ proc processBody(node, retFutureSym: NimNode, else: result.add newCall(newIdentNode("complete"), retFutureSym) else: - result.add newCall(newIdentNode("complete"), retFutureSym, - node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt)) + let x = node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt) + if x.kind == nnkYieldStmt: result.add x + else: + result.add newCall(newIdentNode("complete"), retFutureSym, x) result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt of nnkCommand, nnkCall: if node[0].kind == nnkIdent and node[0].ident == !"await": case node[1].kind - of nnkIdent, nnkInfix: + of nnkIdent, nnkInfix, nnkDotExpr: # await x + # await x or y result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x of nnkCall, nnkCommand: # await foo(p, x) + # await foo p, x var futureValue: NimNode result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, futureValue, node) @@ -1511,38 +1872,40 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # -> complete(retFuture, result) var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) - if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), - newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( - newIdentNode("warning"), newIdentNode("resultshadowed")), - newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} - - procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( - newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T - - procBody.insert(2, newNimNode(nnkPragma).add( - newIdentNode("pop"))) # -> {.pop.}) - - procBody.add( - newCall(newIdentNode("complete"), - retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) - else: - # -> complete(retFuture) - procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + # don't do anything with forward bodies (empty) + if procBody.kind != nnkEmpty: + if not subtypeIsVoid: + procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), + newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( + newIdentNode("warning"), newIdentNode("resultshadowed")), + newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} + + procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( + newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T + + procBody.insert(2, newNimNode(nnkPragma).add( + newIdentNode("pop"))) # -> {.pop.}) + + procBody.add( + newCall(newIdentNode("complete"), + retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) + else: + # -> complete(retFuture) + procBody.add(newCall(newIdentNode("complete"), retFutureSym)) - var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], - procBody, nnkIteratorDef) - closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) - outerProcBody.add(closureIterator) + var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], + procBody, nnkIteratorDef) + closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) + outerProcBody.add(closureIterator) - # -> createCb(retFuture) - #var cbName = newIdentNode("cb") - var procCb = newCall(bindSym"createCb", retFutureSym, iteratorNameSym, - newStrLitNode(prc[0].getName)) - outerProcBody.add procCb + # -> createCb(retFuture) + #var cbName = newIdentNode("cb") + var procCb = getAst createCb(retFutureSym, iteratorNameSym, + newStrLitNode(prc[0].getName)) + outerProcBody.add procCb - # -> return retFuture - outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) + # -> return retFuture + outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) result = prc @@ -1550,19 +1913,19 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = for i in 0 .. <result[4].len: if result[4][i].kind == nnkIdent and result[4][i].ident == !"async": result[4].del(i) + result[4] = newEmptyNode() if subtypeIsVoid: # Add discardable pragma. if returnType.kind == nnkEmpty: # Add Future[void] result[3][0] = parseExpr("Future[void]") - - result[6] = outerProcBody - + if procBody.kind != nnkEmpty: + result[6] = outerProcBody #echo(treeRepr(result)) - #if prc[0].getName == "hubConnectionLoop": + #if prc[0].getName == "testInfix": # echo(toStrLit(result)) -macro async*(prc: stmt): stmt {.immediate.} = +macro async*(prc: untyped): untyped = ## Macro which processes async procedures into the appropriate ## iterators and yield statements. if prc.kind == nnkStmtList: @@ -1571,6 +1934,8 @@ macro async*(prc: stmt): stmt {.immediate.} = result.add asyncSingleProc(oneProc) else: result = asyncSingleProc(prc) + when defined(nimDumpAsync): + echo repr result proc recvLine*(socket: AsyncFD): Future[string] {.async.} = ## Reads a line of data from ``socket``. Returned future will complete once @@ -1611,6 +1976,11 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} = return add(result, c) +proc callSoon*(cbproc: proc ()) = + ## Schedule `cbproc` to be called as soon as possible. + ## The callback is called when control returns to the event loop. + getGlobalDispatcher().callbacks.enqueue(cbproc) + proc runForever*() = ## Begins a never ending global dispatcher poll loop. while true: diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index c7b9fac18..5df606ea8 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -162,7 +162,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = # Request completed immediately. var bytesRead: DWord let overlappedRes = getOverlappedResult(f.fd.Handle, - cast[POverlapped](ol)[], bytesRead, false.WinBool) + cast[POverlapped](ol), bytesRead, false.WinBool) if not overlappedRes.bool: let err = osLastError() if err.int32 == ERROR_HANDLE_EOF: @@ -282,7 +282,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = # Request completed immediately. var bytesWritten: DWord let overlappedRes = getOverlappedResult(f.fd.Handle, - cast[POverlapped](ol)[], bytesWritten, false.WinBool) + cast[POverlapped](ol), bytesWritten, false.WinBool) if not overlappedRes.bool: retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) else: @@ -316,6 +316,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = proc close*(f: AsyncFile) = ## Closes the file specified. + unregister(f.fd) when defined(windows) or defined(nimdoc): if not closeHandle(f.fd.Handle).bool: raiseOSError(osLastError()) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 590b52c1a..6a7326e83 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -25,12 +25,21 @@ ## ## waitFor server.serve(Port(8080), cb) -import strtabs, asyncnet, asyncdispatch, parseutils, uri, strutils +import tables, asyncnet, asyncdispatch, parseutils, uri, strutils +import httpcore + +export httpcore except parseHeader + +# TODO: If it turns out that the decisions that asynchttpserver makes +# explicitly, about whether to close the client sockets or upgrade them are +# wrong, then add a return value which determines what to do for the callback. +# Also, maybe move `client` out of `Request` object and into the args for +# the proc. type Request* = object client*: AsyncSocket # TODO: Separate this into a Response object? reqMethod*: string - headers*: StringTableRef + headers*: HttpHeaders protocol*: tuple[orig: string, major, minor: int] url*: Uri hostname*: string ## The hostname of the client that made the request. @@ -39,83 +48,29 @@ type AsyncHttpServer* = ref object socket: AsyncSocket reuseAddr: bool - - HttpCode* = enum - Http100 = "100 Continue", - Http101 = "101 Switching Protocols", - Http200 = "200 OK", - Http201 = "201 Created", - Http202 = "202 Accepted", - Http204 = "204 No Content", - Http205 = "205 Reset Content", - Http206 = "206 Partial Content", - Http300 = "300 Multiple Choices", - Http301 = "301 Moved Permanently", - Http302 = "302 Found", - Http303 = "303 See Other", - Http304 = "304 Not Modified", - Http305 = "305 Use Proxy", - Http307 = "307 Temporary Redirect", - Http400 = "400 Bad Request", - Http401 = "401 Unauthorized", - Http403 = "403 Forbidden", - Http404 = "404 Not Found", - Http405 = "405 Method Not Allowed", - Http406 = "406 Not Acceptable", - Http407 = "407 Proxy Authentication Required", - Http408 = "408 Request Timeout", - Http409 = "409 Conflict", - Http410 = "410 Gone", - Http411 = "411 Length Required", - Http412 = "412 Precondition Failed", - Http413 = "413 Request Entity Too Large", - Http414 = "414 Request-URI Too Long", - Http415 = "415 Unsupported Media Type", - Http416 = "416 Requested Range Not Satisfiable", - Http417 = "417 Expectation Failed", - Http418 = "418 I'm a teapot", - Http500 = "500 Internal Server Error", - Http501 = "501 Not Implemented", - Http502 = "502 Bad Gateway", - Http503 = "503 Service Unavailable", - Http504 = "504 Gateway Timeout", - Http505 = "505 HTTP Version Not Supported" - - HttpVersion* = enum - HttpVer11, - HttpVer10 + reusePort: bool {.deprecated: [TRequest: Request, PAsyncHttpServer: AsyncHttpServer, THttpCode: HttpCode, THttpVersion: HttpVersion].} -proc `==`*(protocol: tuple[orig: string, major, minor: int], - ver: HttpVersion): bool = - let major = - case ver - of HttpVer11, HttpVer10: 1 - let minor = - case ver - of HttpVer11: 1 - of HttpVer10: 0 - result = protocol.major == major and protocol.minor == minor - -proc newAsyncHttpServer*(reuseAddr = true): AsyncHttpServer = +proc newAsyncHttpServer*(reuseAddr = true, reusePort = false): AsyncHttpServer = ## Creates a new ``AsyncHttpServer`` instance. new result result.reuseAddr = reuseAddr + result.reusePort = reusePort -proc addHeaders(msg: var string, headers: StringTableRef) = +proc addHeaders(msg: var string, headers: HttpHeaders) = for k, v in headers: msg.add(k & ": " & v & "\c\L") -proc sendHeaders*(req: Request, headers: StringTableRef): Future[void] = +proc sendHeaders*(req: Request, headers: HttpHeaders): Future[void] = ## Sends the specified headers to the requesting client. var msg = "" addHeaders(msg, headers) return req.client.send(msg) proc respond*(req: Request, code: HttpCode, content: string, - headers: StringTableRef = nil): Future[void] = + headers: HttpHeaders = nil): Future[void] = ## Responds to the request with the specified ``HttpCode``, headers and ## content. ## @@ -128,16 +83,6 @@ proc respond*(req: Request, code: HttpCode, content: string, msg.add(content) result = req.client.send(msg) -proc parseHeader(line: string): tuple[key, value: string] = - var i = 0 - i = line.parseUntil(result.key, ':') - inc(i) # skip : - if i < len(line): - i += line.skipWhiteSpace(i) - i += line.parseUntil(result.value, {'\c', '\L'}, i) - else: - result.value = "" - proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = var i = protocol.skipIgnoreCase("HTTP/") if i != 5: @@ -156,7 +101,7 @@ proc processClient(client: AsyncSocket, address: string, Future[void] {.closure, gcsafe.}) {.async.} = var request: Request request.url = initUri() - request.headers = newStringTable(modeCaseInsensitive) + request.headers = newHttpHeaders() var lineFut = newFutureVar[string]("asynchttpserver.processClient") lineFut.mget() = newStringOfCap(80) var key, value = "" @@ -165,7 +110,7 @@ proc processClient(client: AsyncSocket, address: string, # GET /path HTTP/1.1 # Header: val # \n - request.headers.clear(modeCaseInsensitive) + request.headers.clear() request.body = "" request.hostname.shallowCopy(address) assert client != nil @@ -208,29 +153,34 @@ proc processClient(client: AsyncSocket, address: string, if lineFut.mget == "\c\L": break let (key, value) = parseHeader(lineFut.mget) request.headers[key] = value + # Ensure the client isn't trying to DoS us. + if request.headers.len > headerLimit: + await client.sendStatus("400 Bad Request") + request.client.close() + return if request.reqMethod == "post": # Check for Expect header if request.headers.hasKey("Expect"): - if request.headers.getOrDefault("Expect").toLower == "100-continue": + if "100-continue" in request.headers["Expect"]: await client.sendStatus("100 Continue") else: await client.sendStatus("417 Expectation Failed") - # Read the body - # - Check for Content-length header - if request.headers.hasKey("Content-Length"): - var contentLength = 0 - if parseInt(request.headers.getOrDefault("Content-Length"), - contentLength) == 0: - await request.respond(Http400, "Bad Request. Invalid Content-Length.") - continue - else: - request.body = await client.recv(contentLength) - assert request.body.len == contentLength - else: - await request.respond(Http400, "Bad Request. No Content-Length.") + # Read the body + # - Check for Content-length header + if request.headers.hasKey("Content-Length"): + var contentLength = 0 + if parseInt(request.headers["Content-Length"], + contentLength) == 0: + await request.respond(Http400, "Bad Request. Invalid Content-Length.") continue + else: + request.body = await client.recv(contentLength) + assert request.body.len == contentLength + elif request.reqMethod == "post": + await request.respond(Http400, "Bad Request. No Content-Length.") + continue case request.reqMethod of "get", "post", "head", "put", "delete", "trace", "options", @@ -240,6 +190,9 @@ proc processClient(client: AsyncSocket, address: string, await request.respond(Http400, "Invalid request method. Got: " & request.reqMethod) + if "upgrade" in request.headers.getOrDefault("connection"): + return + # Persistent connections if (request.protocol == HttpVer11 and request.headers.getOrDefault("connection").normalize != "close") or @@ -264,6 +217,8 @@ proc serve*(server: AsyncHttpServer, port: Port, server.socket = newAsyncSocket() if server.reuseAddr: server.socket.setSockOpt(OptReuseAddr, true) + if server.reusePort: + server.socket.setSockOpt(OptReusePort, true) server.socket.bindAddr(port, address) server.socket.listen() @@ -287,7 +242,7 @@ when not defined(testing) and isMainModule: #echo(req.headers) let headers = {"Date": "Tue, 29 Apr 2014 23:40:08 GMT", "Content-type": "text/plain; charset=utf-8"} - await req.respond(Http200, "Hello World", headers.newStringTable()) + await req.respond(Http200, "Hello World", headers.newHttpHeaders()) asyncCheck server.serve(Port(5555), cb) runForever() diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 6b19a48be..a1988f4a6 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -11,7 +11,7 @@ ## asynchronous dispatcher defined in the ``asyncdispatch`` module. ## ## SSL -## --- +## ---- ## ## SSL can be enabled by compiling with the ``-d:ssl`` flag. ## @@ -62,7 +62,9 @@ import os export SOBool -when defined(ssl): +const defineSsl = defined(ssl) or defined(nimdoc) + +when defineSsl: import openssl type @@ -79,7 +81,7 @@ type of false: nil case isSsl: bool of true: - when defined(ssl): + when defineSsl: sslHandle: SslPtr sslContext: SslContext bioIn: BIO @@ -125,7 +127,7 @@ proc newAsyncSocket*(domain, sockType, protocol: cint, Domain(domain), SockType(sockType), Protocol(protocol), buffered) -when defined(ssl): +when defineSsl: proc getSslError(handle: SslPtr, err: cint): cint = assert err < 0 var ret = SSLGetError(handle, err.cint) @@ -186,7 +188,7 @@ proc connect*(socket: AsyncSocket, address: string, port: Port) {.async.} = ## or an error occurs. await connect(socket.fd.AsyncFD, address, port, socket.domain) if socket.isSsl: - when defined(ssl): + when defineSsl: let flags = {SocketFlag.SafeDisconn} sslSetConnectState(socket.sslHandle) sslLoop(socket, flags, sslDoHandshake(socket.sslHandle)) @@ -197,7 +199,7 @@ template readInto(buf: cstring, size: int, socket: AsyncSocket, ## this is a template and not a proc. var res = 0 if socket.isSsl: - when defined(ssl): + when defineSsl: # SSL mode. sslLoop(socket, flags, sslRead(socket.sslHandle, buf, size.cint)) @@ -274,7 +276,7 @@ proc send*(socket: AsyncSocket, data: string, ## data has been sent. assert socket != nil if socket.isSsl: - when defined(ssl): + when defineSsl: var copy = data sslLoop(socket, flags, sslWrite(socket.sslHandle, addr copy[0], copy.len.cint)) @@ -426,9 +428,6 @@ proc recvLine*(socket: AsyncSocket, ## ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol ## uses ``\r\L`` to delimit a new line. - template addNLIfEmpty(): stmt = - if result.len == 0: - result.add("\c\L") assert SocketFlag.Peek notin flags ## TODO: # TODO: Optimise this @@ -456,7 +455,8 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. of AF_INET6: realaddr = "::" of AF_INET: realaddr = "0.0.0.0" else: - raiseOSError("Unknown socket address family and no address specified to bindAddr") + raise newException(ValueError, + "Unknown socket address family and no address specified to bindAddr") var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: @@ -468,7 +468,7 @@ proc close*(socket: AsyncSocket) = ## Closes the socket. defer: socket.fd.AsyncFD.closeSocket() - when defined(ssl): + when defineSsl: if socket.isSSL: let res = SslShutdown(socket.sslHandle) SSLFree(socket.sslHandle) @@ -478,7 +478,7 @@ proc close*(socket: AsyncSocket) = raiseSslError() socket.closed = true # TODO: Add extra debugging checks for this. -when defined(ssl): +when defineSsl: proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) = ## Wraps a socket in an SSL context. This function effectively turns ## ``socket`` into an SSL socket. @@ -487,7 +487,7 @@ when defined(ssl): ## prone to security vulnerabilities. socket.isSsl = true socket.sslContext = ctx - socket.sslHandle = SSLNew(SSLCTX(socket.sslContext)) + socket.sslHandle = SSLNew(socket.sslContext.context) if socket.sslHandle == nil: raiseSslError() diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index 32d37ce02..2fc0b1c5e 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -90,6 +90,8 @@ template encodeInternal(s: expr, lineLen: int, newLine: string): stmt {.immediat if r+4 != result.len: setLen(result, r+4) else: + if r != result.len: + setLen(result, r) #assert(r == result.len) discard @@ -162,4 +164,3 @@ when isMainModule: "asure.", longText] for t in items(tests): assert decode(encode(t)) == t - diff --git a/lib/pure/collections/LockFreeHash.nim b/lib/pure/collections/LockFreeHash.nim index a3ead81e3..954d62491 100644 --- a/lib/pure/collections/LockFreeHash.nim +++ b/lib/pure/collections/LockFreeHash.nim @@ -52,7 +52,7 @@ when sizeof(int) == 4: # 32bit {.deprecated: [TRaw: Raw].} elif sizeof(int) == 8: # 64bit type - Raw = range[0..4611686018427387903] + Raw = range[0'i64..4611686018427387903'i64] ## The range of uint values that can be stored directly in a value slot ## when on a 64 bit platform {.deprecated: [TRaw: Raw].} @@ -74,7 +74,7 @@ type copyDone: int next: PConcTable[K,V] data: EntryArr -{.deprecated: [TEntry: Entry, TEntryArr: EntryArr.} +{.deprecated: [TEntry: Entry, TEntryArr: EntryArr].} proc setVal[K,V](table: var PConcTable[K,V], key: int, val: int, expVal: int, match: bool): int @@ -244,9 +244,9 @@ proc copySlot[K,V](idx: int, oldTbl: var PConcTable[K,V], newTbl: var PConcTable echo("oldVal is Tomb!!!, should not happen") if pop(oldVal) != 0: result = setVal(newTbl, pop(oldKey), pop(oldVal), 0, true) == 0 - if result: + #if result: #echo("Copied a Slot! idx= " & $idx & " key= " & $oldKey & " val= " & $oldVal) - else: + #else: #echo("copy slot failed") # Our copy is done so we disable the old slot while not ok: diff --git a/lib/pure/collections/chains.nim b/lib/pure/collections/chains.nim new file mode 100644 index 000000000..6b2ecd272 --- /dev/null +++ b/lib/pure/collections/chains.nim @@ -0,0 +1,44 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Template based implementation of singly and doubly linked lists. +## The involved types should have 'prev' or 'next' fields and the +## list header should have 'head' or 'tail' fields. + +template prepend*(header, node) = + when compiles(header.head): + when compiles(node.prev): + if header.head != nil: + header.head.prev = node + node.next = header.head + header.head = node + when compiles(header.tail): + if header.tail == nil: + header.tail = node + +template append*(header, node) = + when compiles(header.head): + if header.head == nil: + header.head = node + when compiles(header.tail): + when compiles(node.prev): + node.prev = header.tail + if header.tail != nil: + header.tail.next = node + header.tail = node + +template unlink*(header, node) = + if node.next != nil: + node.next.prev = node.prev + if node.prev != nil: + node.prev.next = node.next + if header.head == node: + header.head = node.prev + if header.tail == node: + header.tail = node.next diff --git a/lib/pure/collections/heapqueue.nim b/lib/pure/collections/heapqueue.nim new file mode 100644 index 000000000..149a1c9fc --- /dev/null +++ b/lib/pure/collections/heapqueue.nim @@ -0,0 +1,107 @@ +##[ Heap queue algorithm (a.k.a. priority queue). Ported from Python heapq. + +Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for +all k, counting elements from 0. For the sake of comparison, +non-existing elements are considered to be infinite. The interesting +property of a heap is that a[0] is always its smallest element. + +]## + +type HeapQueue*[T] = distinct seq[T] + +proc newHeapQueue*[T](): HeapQueue[T] {.inline.} = HeapQueue[T](newSeq[T]()) +proc newHeapQueue*[T](h: var HeapQueue[T]) {.inline.} = h = HeapQueue[T](newSeq[T]()) + +proc len*[T](h: HeapQueue[T]): int {.inline.} = seq[T](h).len +proc `[]`*[T](h: HeapQueue[T], i: int): T {.inline.} = seq[T](h)[i] +proc `[]=`[T](h: var HeapQueue[T], i: int, v: T) {.inline.} = seq[T](h)[i] = v +proc add[T](h: var HeapQueue[T], v: T) {.inline.} = seq[T](h).add(v) + +proc heapCmp[T](x, y: T): bool {.inline.} = + return (x < y) + +# 'heap' is a heap at all indices >= startpos, except possibly for pos. pos +# is the index of a leaf with a possibly out-of-order value. Restore the +# heap invariant. +proc siftdown[T](heap: var HeapQueue[T], startpos, p: int) = + var pos = p + var newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + let parentpos = (pos - 1) shr 1 + let parent = heap[parentpos] + if heapCmp(newitem, parent): + heap[pos] = parent + pos = parentpos + else: + break + heap[pos] = newitem + +proc siftup[T](heap: var HeapQueue[T], p: int) = + let endpos = len(heap) + var pos = p + let startpos = pos + let newitem = heap[pos] + # Bubble up the smaller child until hitting a leaf. + var childpos = 2*pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of smaller child. + let rightpos = childpos + 1 + if rightpos < endpos and not heapCmp(heap[childpos], heap[rightpos]): + childpos = rightpos + # Move the smaller child up. + heap[pos] = heap[childpos] + pos = childpos + childpos = 2*pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap[pos] = newitem + siftdown(heap, startpos, pos) + +proc push*[T](heap: var HeapQueue[T], item: T) = + ## Push item onto heap, maintaining the heap invariant. + (seq[T](heap)).add(item) + siftdown(heap, 0, len(heap)-1) + +proc pop*[T](heap: var HeapQueue[T]): T = + ## Pop the smallest item off the heap, maintaining the heap invariant. + let lastelt = seq[T](heap).pop() + if heap.len > 0: + result = heap[0] + heap[0] = lastelt + siftup(heap, 0) + else: + result = lastelt + +proc replace*[T](heap: var HeapQueue[T], item: T): T = + ## Pop and return the current smallest value, and add the new item. + ## This is more efficient than pop() followed by push(), and can be + ## more appropriate when using a fixed-size heap. Note that the value + ## returned may be larger than item! That constrains reasonable uses of + ## this routine unless written as part of a conditional replacement: + + ## if item > heap[0]: + ## item = replace(heap, item) + result = heap[0] + heap[0] = item + siftup(heap, 0) + +proc pushpop*[T](heap: var HeapQueue[T], item: T): T = + ## Fast version of a push followed by a pop. + if heap.len > 0 and heapCmp(heap[0], item): + swap(item, heap[0]) + siftup(heap, 0) + return item + +when isMainModule: + # Simple sanity test + var heap = newHeapQueue[int]() + let data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0] + for item in data: + push(heap, item) + doAssert(heap[0] == 0) + var sort = newSeq[int]() + while heap.len > 0: + sort.add(pop(heap)) + doAssert(sort == @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) diff --git a/lib/pure/collections/queues.nim b/lib/pure/collections/queues.nim index b9bf33bff..399e4d413 100644 --- a/lib/pure/collections/queues.nim +++ b/lib/pure/collections/queues.nim @@ -8,56 +8,142 @@ # ## Implementation of a `queue`:idx:. The underlying implementation uses a ``seq``. +## +## None of the procs that get an individual value from the queue can be used +## on an empty queue. +## If compiled with `boundChecks` option, those procs will raise an `IndexError` +## on such access. This should not be relied upon, as `-d:release` will +## disable those checks and may return garbage or crash the program. +## +## As such, a check to see if the queue is empty is needed before any +## access, unless your program logic guarantees it indirectly. +## +## .. code-block:: Nim +## proc foo(a, b: Positive) = # assume random positive values for `a` and `b` +## var q = initQueue[int]() # initializes the object +## for i in 1 ..< a: q.add i # populates the queue +## +## if b < q.len: # checking before indexed access +## echo "The element at index position ", b, " is ", q[b] +## +## # The following two lines don't need any checking on access due to the +## # logic of the program, but that would not be the case if `a` could be 0. +## assert q.front == 1 +## assert q.back == a +## +## while q.len > 0: # checking if the queue is empty +## echo q.pop() +## ## Note: For inter thread communication use ## a `Channel <channels.html>`_ instead. import math type - Queue*[T] = object ## a queue + Queue*[T] = object ## A queue. data: seq[T] rd, wr, count, mask: int {.deprecated: [TQueue: Queue].} -proc initQueue*[T](initialSize=4): Queue[T] = - ## creates a new queue. `initialSize` needs to be a power of 2. +proc initQueue*[T](initialSize: int = 4): Queue[T] = + ## Create a new queue. + ## Optionally, the initial capacity can be reserved via `initialSize` as a + ## performance optimization. The length of a newly created queue will still + ## be 0. + ## + ## `initialSize` needs to be a power of two. If you need to accept runtime + ## values for this you could use the ``nextPowerOfTwo`` proc from the + ## `math <math.html>`_ module. assert isPowerOfTwo(initialSize) result.mask = initialSize-1 newSeq(result.data, initialSize) -proc len*[T](q: Queue[T]): int = - ## returns the number of elements of `q`. +proc len*[T](q: Queue[T]): int {.inline.}= + ## Return the number of elements of `q`. result = q.count +template emptyCheck(q) = + # Bounds check for the regular queue access. + when compileOption("boundChecks"): + if unlikely(q.count < 1): + raise newException(IndexError, "Empty queue.") + +template xBoundsCheck(q, i) = + # Bounds check for the array like accesses. + when compileOption("boundChecks"): # d:release should disable this. + if unlikely(i >= q.count): # x < q.low is taken care by the Natural parameter + raise newException(IndexError, + "Out of bounds: " & $i & " > " & $(q.count - 1)) + +proc front*[T](q: Queue[T]): T {.inline.}= + ## Return the oldest element of `q`. Equivalent to `q.pop()` but does not + ## remove it from the queue. + emptyCheck(q) + result = q.data[q.rd] + +proc back*[T](q: Queue[T]): T {.inline.} = + ## Return the newest element of `q` but does not remove it from the queue. + emptyCheck(q) + result = q.data[q.wr - 1 and q.mask] + +proc `[]`*[T](q: Queue[T], i: Natural) : T {.inline.} = + ## Access the i-th element of `q` by order of insertion. + ## q[0] is the oldest (the next one q.pop() will extract), + ## q[^1] is the newest (last one added to the queue). + xBoundsCheck(q, i) + return q.data[q.rd + i and q.mask] + +proc `[]`*[T](q: var Queue[T], i: Natural): var T {.inline.} = + ## Access the i-th element of `q` and returns a mutable + ## reference to it. + xBoundsCheck(q, i) + return q.data[q.rd + i and q.mask] + +proc `[]=`* [T] (q: var Queue[T], i: Natural, val : T) {.inline.} = + ## Change the i-th element of `q`. + xBoundsCheck(q, i) + q.data[q.rd + i and q.mask] = val + iterator items*[T](q: Queue[T]): T = - ## yields every element of `q`. + ## Yield every element of `q`. var i = q.rd - var c = q.count - while c > 0: - dec c + for c in 0 ..< q.count: yield q.data[i] i = (i + 1) and q.mask iterator mitems*[T](q: var Queue[T]): var T = - ## yields every element of `q`. + ## Yield every element of `q`. var i = q.rd - var c = q.count - while c > 0: - dec c + for c in 0 ..< q.count: yield q.data[i] i = (i + 1) and q.mask +iterator pairs*[T](q: Queue[T]): tuple[key: int, val: T] = + ## Yield every (position, value) of `q`. + var i = q.rd + for c in 0 ..< q.count: + yield (c, q.data[i]) + i = (i + 1) and q.mask + +proc contains*[T](q: Queue[T], item: T): bool {.inline.} = + ## Return true if `item` is in `q` or false if not found. Usually used + ## via the ``in`` operator. It is the equivalent of ``q.find(item) >= 0``. + ## + ## .. code-block:: Nim + ## if x in q: + ## assert q.contains x + for e in q: + if e == item: return true + return false + proc add*[T](q: var Queue[T], item: T) = - ## adds an `item` to the end of the queue `q`. + ## Add an `item` to the end of the queue `q`. var cap = q.mask+1 - if q.count >= cap: - var n: seq[T] - newSeq(n, cap*2) - var i = 0 - for x in items(q): + if unlikely(q.count >= cap): + var n = newSeq[T](cap*2) + for i, x in q: # don't use copyMem because the GC and because it's slower. shallowCopy(n[i], x) - inc i shallowCopy(q.data, n) q.mask = cap*2 - 1 q.wr = q.count @@ -66,37 +152,104 @@ proc add*[T](q: var Queue[T], item: T) = q.data[q.wr] = item q.wr = (q.wr + 1) and q.mask -proc enqueue*[T](q: var Queue[T], item: T) = - ## alias for the ``add`` operation. - add(q, item) - -proc dequeue*[T](q: var Queue[T]): T = - ## removes and returns the first element of the queue `q`. - assert q.count > 0 +proc default[T](t: typedesc[T]): T {.inline.} = discard +proc pop*[T](q: var Queue[T]): T {.inline, discardable.} = + ## Remove and returns the first (oldest) element of the queue `q`. + emptyCheck(q) dec q.count result = q.data[q.rd] + q.data[q.rd] = default(type(result)) q.rd = (q.rd + 1) and q.mask +proc enqueue*[T](q: var Queue[T], item: T) = + ## Alias for the ``add`` operation. + q.add(item) + +proc dequeue*[T](q: var Queue[T]): T = + ## Alias for the ``pop`` operation. + q.pop() + proc `$`*[T](q: Queue[T]): string = - ## turns a queue into its string representation. + ## Turn a queue into its string representation. result = "[" - for x in items(q): + for x in items(q): # Don't remove the items here for reasons that don't fit in this margin. if result.len > 1: result.add(", ") result.add($x) result.add("]") when isMainModule: - var q = initQueue[int]() + var q = initQueue[int](1) q.add(123) q.add(9) - q.add(4) - var first = q.dequeue + q.enqueue(4) + var first = q.dequeue() q.add(56) q.add(6) - var second = q.dequeue + var second = q.pop() q.add(789) assert first == 123 assert second == 9 assert($q == "[4, 56, 6, 789]") + assert q[0] == q.front and q.front == 4 + assert q[^1] == q.back and q.back == 789 + q[0] = 42 + q[^1] = 7 + + assert 6 in q and 789 notin q + assert q.find(6) >= 0 + assert q.find(789) < 0 + + for i in -2 .. 10: + if i in q: + assert q.contains(i) and q.find(i) >= 0 + else: + assert(not q.contains(i) and q.find(i) < 0) + + when compileOption("boundChecks"): + try: + echo q[99] + assert false + except IndexError: + discard + + try: + assert q.len == 4 + for i in 0 ..< 5: q.pop() + assert false + except IndexError: + discard + + # grabs some types of resize error. + q = initQueue[int]() + for i in 1 .. 4: q.add i + q.pop() + q.pop() + for i in 5 .. 8: q.add i + assert $q == "[3, 4, 5, 6, 7, 8]" + + # Similar to proc from the documentation example + proc foo(a, b: Positive) = # assume random positive values for `a` and `b`. + var q = initQueue[int]() + assert q.len == 0 + for i in 1 .. a: q.add i + + if b < q.len: # checking before indexed access. + assert q[b] == b + 1 + + # The following two lines don't need any checking on access due to the logic + # of the program, but that would not be the case if `a` could be 0. + assert q.front == 1 + assert q.back == a + + while q.len > 0: # checking if the queue is empty + assert q.pop() > 0 + + #foo(0,0) + foo(8,5) + foo(10,9) + foo(1,1) + foo(2,1) + foo(1,5) + foo(3,2) diff --git a/lib/pure/collections/rtarrays.nim b/lib/pure/collections/rtarrays.nim index 9d8085643..2abe9d1f8 100644 --- a/lib/pure/collections/rtarrays.nim +++ b/lib/pure/collections/rtarrays.nim @@ -18,7 +18,7 @@ type RtArray*[T] = object ## L: Natural spart: seq[T] - apart: array [ArrayPartSize, T] + apart: array[ArrayPartSize, T] UncheckedArray* {.unchecked.}[T] = array[0..100_000_000, T] template usesSeqPart(x): expr = x.L > ArrayPartSize diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 0e3824a81..f458b7636 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -20,6 +20,8 @@ ## **Note**: This interface will change as soon as the compiler supports ## closures and proper coroutines. +include "system/inclrtl" + when not defined(nimhygiene): {.pragma: dirty.} @@ -355,7 +357,7 @@ proc insert*[T](dest: var seq[T], src: openArray[T], pos=0) = inc(j) -template filterIt*(seq1, pred: expr): expr = +template filterIt*(seq1, pred: untyped): untyped = ## Returns a new sequence with all the items that fulfilled the predicate. ## ## Unlike the `proc` version, the predicate needs to be an expression using @@ -369,12 +371,12 @@ template filterIt*(seq1, pred: expr): expr = ## notAcceptable = filterIt(temperatures, it > 50 or it < -10) ## assert acceptable == @[-2.0, 24.5, 44.31] ## assert notAcceptable == @[-272.15, 99.9, -113.44] - var result {.gensym.} = newSeq[type(seq1[0])]() + var result = newSeq[type(seq1[0])]() for it {.inject.} in items(seq1): if pred: result.add(it) result -template keepItIf*(varSeq: seq, pred: expr) = +template keepItIf*(varSeq: seq, pred: untyped) = ## Convenience template around the ``keepIf`` proc to reduce typing. ## ## Unlike the `proc` version, the predicate needs to be an expression using @@ -409,7 +411,7 @@ proc all*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool = return false return true -template allIt*(seq1, pred: expr): bool {.immediate.} = +template allIt*(seq1, pred: untyped): bool = ## Checks if every item fulfills the predicate. ## ## Example: @@ -418,7 +420,7 @@ template allIt*(seq1, pred: expr): bool {.immediate.} = ## let numbers = @[1, 4, 5, 8, 9, 7, 4] ## assert allIt(numbers, it < 10) == true ## assert allIt(numbers, it < 9) == false - var result {.gensym.} = true + var result = true for it {.inject.} in items(seq1): if not pred: result = false @@ -440,7 +442,7 @@ proc any*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): bool = return true return false -template anyIt*(seq1, pred: expr): bool {.immediate.} = +template anyIt*(seq1, pred: untyped): bool = ## Checks if some item fulfills the predicate. ## ## Example: @@ -449,14 +451,14 @@ template anyIt*(seq1, pred: expr): bool {.immediate.} = ## let numbers = @[1, 4, 5, 8, 9, 7, 4] ## assert anyIt(numbers, it > 8) == true ## assert anyIt(numbers, it > 9) == false - var result {.gensym.} = false + var result = false for it {.inject.} in items(seq1): if pred: result = true break result -template toSeq*(iter: expr): expr {.immediate.} = +template toSeq*(iter: untyped): untyped {.oldimmediate.} = ## Transforms any iterator into a sequence. ## ## Example: @@ -482,7 +484,7 @@ template toSeq*(iter: expr): expr {.immediate.} = result.add(x) result -template foldl*(sequence, operation: expr): expr = +template foldl*(sequence, operation: untyped): untyped = ## Template to fold a sequence from left to right, returning the accumulation. ## ## The sequence is required to have at least a single element. Debug versions @@ -510,7 +512,7 @@ template foldl*(sequence, operation: expr): expr = ## assert concatenation == "nimiscool" let s = sequence assert s.len > 0, "Can't fold empty sequences" - var result {.gensym.}: type(s[0]) + var result: type(s[0]) result = s[0] for i in 1..<s.len: let @@ -519,7 +521,7 @@ template foldl*(sequence, operation: expr): expr = result = operation result -template foldl*(sequence, operation: expr, first): expr = +template foldl*(sequence, operation, first): untyped = ## Template to fold a sequence from left to right, returning the accumulation. ## ## This version of ``foldl`` gets a starting parameter. This makes it possible @@ -535,7 +537,7 @@ template foldl*(sequence, operation: expr, first): expr = ## numbers = @[0, 8, 1, 5] ## digits = foldl(numbers, a & (chr(b + ord('0'))), "") ## assert digits == "0815" - var result {.gensym.}: type(first) + var result: type(first) result = first for x in items(sequence): let @@ -544,7 +546,7 @@ template foldl*(sequence, operation: expr, first): expr = result = operation result -template foldr*(sequence, operation: expr): expr = +template foldr*(sequence, operation: untyped): untyped = ## Template to fold a sequence from right to left, returning the accumulation. ## ## The sequence is required to have at least a single element. Debug versions @@ -572,7 +574,7 @@ template foldr*(sequence, operation: expr): expr = ## assert concatenation == "nimiscool" let s = sequence assert s.len > 0, "Can't fold empty sequences" - var result {.gensym.}: type(s[0]) + var result: type(s[0]) result = sequence[s.len - 1] for i in countdown(s.len - 2, 0): let @@ -581,7 +583,7 @@ template foldr*(sequence, operation: expr): expr = result = operation result -template mapIt*(seq1, typ, op: expr): expr {.deprecated.}= +template mapIt*(seq1, typ, op: untyped): untyped = ## Convenience template around the ``map`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an @@ -596,13 +598,13 @@ template mapIt*(seq1, typ, op: expr): expr {.deprecated.}= ## assert strings == @["4", "8", "12", "16"] ## **Deprecated since version 0.12.0:** Use the ``mapIt(seq1, op)`` ## template instead. - var result {.gensym.}: seq[typ] = @[] + var result: seq[typ] = @[] for it {.inject.} in items(seq1): result.add(op) result -template mapIt*(seq1, op: expr): expr = +template mapIt*(seq1, op: untyped): untyped = ## Convenience template around the ``map`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an @@ -631,7 +633,7 @@ template mapIt*(seq1, op: expr): expr = result.add(op) result -template applyIt*(varSeq, op: expr) = +template applyIt*(varSeq, op: untyped) = ## Convenience template around the mutable ``apply`` proc to reduce typing. ## ## The template injects the ``it`` variable which you can use directly in an @@ -648,7 +650,7 @@ template applyIt*(varSeq, op: expr) = -template newSeqWith*(len: int, init: expr): expr = +template newSeqWith*(len: int, init: untyped): untyped = ## creates a new sequence, calling `init` to initialize each value. Example: ## ## .. code-block:: @@ -657,10 +659,10 @@ template newSeqWith*(len: int, init: expr): expr = ## seq2D[1][0] = true ## seq2D[0][1] = true ## - ## import math + ## import random ## var seqRand = newSeqWith(20, random(10)) ## echo seqRand - var result {.gensym.} = newSeq[type(init)](len) + var result = newSeq[type(init)](len) for i in 0 .. <len: result[i] = init result diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 20f73ded3..552e41ef7 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -58,7 +58,7 @@ proc isValid*[A](s: HashSet[A]): bool = ## initialized. Example: ## ## .. code-block :: - ## proc savePreferences(options: Set[string]) = + ## proc savePreferences(options: HashSet[string]) = ## assert options.isValid, "Pass an initialized set!" ## # Do stuff here, may crash in release builds! result = not s.data.isNil @@ -72,7 +72,7 @@ proc len*[A](s: HashSet[A]): int = ## ## .. code-block:: ## - ## var values: Set[int] + ## var values: HashSet[int] ## assert(not values.isValid) ## assert values.len == 0 result = s.counter @@ -256,11 +256,13 @@ proc incl*[A](s: var HashSet[A], other: HashSet[A]) = assert other.isValid, "The set `other` needs to be initialized." for item in other: incl(s, item) -template doWhile(a: expr, b: stmt): stmt = +template doWhile(a, b) = while true: b if not a: break +proc default[T](t: typedesc[T]): T {.inline.} = discard + proc excl*[A](s: var HashSet[A], key: A) = ## Excludes `key` from the set `s`. ## @@ -277,12 +279,14 @@ proc excl*[A](s: var HashSet[A], key: A) = var msk = high(s.data) if i >= 0: s.data[i].hcode = 0 + s.data[i].key = default(type(s.data[i].key)) dec(s.counter) while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 var j = i # The correctness of this depends on (h+1) in nextTry, var r = j # though may be adaptable to other simple sequences. s.data[i].hcode = 0 # mark current EMPTY - doWhile ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): + s.data[i].key = default(type(s.data[i].key)) + doWhile((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): i = (i + 1) and msk # increment mod table size if isEmpty(s.data[i].hcode): # end of collision cluster; So all done return @@ -334,7 +338,7 @@ proc init*[A](s: var HashSet[A], initialSize=64) = ## existing values and calling `excl() <#excl,TSet[A],A>`_ on them. Example: ## ## .. code-block :: - ## var a: Set[int] + ## var a: HashSet[int] ## a.init(4) ## a.incl(2) ## a.init @@ -367,7 +371,7 @@ proc toSet*[A](keys: openArray[A]): HashSet[A] = result = initSet[A](rightSize(keys.len)) for key in items(keys): result.incl(key) -template dollarImpl(): stmt {.dirty.} = +template dollarImpl() {.dirty.} = result = "{" for key in items(s): if result.len > 1: result.add(", ") @@ -611,7 +615,7 @@ proc card*[A](s: OrderedSet[A]): int {.inline.} = ## <http://en.wikipedia.org/wiki/Cardinality>`_ of a set. result = s.counter -template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = +template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} = var h = s.first while h >= 0: var nxt = s.data[h].next diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 20e1bb7a9..17600b272 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -45,6 +45,59 @@ template withLock(t, x: untyped) = x release(t.lock) +template withValue*[A, B](t: var SharedTable[A, B], key: A, + value, body: untyped) = + ## retrieves the value at ``t[key]``. + ## `value` can be modified in the scope of the ``withValue`` call. + ## + ## .. code-block:: nim + ## + ## sharedTable.withValue(key, value) do: + ## # block is executed only if ``key`` in ``t`` + ## # value is threadsafe in block + ## value.name = "username" + ## value.uid = 1000 + ## + acquire(t.lock) + try: + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body + finally: + release(t.lock) + +template withValue*[A, B](t: var SharedTable[A, B], key: A, + value, body1, body2: untyped) = + ## retrieves the value at ``t[key]``. + ## `value` can be modified in the scope of the ``withValue`` call. + ## + ## .. code-block:: nim + ## + ## sharedTable.withValue(key, value) do: + ## # block is executed only if ``key`` in ``t`` + ## # value is threadsafe in block + ## value.name = "username" + ## value.uid = 1000 + ## do: + ## # block is executed when ``key`` not in ``t`` + ## raise newException(KeyError, "Key not found") + ## + acquire(t.lock) + try: + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body1 + else: + body2 + finally: + release(t.lock) + proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = ## retrieves the value at ``t[key]``. The value can be modified. ## If `key` is not in `t`, the ``KeyError`` exception is raised. diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index e4ec05b1c..be3507137 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -72,14 +72,14 @@ proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B], key: A, val: B, hc: Hash, h: Hash) = rawInsertImpl() -template addImpl(enlarge) {.dirty, immediate.} = +template addImpl(enlarge) {.dirty.} = if mustRehash(t.dataLen, t.counter): enlarge(t) var hc: Hash var j = rawGetDeep(t, key, hc) rawInsert(t, t.data, key, val, hc, j) inc(t.counter) -template maybeRehashPutImpl(enlarge) {.dirty, immediate.} = +template maybeRehashPutImpl(enlarge) {.oldimmediate, dirty.} = if mustRehash(t.dataLen, t.counter): enlarge(t) index = rawGetKnownHC(t, key, hc) @@ -87,13 +87,13 @@ template maybeRehashPutImpl(enlarge) {.dirty, immediate.} = rawInsert(t, t.data, key, val, hc, index) inc(t.counter) -template putImpl(enlarge) {.dirty, immediate.} = +template putImpl(enlarge) {.oldimmediate, dirty.} = var hc: Hash var index = rawGet(t, key, hc) if index >= 0: t.data[index].val = val else: maybeRehashPutImpl(enlarge) -template mgetOrPutImpl(enlarge) {.dirty, immediate.} = +template mgetOrPutImpl(enlarge) {.dirty.} = var hc: Hash var index = rawGet(t, key, hc) if index < 0: @@ -102,7 +102,7 @@ template mgetOrPutImpl(enlarge) {.dirty, immediate.} = # either way return modifiable val result = t.data[index].val -template hasKeyOrPutImpl(enlarge) {.dirty, immediate.} = +template hasKeyOrPutImpl(enlarge) {.dirty.} = var hc: Hash var index = rawGet(t, key, hc) if index < 0: @@ -110,18 +110,24 @@ template hasKeyOrPutImpl(enlarge) {.dirty, immediate.} = maybeRehashPutImpl(enlarge) else: result = true -template delImpl() {.dirty, immediate.} = +proc default[T](t: typedesc[T]): T {.inline.} = discard + +template delImpl() {.dirty.} = var hc: Hash var i = rawGet(t, key, hc) let msk = maxHash(t) if i >= 0: t.data[i].hcode = 0 + t.data[i].key = default(type(t.data[i].key)) + t.data[i].val = default(type(t.data[i].val)) dec(t.counter) block outer: while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 var j = i # The correctness of this depends on (h+1) in nextTry, var r = j # though may be adaptable to other simple sequences. t.data[i].hcode = 0 # mark current EMPTY + t.data[i].key = default(type(t.data[i].key)) + t.data[i].val = default(type(t.data[i].val)) while true: i = (i + 1) and msk # increment mod table size if isEmpty(t.data[i].hcode): # end of collision cluster; So all done @@ -133,3 +139,10 @@ template delImpl() {.dirty, immediate.} = t.data[j] = t.data[i] else: shallowCopy(t.data[j], t.data[i]) # data[j] will be marked EMPTY next loop + +template clearImpl() {.dirty.} = + for i in 0 .. <t.data.len: + t.data[i].hcode = 0 + t.data[i].key = default(type(t.data[i].key)) + t.data[i].val = default(type(t.data[i].val)) + t.counter = 0 diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 2ed0d2034..9308095aa 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -16,6 +16,39 @@ ## semantics, this means that ``=`` performs a copy of the hash table. ## For **reference** semantics use the ``Ref`` variant: ``TableRef``, ## ``OrderedTableRef``, ``CountTableRef``. +## To give an example, when `a` is a Table, then `var b = a` gives `b` +## as a new independent table. b is initialised with the contents of `a`. +## Changing `b` does not affect `a` and vice versa: +## +## .. code-block:: +## import tables +## +## var +## a = {1: "one", 2: "two"}.toTable # creates a Table +## b = a +## +## echo a, b # output: {1: one, 2: two}{1: one, 2: two} +## +## b[3] = "three" +## echo a, b # output: {1: one, 2: two}{1: one, 2: two, 3: three} +## echo a == b # output: false +## +## On the other hand, when `a` is a TableRef instead, then changes to `b` also affect `a`. +## Both `a` and `b` reference the same data structure: +## +## .. code-block:: +## import tables +## +## var +## a = {1: "one", 2: "two"}.newTable # creates a TableRef +## b = a +## +## echo a, b # output: {1: one, 2: two}{1: one, 2: two} +## +## b[3] = "three" +## echo a, b # output: {1: one, 2: two, 3: three}{1: one, 2: two, 3: three} +## echo a == b # output: true +## ## ## If you are using simple standard types like ``int`` or ``string`` for the ## keys of the table you won't have any problems, but as soon as you try to use @@ -80,11 +113,15 @@ type {.deprecated: [TTable: Table, PTable: TableRef].} -template maxHash(t): expr {.immediate.} = high(t.data) -template dataLen(t): expr = len(t.data) +template maxHash(t): untyped = high(t.data) +template dataLen(t): untyped = len(t.data) include tableimpl +proc clear*[A, B](t: Table[A, B] | TableRef[A, B]) = + ## Resets the table so that it is empty. + clearImpl() + proc rightSize*(count: Natural): int {.inline.} = ## Return the value of `initialSize` to support `count` items. ## @@ -98,7 +135,7 @@ proc len*[A, B](t: Table[A, B]): int = ## returns the number of keys in `t`. result = t.counter -template get(t, key): untyped {.immediate.} = +template get(t, key): untyped = ## retrieves the value at ``t[key]``. The value can be modified. ## If `key` is not in `t`, the ``KeyError`` exception is raised. mixin rawGet @@ -111,7 +148,7 @@ template get(t, key): untyped {.immediate.} = else: raise newException(KeyError, "key not found") -template getOrDefaultImpl(t, key): untyped {.immediate.} = +template getOrDefaultImpl(t, key): untyped = mixin rawGet var hc: Hash var index = rawGet(t, key, hc) @@ -136,6 +173,51 @@ proc mget*[A, B](t: var Table[A, B], key: A): var B {.deprecated.} = proc getOrDefault*[A, B](t: Table[A, B], key: A): B = getOrDefaultImpl(t, key) +template withValue*[A, B](t: var Table[A, B], key: A, + value, body: untyped) = + ## retrieves the value at ``t[key]``. + ## `value` can be modified in the scope of the ``withValue`` call. + ## + ## .. code-block:: nim + ## + ## sharedTable.withValue(key, value) do: + ## # block is executed only if ``key`` in ``t`` + ## value.name = "username" + ## value.uid = 1000 + ## + mixin rawGet + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body + +template withValue*[A, B](t: var Table[A, B], key: A, + value, body1, body2: untyped) = + ## retrieves the value at ``t[key]``. + ## `value` can be modified in the scope of the ``withValue`` call. + ## + ## .. code-block:: nim + ## + ## table.withValue(key, value) do: + ## # block is executed only if ``key`` in ``t`` + ## value.name = "username" + ## value.uid = 1000 + ## do: + ## # block is executed when ``key`` not in ``t`` + ## raise newException(KeyError, "Key not found") + ## + mixin rawGet + var hc: Hash + var index = rawGet(t, key, hc) + let hasKey = index >= 0 + if hasKey: + var value {.inject.} = addr(t.data[index].val) + body1 + else: + body2 + iterator allValues*[A, B](t: Table[A, B]; key: A): B = ## iterates over any value in the table `t` that belongs to the given `key`. var h: Hash = hash(key) and high(t.data) @@ -229,7 +311,7 @@ proc toTable*[A, B](pairs: openArray[(A, result = initTable[A, B](rightSize(pairs.len)) for key, val in items(pairs): result[key] = val -template dollarImpl(): stmt {.dirty.} = +template dollarImpl(): untyped {.dirty.} = if t.len == 0: result = "{:}" else: @@ -249,18 +331,17 @@ proc hasKey*[A, B](t: TableRef[A, B], key: A): bool = ## returns true iff `key` is in the table `t`. result = t[].hasKey(key) -template equalsImpl() = +template equalsImpl(t) = if s.counter == t.counter: # different insertion orders mean different 'data' seqs, so we have # to use the slow route here: for key, val in s: - # prefix notation leads to automatic dereference in case of PTable if not t.hasKey(key): return false - if t[key] != val: return false + if t.getOrDefault(key) != val: return false return true proc `==`*[A, B](s, t: Table[A, B]): bool = - equalsImpl() + equalsImpl(t) proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = ## Index the collection with the proc provided. @@ -350,7 +431,7 @@ proc `$`*[A, B](t: TableRef[A, B]): string = proc `==`*[A, B](s, t: TableRef[A, B]): bool = if isNil(s): result = isNil(t) elif isNil(t): result = false - else: equalsImpl() + else: equalsImpl(t[]) proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. @@ -376,7 +457,13 @@ proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = ## returns the number of keys in `t`. result = t.counter -template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = +proc clear*[A, B](t: OrderedTable[A, B] | OrderedTableRef[A, B]) = + ## Resets the table so that it is empty. + clearImpl() + t.first = -1 + t.last = -1 + +template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} = var h = t.first while h >= 0: var nxt = t.data[h].next @@ -562,7 +649,7 @@ proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} = ## returns the number of keys in `t`. result = t.counter -template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = +template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} = var h = t.first while h >= 0: var nxt = t.data[h].next @@ -663,6 +750,27 @@ proc sort*[A, B](t: OrderedTableRef[A, B], ## contrast to the `sort` for count tables). t[].sort(cmp) +proc del*[A, B](t: var OrderedTable[A, B], key: A) = + ## deletes `key` from ordered hash table `t`. O(n) comlexity. + var prev = -1 + let hc = hash(key) + forAllOrderedPairs: + if t.data[h].hcode == hc: + if t.first == h: + t.first = t.data[h].next + else: + t.data[prev].next = t.data[h].next + var zeroValue : type(t.data[h]) + t.data[h] = zeroValue + dec t.counter + break + else: + prev = h + +proc del*[A, B](t: var OrderedTableRef[A, B], key: A) = + ## deletes `key` from ordered hash table `t`. O(n) comlexity. + t[].del(key) + # ------------------------------ count tables ------------------------------- type @@ -678,6 +786,11 @@ proc len*[A](t: CountTable[A]): int = ## returns the number of keys in `t`. result = t.counter +proc clear*[A](t: CountTable[A] | CountTableRef[A]) = + ## Resets the table so that it is empty. + clearImpl() + t.counter = 0 + iterator pairs*[A](t: CountTable[A]): (A, int) = ## iterates over any (key, value) pair in the table `t`. for h in 0..high(t.data): @@ -711,7 +824,7 @@ proc rawGet[A](t: CountTable[A], key: A): int = h = nextTry(h, high(t.data)) result = -1 - h # < 0 => MISSING; insert idx = -1 - result -template ctget(t, key: untyped): untyped {.immediate.} = +template ctget(t, key: untyped): untyped = var index = rawGet(t, key) if index >= 0: result = t.data[index].val else: @@ -984,6 +1097,26 @@ when isMainModule: s3[p1] = 30_000 s3[p2] = 45_000 + block: # Ordered table should preserve order after deletion + var + s4 = initOrderedTable[int, int]() + s4[1] = 1 + s4[2] = 2 + s4[3] = 3 + + var prev = 0 + for i in s4.values: + doAssert(prev < i) + prev = i + + s4.del(2) + doAssert(2 notin s4) + doAssert(s4.len == 2) + prev = 0 + for i in s4.values: + doAssert(prev < i) + prev = i + var t1 = initCountTable[string]() t2 = initCountTable[string]() @@ -1040,3 +1173,12 @@ when isMainModule: doAssert 0 == t.getOrDefault(testKey) t.inc(testKey,3) doAssert 3 == t.getOrDefault(testKey) + + # Clear tests + var clearTable = newTable[int, string]() + clearTable[42] = "asd" + clearTable[123123] = "piuyqwb " + doAssert clearTable[42] == "asd" + clearTable.clear() + doAssert(not clearTable.hasKey(123123)) + doAssert clearTable.getOrDefault(42) == nil diff --git a/lib/pure/concurrency/cpuload.nim b/lib/pure/concurrency/cpuload.nim index 22598b5c9..b0fd002ed 100644 --- a/lib/pure/concurrency/cpuload.nim +++ b/lib/pure/concurrency/cpuload.nim @@ -79,6 +79,8 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice = inc s.calls when not defined(testing) and isMainModule: + import random + proc busyLoop() = while true: discard random(80) diff --git a/lib/pure/future.nim b/lib/pure/future.nim index 3793edc8b..67975cfcb 100644 --- a/lib/pure/future.nim +++ b/lib/pure/future.nim @@ -29,28 +29,24 @@ proc createProcType(p, b: NimNode): NimNode {.compileTime.} = of nnkExprColonExpr: identDefs.add ident[0] identDefs.add ident[1] - of nnkIdent: + else: identDefs.add newIdentNode("i" & $i) identDefs.add(ident) - else: - error("Incorrect type list in proc type declaration.") identDefs.add newEmptyNode() formalParams.add identDefs - of nnkIdent: + else: var identDefs = newNimNode(nnkIdentDefs) identDefs.add newIdentNode("i0") identDefs.add(p) identDefs.add newEmptyNode() formalParams.add identDefs - else: - error("Incorrect type list in proc type declaration.") result.add formalParams result.add newEmptyNode() #echo(treeRepr(result)) #echo(result.toStrLit()) -macro `=>`*(p, b: expr): expr {.immediate.} = +macro `=>`*(p, b: untyped): untyped = ## Syntax sugar for anonymous procedures. ## ## .. code-block:: nim @@ -111,7 +107,7 @@ macro `=>`*(p, b: expr): expr {.immediate.} = #echo(result.toStrLit()) #return result # TODO: Bug? -macro `->`*(p, b: expr): expr {.immediate.} = +macro `->`*(p, b: untyped): untyped = ## Syntax sugar for procedure types. ## ## .. code-block:: nim @@ -129,7 +125,7 @@ macro `->`*(p, b: expr): expr {.immediate.} = type ListComprehension = object var lc*: ListComprehension -macro `[]`*(lc: ListComprehension, comp, typ: expr): expr = +macro `[]`*(lc: ListComprehension, comp, typ: untyped): untyped = ## List comprehension, returns a sequence. `comp` is the actual list ## comprehension, for example ``x | (x <- 1..10, x mod 2 == 0)``. `typ` is ## the type that will be stored inside the result seq. diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index d620e816e..1fe0b297b 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -433,7 +433,7 @@ proc htmlTag*(n: XmlNode): HtmlTag = proc htmlTag*(s: string): HtmlTag = ## converts `s` to a ``HtmlTag``. If `s` is no HTML tag, ``tagUnknown`` is ## returned. - let s = if allLower(s): s else: s.toLower + let s = if allLower(s): s else: toLowerAscii(s) result = toHtmlTag(s) proc entityToUtf8*(entity: string): string = @@ -464,12 +464,18 @@ proc untilElementEnd(x: var XmlParser, result: XmlNode, case x.kind of xmlElementStart, xmlElementOpen: case result.htmlTag - of tagLi, tagP, tagDt, tagDd, tagInput, tagOption: - # some tags are common to have no ``</end>``, like ``<li>``: + of tagP, tagInput, tagOption: + # some tags are common to have no ``</end>``, like ``<li>`` but + # allow ``<p>`` in `<dd>`, `<dt>` and ``<li>`` in next case if htmlTag(x.elemName) in {tagLi, tagP, tagDt, tagDd, tagInput, tagOption}: errors.add(expected(x, result)) break + of tagDd, tagDt, tagLi: + if htmlTag(x.elemName) in {tagLi, tagDt, tagDd, tagInput, + tagOption}: + errors.add(expected(x, result)) + break of tagTd, tagTh: if htmlTag(x.elemName) in {tagTr, tagTd, tagTh, tagTfoot, tagThead}: errors.add(expected(x, result)) @@ -513,13 +519,13 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = errors.add(errorMsg(x)) next(x) of xmlElementStart: - result = newElement(x.elemName.toLower) + result = newElement(toLowerAscii(x.elemName)) next(x) untilElementEnd(x, result, errors) of xmlElementEnd: errors.add(errorMsg(x, "unexpected ending tag: " & x.elemName)) of xmlElementOpen: - result = newElement(x.elemName.toLower) + result = newElement(toLowerAscii(x.elemName)) next(x) result.attrs = newStringTable() while true: diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 603763386..778ca2cbb 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -79,15 +79,18 @@ ## constructor should be used for this purpose. However, ## currently only basic authentication is supported. -import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, math +import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, + math, random, httpcore import asyncnet, asyncdispatch import nativesockets +export httpcore except parseHeader # TODO: The ``except`` doesn't work + type Response* = tuple[ version: string, status: string, - headers: StringTableRef, + headers: HttpHeaders, body: string] Proxy* = ref object @@ -164,7 +167,7 @@ proc parseChunks(s: Socket, timeout: int): string = # Trailer headers will only be sent if the request specifies that we want # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 -proc parseBody(s: Socket, headers: StringTableRef, timeout: int): string = +proc parseBody(s: Socket, headers: HttpHeaders, httpVersion: string, timeout: int): string = result = "" if headers.getOrDefault"Transfer-Encoding" == "chunked": result = parseChunks(s, timeout) @@ -190,7 +193,7 @@ proc parseBody(s: Socket, headers: StringTableRef, timeout: int): string = # -REGION- Connection: Close # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5 - if headers.getOrDefault"Connection" == "close": + if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0": var buf = "" while true: buf = newString(4000) @@ -204,7 +207,7 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response = var linei = 0 var fullyRead = false var line = "" - result.headers = newStringTable(modeCaseInsensitive) + result.headers = newHttpHeaders() while true: line = "" linei = 0 @@ -239,10 +242,14 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response = inc(linei) # Skip : result.headers[name] = line[linei.. ^1].strip() + # Ensure the server isn't trying to DoS us. + if result.headers.len > headerLimit: + httpError("too many headers") + if not fullyRead: httpError("Connection was closed before full request has been made") if getBody: - result.body = parseBody(s, result.headers, timeout) + result.body = parseBody(s, result.headers, result.version, timeout) else: result.body = "" @@ -335,7 +342,7 @@ proc addFiles*(p: var MultipartData, xs: openarray[tuple[name, file: string]]): var m = newMimetypes() for name, file in xs.items: var contentType: string - let (dir, fName, ext) = splitFile(file) + let (_, fName, ext) = splitFile(file) if ext.len > 0: contentType = m.getMimetype(ext[1..ext.high], nil) p.add(name, readFile(file), fName & ext, contentType) @@ -392,6 +399,59 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", var hostUrl = if proxy == nil: r else: parseUri(url) var headers = substr(httpMethod, len("http")) # TODO: Use generateHeaders further down once it supports proxies. + + var s = newSocket() + defer: s.close() + if s == nil: raiseOSError(osLastError()) + var port = net.Port(80) + if r.scheme == "https": + when defined(ssl): + sslContext.wrapSocket(s) + port = net.Port(443) + else: + raise newException(HttpRequestError, + "SSL support is not available. Cannot connect over SSL.") + if r.port != "": + port = net.Port(r.port.parseInt) + + + # get the socket ready. If we are connecting through a proxy to SSL, + # send the appropiate CONNECT header. If not, simply connect to the proper + # host (which may still be the proxy, for normal HTTP) + if proxy != nil and hostUrl.scheme == "https": + when defined(ssl): + var connectHeaders = "CONNECT " + let targetPort = if hostUrl.port == "": 443 else: hostUrl.port.parseInt + connectHeaders.add(hostUrl.hostname) + connectHeaders.add(":" & $targetPort) + connectHeaders.add(" HTTP/1.1\c\L") + connectHeaders.add("Host: " & hostUrl.hostname & ":" & $targetPort & "\c\L") + if proxy.auth != "": + let auth = base64.encode(proxy.auth, newline = "") + connectHeaders.add("Proxy-Authorization: basic " & auth & "\c\L") + connectHeaders.add("\c\L") + if timeout == -1: + s.connect(r.hostname, port) + else: + s.connect(r.hostname, port, timeout) + + s.send(connectHeaders) + let connectResult = parseResponse(s, false, timeout) + if not connectResult.status.startsWith("200"): + raise newException(HttpRequestError, + "The proxy server rejected a CONNECT request, " & + "so a secure connection could not be established.") + sslContext.wrapConnectedSocket(s, handshakeAsClient) + else: + raise newException(HttpRequestError, "SSL support not available. Cannot connect via proxy over SSL") + else: + if timeout == -1: + s.connect(r.hostname, port) + else: + s.connect(r.hostname, port, timeout) + + + # now that the socket is ready, prepare the headers if proxy == nil: headers.add ' ' if r.path[0] != '/': headers.add '/' @@ -415,29 +475,13 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", add(headers, "Proxy-Authorization: basic " & auth & "\c\L") add(headers, extraHeaders) add(headers, "\c\L") - var s = newSocket() - if s == nil: raiseOSError(osLastError()) - var port = net.Port(80) - if r.scheme == "https": - when defined(ssl): - sslContext.wrapSocket(s) - port = net.Port(443) - else: - raise newException(HttpRequestError, - "SSL support is not available. Cannot connect over SSL.") - if r.port != "": - port = net.Port(r.port.parseInt) - if timeout == -1: - s.connect(r.hostname, port) - else: - s.connect(r.hostname, port, timeout) + # headers are ready. send them, await the result, and close the socket. s.send(headers) if body != "": s.send(body) result = parseResponse(s, httpMethod != "httpHEAD", timeout) - s.close() proc request*(url: string, httpMethod = httpGET, extraHeaders = "", body = "", sslContext = defaultSSLContext, timeout = -1, @@ -455,7 +499,7 @@ proc redirection(status: string): bool = if status.startsWith(i): return true -proc getNewLocation(lastURL: string, headers: StringTableRef): string = +proc getNewLocation(lastURL: string, headers: HttpHeaders): string = result = headers.getOrDefault"Location" if result == "": httpError("location header expected") # Relative URLs. (Not part of the spec, but soon will be.) @@ -624,10 +668,10 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, ## ``sslContext`` specifies the SSL context to use for HTTPS requests. new result result.headers = newStringTable(modeCaseInsensitive) - result.userAgent = defUserAgent + result.userAgent = userAgent result.maxRedirects = maxRedirects when defined(ssl): - result.sslContext = net.SslContext(sslContext) + result.sslContext = sslContext proc close*(client: AsyncHttpClient) = ## Closes any connections held by the HTTP client. @@ -678,7 +722,8 @@ proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} = # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 proc parseBody(client: AsyncHttpClient, - headers: StringTableRef): Future[string] {.async.} = + headers: HttpHeaders, + httpVersion: string): Future[string] {.async.} = result = "" if headers.getOrDefault"Transfer-Encoding" == "chunked": result = await parseChunks(client) @@ -700,7 +745,7 @@ proc parseBody(client: AsyncHttpClient, # -REGION- Connection: Close # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5 - if headers.getOrDefault"Connection" == "close": + if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0": var buf = "" while true: buf = await client.socket.recvFull(4000) @@ -713,7 +758,7 @@ proc parseResponse(client: AsyncHttpClient, var linei = 0 var fullyRead = false var line = "" - result.headers = newStringTable(modeCaseInsensitive) + result.headers = newHttpHeaders() while true: linei = 0 line = await client.socket.recvLine() @@ -748,10 +793,13 @@ proc parseResponse(client: AsyncHttpClient, inc(linei) # Skip : result.headers[name] = line[linei.. ^1].strip() + if result.headers.len > headerLimit: + httpError("too many headers") + if not fullyRead: httpError("Connection was closed before full request has been made") if getBody: - result.body = await parseBody(client, result.headers) + result.body = await parseBody(client, result.headers, result.version) else: result.body = "" diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim new file mode 100644 index 000000000..562d16c19 --- /dev/null +++ b/lib/pure/httpcore.nim @@ -0,0 +1,198 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Contains functionality shared between the ``httpclient`` and +## ``asynchttpserver`` modules. + +import tables, strutils, parseutils + +type + HttpHeaders* = ref object + table*: TableRef[string, seq[string]] + + HttpHeaderValues* = distinct seq[string] + + HttpCode* = enum + Http100 = "100 Continue", + Http101 = "101 Switching Protocols", + Http200 = "200 OK", + Http201 = "201 Created", + Http202 = "202 Accepted", + Http203 = "203 Non-Authoritative Information", + Http204 = "204 No Content", + Http205 = "205 Reset Content", + Http206 = "206 Partial Content", + Http300 = "300 Multiple Choices", + Http301 = "301 Moved Permanently", + Http302 = "302 Found", + Http303 = "303 See Other", + Http304 = "304 Not Modified", + Http305 = "305 Use Proxy", + Http307 = "307 Temporary Redirect", + Http400 = "400 Bad Request", + Http401 = "401 Unauthorized", + Http403 = "403 Forbidden", + Http404 = "404 Not Found", + Http405 = "405 Method Not Allowed", + Http406 = "406 Not Acceptable", + Http407 = "407 Proxy Authentication Required", + Http408 = "408 Request Timeout", + Http409 = "409 Conflict", + Http410 = "410 Gone", + Http411 = "411 Length Required", + Http412 = "412 Precondition Failed", + Http413 = "413 Request Entity Too Large", + Http414 = "414 Request-URI Too Long", + Http415 = "415 Unsupported Media Type", + Http416 = "416 Requested Range Not Satisfiable", + Http417 = "417 Expectation Failed", + Http418 = "418 I'm a teapot", + Http421 = "421 Misdirected Request", + Http422 = "422 Unprocessable Entity", + Http426 = "426 Upgrade Required", + Http428 = "428 Precondition Required", + Http429 = "429 Too Many Requests", + Http431 = "431 Request Header Fields Too Large", + Http451 = "451 Unavailable For Legal Reasons", + Http500 = "500 Internal Server Error", + Http501 = "501 Not Implemented", + Http502 = "502 Bad Gateway", + Http503 = "503 Service Unavailable", + Http504 = "504 Gateway Timeout", + Http505 = "505 HTTP Version Not Supported" + + HttpVersion* = enum + HttpVer11, + HttpVer10 + +const headerLimit* = 10_000 + +proc newHttpHeaders*(): HttpHeaders = + new result + result.table = newTable[string, seq[string]]() + +proc newHttpHeaders*(keyValuePairs: + openarray[tuple[key: string, val: string]]): HttpHeaders = + var pairs: seq[tuple[key: string, val: seq[string]]] = @[] + for pair in keyValuePairs: + pairs.add((pair.key.toLower(), @[pair.val])) + new result + result.table = newTable[string, seq[string]](pairs) + +proc clear*(headers: HttpHeaders) = + headers.table.clear() + +proc `[]`*(headers: HttpHeaders, key: string): HttpHeaderValues = + ## Returns the values associated with the given ``key``. If the returned + ## values are passed to a procedure expecting a ``string``, the first + ## value is automatically picked. If there are + ## no values associated with the key, an exception is raised. + ## + ## To access multiple values of a key, use the overloaded ``[]`` below or + ## to get all of them access the ``table`` field directly. + return headers.table[key.toLower].HttpHeaderValues + +converter toString*(values: HttpHeaderValues): string = + return seq[string](values)[0] + +proc `[]`*(headers: HttpHeaders, key: string, i: int): string = + ## Returns the ``i``'th value associated with the given key. If there are + ## no values associated with the key or the ``i``'th value doesn't exist, + ## an exception is raised. + return headers.table[key.toLower][i] + +proc `[]=`*(headers: HttpHeaders, key, value: string) = + ## Sets the header entries associated with ``key`` to the specified value. + ## Replaces any existing values. + headers.table[key.toLower] = @[value] + +proc `[]=`*(headers: HttpHeaders, key: string, value: seq[string]) = + ## Sets the header entries associated with ``key`` to the specified list of + ## values. + ## Replaces any existing values. + headers.table[key.toLower] = value + +proc add*(headers: HttpHeaders, key, value: string) = + ## Adds the specified value to the specified key. Appends to any existing + ## values associated with the key. + if not headers.table.hasKey(key.toLower): + headers.table[key.toLower] = @[value] + else: + headers.table[key.toLower].add(value) + +iterator pairs*(headers: HttpHeaders): tuple[key, value: string] = + ## Yields each key, value pair. + for k, v in headers.table: + for value in v: + yield (k, value) + +proc contains*(values: HttpHeaderValues, value: string): bool = + ## Determines if ``value`` is one of the values inside ``values``. Comparison + ## is performed without case sensitivity. + for val in seq[string](values): + if val.toLower == value.toLower: return true + +proc hasKey*(headers: HttpHeaders, key: string): bool = + return headers.table.hasKey(key.toLower()) + +proc getOrDefault*(headers: HttpHeaders, key: string, + default = @[""].HttpHeaderValues): HttpHeaderValues = + ## Returns the values associated with the given ``key``. If there are no + ## values associated with the key, then ``default`` is returned. + if headers.hasKey(key): + return headers[key] + else: + return default + +proc len*(headers: HttpHeaders): int = return headers.table.len + +proc parseList(line: string, list: var seq[string], start: int): int = + var i = 0 + var current = "" + while line[start + i] notin {'\c', '\l', '\0'}: + i += line.skipWhitespace(start + i) + i += line.parseUntil(current, {'\c', '\l', ','}, start + i) + list.add(current) + if line[start + i] == ',': + i.inc # Skip , + current.setLen(0) + +proc parseHeader*(line: string): tuple[key: string, value: seq[string]] = + ## Parses a single raw header HTTP line into key value pairs. + ## + ## Used by ``asynchttpserver`` and ``httpclient`` internally and should not + ## be used by you. + result.value = @[] + var i = 0 + i = line.parseUntil(result.key, ':') + inc(i) # skip : + if i < len(line): + i += parseList(line, result.value, i) + else: + result.value = @[] + +proc `==`*(protocol: tuple[orig: string, major, minor: int], + ver: HttpVersion): bool = + let major = + case ver + of HttpVer11, HttpVer10: 1 + let minor = + case ver + of HttpVer11: 1 + of HttpVer10: 0 + result = protocol.major == major and protocol.minor == minor + +when isMainModule: + var test = newHttpHeaders() + test["Connection"] = @["Upgrade", "Close"] + doAssert test["Connection", 0] == "Upgrade" + doAssert test["Connection", 1] == "Close" + test.add("Connection", "Test") + doAssert test["Connection", 2] == "Test" + doAssert "upgrade" in test["Connection"] diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim new file mode 100644 index 000000000..a5d5d2c01 --- /dev/null +++ b/lib/pure/ioselectors.nim @@ -0,0 +1,255 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module allows high-level and efficient I/O multiplexing. +## +## Supported OS primitives: ``epoll``, ``kqueue``, ``poll`` and +## Windows ``select``. +## +## To use threadsafe version of this module, it needs to be compiled +## with both ``-d:threadsafe`` and ``--threads:on`` options. +## +## Supported features: files, sockets, pipes, timers, processes, signals +## and user events. +## +## Fully supported OS: MacOSX, FreeBSD, OpenBSD, NetBSD, Linux. +## +## Partially supported OS: Windows (only sockets and user events), +## Solaris (files, sockets, handles and user events). +## +## TODO: ``/dev/poll``, ``event ports`` and filesystem events. + +import os + +const hasThreadSupport = compileOption("threads") and defined(threadsafe) + +const supportedPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(linux) + +const bsdPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) + + +when defined(nimdoc): + type + Selector*[T] = ref object + ## An object which holds descriptors to be checked for read/write status + + Event* {.pure.} = enum + ## An enum which hold event types + Read, ## Descriptor is available for read + Write, ## Descriptor is available for write + Timer, ## Timer descriptor is completed + Signal, ## Signal is raised + Process, ## Process is finished + Vnode, ## Currently not supported + User, ## User event is raised + Error ## Error happens while waiting, for descriptor + + ReadyKey*[T] = object + ## An object which holds result for descriptor + fd* : int ## file/socket descriptor + events*: set[Event] ## set of events + data*: T ## application-defined data + + SelectEvent* = object + ## An object which holds user defined event + + proc newSelector*[T](): Selector[T] = + ## Creates a new selector + + proc close*[T](s: Selector[T]) = + ## Closes selector + + proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], + data: T) = + ## Registers file/socket descriptor ``fd`` to selector ``s`` + ## with events set in ``events``. The ``data`` is application-defined + ## data, which to be passed when event happens. + + proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + ## Update file/socket descriptor ``fd``, registered in selector + ## ``s`` with new events set ``event``. + + proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + ## Registers timer notification with ``timeout`` in milliseconds + ## to selector ``s``. + ## If ``oneshot`` is ``true`` timer will be notified only once. + ## Set ``oneshot`` to ``false`` if your want periodic notifications. + ## The ``data`` is application-defined data, which to be passed, when + ## time limit expired. + + proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + ## Registers Unix signal notification with ``signal`` to selector + ## ``s``. The ``data`` is application-defined data, which to be + ## passed, when signal raises. + ## + ## This function is not supported for ``Windows``. + + proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + ## Registers process id (pid) notification when process has + ## exited to selector ``s``. + ## The ``data`` is application-defined data, which to be passed, when + ## process with ``pid`` has exited. + + proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + ## Registers selector event ``ev`` to selector ``s``. + ## ``data`` application-defined data, which to be passed, when + ## ``ev`` happens. + + proc newSelectEvent*(): SelectEvent = + ## Creates new event ``SelectEvent``. + + proc setEvent*(ev: SelectEvent) = + ## Trigger event ``ev``. + + proc close*(ev: SelectEvent) = + ## Closes selector event ``ev``. + + proc unregister*[T](s: Selector[T], ev: SelectEvent) = + ## Unregisters event ``ev`` from selector ``s``. + + proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = + ## Unregisters file/socket descriptor ``fd`` from selector ``s``. + + proc flush*[T](s: Selector[T]) = + ## Flushes all changes was made to kernel pool/queue. + ## This function is usefull only for BSD and MacOS, because + ## kqueue supports bulk changes to be made. + ## On Linux/Windows and other Posix compatible operation systems, + ## ``flush`` is alias for `discard`. + + proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + ## Process call waiting for events registered in selector ``s``. + ## The ``timeout`` argument specifies the minimum number of milliseconds + ## the function will be blocked, if no events are not ready. Specifying a + ## timeout of ``-1`` causes function to block indefinitely. + ## All available events will be stored in ``results`` array. + ## + ## Function returns number of triggered events. + + proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + ## Process call waiting for events registered in selector ``s``. + ## The ``timeout`` argument specifies the minimum number of milliseconds + ## the function will be blocked, if no events are not ready. Specifying a + ## timeout of -1 causes function to block indefinitely. + ## + ## Function returns sequence of triggered events. + + template isEmpty*[T](s: Selector[T]): bool = + ## Returns ``true``, if there no registered events or descriptors + ## in selector. + + template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + ## retrieves the application-data assigned with descriptor ``fd`` + ## to ``value``. This ``value`` can be modified in the scope of + ## the ``withData`` call. + ## + ## .. code-block:: nim + ## + ## s.withData(fd, value) do: + ## # block is executed only if ``fd`` registered in selector ``s`` + ## value.uid = 1000 + ## + + template withData*[T](s: Selector[T], fd: SocketHandle, value, + body1, body2: untyped) = + ## retrieves the application-data assigned with descriptor ``fd`` + ## to ``value``. This ``value`` can be modified in the scope of + ## the ``withData`` call. + ## + ## .. code-block:: nim + ## + ## s.withData(fd, value) do: + ## # block is executed only if ``fd`` registered in selector ``s``. + ## value.uid = 1000 + ## do: + ## # block is executed if ``fd`` not registered in selector ``s``. + ## raise + ## + +else: + when hasThreadSupport: + import locks + + type + SharedArray {.unchecked.}[T] = array[0..100, T] + + proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) + + proc deallocSharedArray[T](sa: ptr SharedArray[T]) = + deallocShared(cast[pointer](sa)) + type + Event* {.pure.} = enum + Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot + + ReadyKey*[T] = object + fd* : int + events*: set[Event] + data*: T + + SelectorKey[T] = object + ident: int + events: set[Event] + param: int + key: ReadyKey[T] + + when not defined(windows): + import posix + proc setNonBlocking(fd: cint) {.inline.} = + var x = fcntl(fd, F_GETFL, 0) + if x == -1: + raiseOSError(osLastError()) + else: + var mode = x or O_NONBLOCK + if fcntl(fd, F_SETFL, mode) == -1: + raiseOSError(osLastError()) + + template setKey(s, pident, pkeyfd, pevents, pparam, pdata) = + var skey = addr(s.fds[pident]) + skey.ident = pident + skey.events = pevents + skey.param = pparam + skey.key.fd = pkeyfd + skey.key.data = pdata + + when supportedPlatform: + template blockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + else: + if posix.sigprocmask(SIG_BLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + + template unblockSignals(newmask: var Sigset, oldmask: var Sigset) = + when hasThreadSupport: + if posix.pthread_sigmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + else: + if posix.sigprocmask(SIG_UNBLOCK, newmask, oldmask) == -1: + raiseOSError(osLastError()) + + when defined(linux): + include ioselects/ioselectors_epoll + elif bsdPlatform: + include ioselects/ioselectors_kqueue + elif defined(windows): + include ioselects/ioselectors_select + elif defined(solaris): + include ioselects/ioselectors_poll # need to replace it with event ports + else: + include ioselects/ioselectors_poll diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim new file mode 100644 index 000000000..92b2cdc07 --- /dev/null +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -0,0 +1,461 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Linux epoll(). + +import posix, times + +# Maximum number of events that can be returned +const MAX_EPOLL_RESULT_EVENTS = 64 + +type + SignalFdInfo* {.importc: "struct signalfd_siginfo", + header: "<sys/signalfd.h>", pure, final.} = object + ssi_signo*: uint32 + ssi_errno*: int32 + ssi_code*: int32 + ssi_pid*: uint32 + ssi_uid*: uint32 + ssi_fd*: int32 + ssi_tid*: uint32 + ssi_band*: uint32 + ssi_overrun*: uint32 + ssi_trapno*: uint32 + ssi_status*: int32 + ssi_int*: int32 + ssi_ptr*: uint64 + ssi_utime*: uint64 + ssi_stime*: uint64 + ssi_addr*: uint64 + pad* {.importc: "__pad".}: array[0..47, uint8] + + eventFdData {.importc: "eventfd_t", + header: "<sys/eventfd.h>", pure, final.} = uint64 + epoll_data {.importc: "union epoll_data", header: "<sys/epoll.h>", + pure, final.} = object + u64 {.importc: "u64".}: uint64 + epoll_event {.importc: "struct epoll_event", + header: "<sys/epoll.h>", pure, final.} = object + events: uint32 # Epoll events + data: epoll_data # User data variable + +const + EPOLL_CTL_ADD = 1 # Add a file descriptor to the interface. + EPOLL_CTL_DEL = 2 # Remove a file descriptor from the interface. + EPOLL_CTL_MOD = 3 # Change file descriptor epoll_event structure. + EPOLLIN = 0x00000001 + EPOLLOUT = 0x00000004 + EPOLLERR = 0x00000008 + EPOLLHUP = 0x00000010 + EPOLLRDHUP = 0x00002000 + EPOLLONESHOT = 1 shl 30 + +proc epoll_create(size: cint): cint + {.importc: "epoll_create", header: "<sys/epoll.h>".} +proc epoll_ctl(epfd: cint; op: cint; fd: cint; event: ptr epoll_event): cint + {.importc: "epoll_ctl", header: "<sys/epoll.h>".} +proc epoll_wait(epfd: cint; events: ptr epoll_event; maxevents: cint; + timeout: cint): cint + {.importc: "epoll_wait", header: "<sys/epoll.h>".} +proc timerfd_create(clock_id: ClockId, flags: cint): cint + {.cdecl, importc: "timerfd_create", header: "<sys/timerfd.h>".} +proc timerfd_settime(ufd: cint, flags: cint, + utmr: var Itimerspec, otmr: var Itimerspec): cint + {.cdecl, importc: "timerfd_settime", header: "<sys/timerfd.h>".} +proc signalfd(fd: cint, mask: var Sigset, flags: cint): cint + {.cdecl, importc: "signalfd", header: "<sys/signalfd.h>".} +proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "<sys/eventfd.h>".} +proc ulimit(cmd: cint): clong + {.importc: "ulimit", header: "<ulimit.h>", varargs.} + +when hasThreadSupport: + type + SelectorImpl[T] = object + epollFD : cint + maxFD : int + fds: ptr SharedArray[SelectorKey[T]] + count: int + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + epollFD : cint + maxFD : int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] +type + SelectEventImpl = object + efd: cint + SelectEvent* = ptr SelectEventImpl + +proc newSelector*[T](): Selector[T] = + var maxFD = int(ulimit(4, 0)) + doAssert(maxFD > 0) + + var epollFD = epoll_create(MAX_EPOLL_RESULT_EVENTS) + if epollFD < 0: + raiseOsError(osLastError()) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.epollFD = epollFD + result.maxFD = maxFD + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + else: + result = Selector[T]() + result.epollFD = epollFD + result.maxFD = maxFD + result.fds = newSeq[SelectorKey[T]](maxFD) + +proc close*[T](s: Selector[T]) = + if posix.close(s.epollFD) != 0: + raiseOSError(osLastError()) + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +proc newSelectEvent*(): SelectEvent = + let fdci = eventfd(0, 0) + if fdci == -1: + raiseOSError(osLastError()) + setNonBlocking(fdci) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.efd = fdci + +proc setEvent*(ev: SelectEvent) = + var data : uint64 = 1 + if posix.write(ev.efd, addr data, sizeof(uint64)) == -1: + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(ev.efd) + deallocShared(cast[pointer](ev)) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: + var epv = epoll_event(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + var epv = epoll_event(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + + if pkey.events == {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + else: + if events != {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + else: + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + + if pkey.events != {}: + if pkey.events * {Event.Read, Event.Write} != {}: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + elif Event.Timer in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + discard posix.close(fdi.cint) + dec(s.count) + elif Event.Signal in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(s.fds[fdi].param)) + unblockSignals(nmask, omask) + discard posix.close(fdi.cint) + dec(s.count) + elif Event.Process in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, SIGCHLD) + unblockSignals(nmask, omask) + discard posix.close(fdi.cint) + dec(s.count) + pkey.ident = 0 + pkey.events = {} + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.efd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + var + new_ts: Itimerspec + old_ts: Itimerspec + let fdi = timerfd_create(CLOCK_MONOTONIC, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var events = {Event.Timer} + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + if oneshot: + new_ts.it_interval.tv_sec = 0.Time + new_ts.it_interval.tv_nsec = 0 + new_ts.it_value.tv_sec = (timeout div 1_000).Time + new_ts.it_value.tv_nsec = (timeout %% 1_000) * 1_000_000 + incl(events, Event.Oneshot) + epv.events = epv.events or EPOLLONESHOT + else: + new_ts.it_interval.tv_sec = (timeout div 1000).Time + new_ts.it_interval.tv_nsec = (timeout %% 1_000) * 1_000_000 + new_ts.it_value.tv_sec = new_ts.it_interval.tv_sec + new_ts.it_value.tv_nsec = new_ts.it_interval.tv_nsec + + if timerfd_settime(fdi.cint, cint(0), new_ts, old_ts) == -1: + raiseOSError(osLastError()) + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, fdi, events, 0, data) + inc(s.count) + result = fdi + +proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, signal, {Event.Signal}, signal, data) + inc(s.count) + result = fdi + +proc registerProcess*[T](s: Selector, pid: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, posix.SIGCHLD) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + epv.events = EPOLLIN or EPOLLRDHUP + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, pid, {Event.Process, Event.Oneshot}, pid, data) + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = int(ev.efd) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, {Event.User}, 0, data) + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = ev.efd.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, ev.efd, addr epv) == -1: + raiseOSError(osLastError()) + inc(s.count) + +proc flush*[T](s: Selector[T]) = + discard + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var + resTable: array[MAX_EPOLL_RESULT_EVENTS, epoll_event] + maxres = MAX_EPOLL_RESULT_EVENTS + events: set[Event] = {} + i, k: int + + if maxres > len(results): + maxres = len(results) + + let count = epoll_wait(s.epollFD, addr(resTable[0]), maxres.cint, + timeout.cint) + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + i = 0 + k = 0 + while i < count: + let fdi = int(resTable[i].data.u64) + let pevents = resTable[i].events + var skey = addr(s.fds[fdi]) + doAssert(skey.ident != 0) + events = {} + + if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0: + events.incl(Event.Error) + if (pevents and EPOLLOUT) != 0: + events.incl(Event.Write) + if (pevents and EPOLLIN) != 0: + if Event.Read in skey.events: + events.incl(Event.Read) + elif Event.Timer in skey.events: + var data: uint64 = 0 + if posix.read(fdi.cint, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + events = {Event.Timer} + elif Event.Signal in skey.events: + var data = SignalFdInfo() + if posix.read(fdi.cint, addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseOsError(osLastError()) + events = {Event.Signal} + elif Event.Process in skey.events: + var data = SignalFdInfo() + if posix.read(fdi.cint, addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseOsError(osLastError()) + if cast[int](data.ssi_pid) == skey.param: + events = {Event.Process} + else: + inc(i) + continue + elif Event.User in skey.events: + var data: uint = 0 + if posix.read(fdi.cint, addr data, sizeof(uint)) != sizeof(uint): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + inc(i) + continue + else: + raiseOSError(err) + events = {Event.User} + + skey.key.events = events + results[k] = skey.key + inc(k) + + if Event.Oneshot in skey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + discard posix.close(fdi.cint) + skey.ident = 0 + skey.events = {} + dec(s.count) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_EPOLL_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim new file mode 100644 index 000000000..3e86f19aa --- /dev/null +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -0,0 +1,443 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements BSD kqueue(). + +import posix, times, kqueue + +const + # Maximum number of cached changes. + MAX_KQUEUE_CHANGE_EVENTS = 64 + # Maximum number of events that can be returned. + MAX_KQUEUE_RESULT_EVENTS = 64 + # SIG_IGN and SIG_DFL declared in posix.nim as variables, but we need them + # to be constants and GC-safe. + SIG_DFL = cast[proc(x: cint) {.noconv,gcsafe.}](0) + SIG_IGN = cast[proc(x: cint) {.noconv,gcsafe.}](1) + +when defined(macosx) or defined(freebsd): + when defined(macosx): + const MAX_DESCRIPTORS_ID = 29 # KERN_MAXFILESPERPROC (MacOS) + else: + const MAX_DESCRIPTORS_ID = 27 # KERN_MAXFILESPERPROC (FreeBSD) + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, + newp: pointer, newplen: int): cint + {.importc: "sysctl",header: """#include <sys/types.h> + #include <sys/sysctl.h>"""} +elif defined(netbsd) or defined(openbsd): + # OpenBSD and NetBSD don't have KERN_MAXFILESPERPROC, so we are using + # KERN_MAXFILES, because KERN_MAXFILES is always bigger, + # than KERN_MAXFILESPERPROC. + const MAX_DESCRIPTORS_ID = 7 # KERN_MAXFILES + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, + newp: pointer, newplen: int): cint + {.importc: "sysctl",header: """#include <sys/param.h> + #include <sys/sysctl.h>"""} + +when hasThreadSupport: + type + SelectorImpl[T] = object + kqFD : cint + maxFD : int + changesTable: array[MAX_KQUEUE_CHANGE_EVENTS, KEvent] + changesCount: int + fds: ptr SharedArray[SelectorKey[T]] + count: int + changesLock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + kqFD : cint + maxFD : int + changesTable: array[MAX_KQUEUE_CHANGE_EVENTS, KEvent] + changesCount: int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint +# SelectEvent is declared as `ptr` to be placed in `shared memory`, +# so you can share one SelectEvent handle between threads. +type SelectEvent* = ptr SelectEventImpl + +proc newSelector*[T](): Selector[T] = + var maxFD = 0.cint + var size = sizeof(cint) + var namearr = [1.cint, MAX_DESCRIPTORS_ID.cint] + # Obtain maximum number of file descriptors for process + if sysctl(addr(namearr[0]), 2, cast[pointer](addr maxFD), addr size, + nil, 0) != 0: + raiseOsError(osLastError()) + + var kqFD = kqueue() + if kqFD < 0: + raiseOsError(osLastError()) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.kqFD = kqFD + result.maxFD = maxFD.int + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + initLock(result.changesLock) + else: + result = Selector[T]() + result.kqFD = kqFD + result.maxFD = maxFD.int + result.fds = newSeq[SelectorKey[T]](maxFD) + +proc close*[T](s: Selector[T]) = + if posix.close(s.kqFD) != 0: + raiseOSError(osLastError()) + when hasThreadSupport: + deinitLock(s.changesLock) + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + +proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rfd)) + discard posix.close(cint(ev.wfd)) + deallocShared(cast[pointer](ev)) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +when hasThreadSupport: + template withChangeLock[T](s: Selector[T], body: untyped) = + acquire(s.changesLock) + {.locks: [s.changesLock].}: + try: + body + finally: + release(s.changesLock) +else: + template withChangeLock(s, body: untyped) = + body + +template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, + nflags: cushort, nfflags: cuint, ndata: int, + nudata: pointer) = + mixin withChangeLock + s.withChangeLock(): + s.changesTable[s.changesCount] = KEvent(ident: nident, + filter: nfilter, flags: nflags, + fflags: nfflags, data: ndata, + udata: nudata) + inc(s.changesCount) + if s.changesCount == MAX_KQUEUE_CHANGE_EVENTS: + if kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + nil, 0, nil) == -1: + raiseOSError(osLastError()) + s.changesCount = 0 + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: + if Event.Read in events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if Event.Write in events: + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + pkey.events = events + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + let events = if oneshot: {Event.Timer, Event.Oneshot} else: {Event.Timer} + let flags: cushort = if oneshot: EV_ONESHOT or EV_ADD else: EV_ADD + + s.setKey(fdi, fdi, events, 0, data) + # EVFILT_TIMER on Open/Net(BSD) has granularity of only milliseconds, + # but MacOS and FreeBSD allow use `0` as `fflags` to use milliseconds + # too + modifyKQueue(s, fdi.uint, EVFILT_TIMER, flags, 0, cint(timeout), nil) + inc(s.count) + result = fdi + +proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + s.setKey(fdi, signal, {Event.Signal}, signal, data) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + # to be compatible with linux semantic we need to "eat" signals + posix.signal(cint(signal), SIG_IGN) + modifyKQueue(s, signal.uint, EVFILT_SIGNAL, EV_ADD, 0, 0, + cast[pointer](fdi)) + inc(s.count) + result = fdi + +proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + var fdi = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).int + if fdi == -1: + raiseOsError(osLastError()) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + + var kflags: cushort = EV_ONESHOT or EV_ADD + setKey(s, fdi, pid, {Event.Process, Event.Oneshot}, pid, data) + modifyKQueue(s, pid.uint, EVFILT_PROC, kflags, NOTE_EXIT, 0, + cast[pointer](fdi)) + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = ev.rfd.int + doAssert(s.fds[fdi].ident == 0) + setKey(s, fdi, fdi, {Event.User}, 0, data) + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + + if pkey.events != {}: + if pkey.events * {Event.Read, Event.Write} != {}: + if Event.Read in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if Event.Write in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Timer in pkey.events: + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_TIMER, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Signal in pkey.events: + var nmask, omask: Sigset + var signal = cint(pkey.param) + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, signal) + unblockSignals(nmask, omask) + posix.signal(signal, SIG_DFL) + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_SIGNAL, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.Process in pkey.events: + discard posix.close(cint(pkey.key.fd)) + modifyKQueue(s, fdi.uint, EVFILT_PROC, EV_DELETE, 0, 0, nil) + dec(s.count) + elif Event.User in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + pkey.ident = 0 + pkey.events = {} + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + +proc flush*[T](s: Selector[T]) = + s.withChangeLock(): + var tv = Timespec() + if kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + nil, 0, addr tv) == -1: + raiseOSError(osLastError()) + s.changesCount = 0 + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var + tv: Timespec + resTable: array[MAX_KQUEUE_RESULT_EVENTS, KEvent] + ptv = addr tv + maxres = MAX_KQUEUE_RESULT_EVENTS + + if timeout != -1: + if timeout >= 1000: + tv.tv_sec = (timeout div 1_000).Time + tv.tv_nsec = (timeout %% 1_000) * 1_000_000 + else: + tv.tv_sec = 0.Time + tv.tv_nsec = timeout * 1_000_000 + else: + ptv = nil + + if maxres > len(results): + maxres = len(results) + + var count = 0 + s.withChangeLock(): + count = kevent(s.kqFD, addr(s.changesTable[0]), cint(s.changesCount), + addr(resTable[0]), cint(maxres), ptv) + s.changesCount = 0 + + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 + var pkey: ptr SelectorKey[T] + while i < count: + let kevent = addr(resTable[i]) + if (kevent.flags and EV_ERROR) == 0: + case kevent.filter: + of EVFILT_READ: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Read} + if Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(kevent.ident.cint, addr data, + sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + # someone already consumed event data + inc(i) + continue + else: + raiseOSError(osLastError()) + pkey.key.events = {Event.User} + of EVFILT_WRITE: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Write} + of EVFILT_TIMER: + pkey = addr(s.fds[kevent.ident.int]) + if Event.Oneshot in pkey.events: + if posix.close(cint(pkey.ident)) == -1: + raiseOSError(osLastError()) + pkey.ident = 0 + pkey.events = {} + dec(s.count) + pkey.key.events = {Event.Timer} + of EVFILT_VNODE: + pkey = addr(s.fds[kevent.ident.int]) + pkey.key.events = {Event.Vnode} + of EVFILT_SIGNAL: + pkey = addr(s.fds[cast[int](kevent.udata)]) + pkey.key.events = {Event.Signal} + of EVFILT_PROC: + pkey = addr(s.fds[cast[int](kevent.udata)]) + if posix.close(cint(pkey.ident)) == -1: + raiseOSError(osLastError()) + pkey.ident = 0 + pkey.events = {} + dec(s.count) + pkey.key.events = {Event.Process} + else: + raise newException(ValueError, "Unsupported kqueue filter in queue") + + if (kevent.flags and EV_EOF) != 0: + pkey.key.events.incl(Event.Error) + + results[k] = pkey.key + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_KQUEUE_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim new file mode 100644 index 000000000..d2a0a1273 --- /dev/null +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -0,0 +1,295 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix poll(). + +import posix, times + +# Maximum number of events that can be returned +const MAX_POLL_RESULT_EVENTS = 64 + +when hasThreadSupport: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: ptr SharedArray[SelectorKey[T]] + pollfds: ptr SharedArray[TPollFd] + count: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: seq[SelectorKey[T]] + pollfds: seq[TPollFd] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint + SelectEvent* = ptr SelectEventImpl + +var RLIMIT_NOFILE {.importc: "RLIMIT_NOFILE", + header: "<sys/resource.h>".}: cint +type + rlimit {.importc: "struct rlimit", + header: "<sys/resource.h>", pure, final.} = object + rlim_cur: int + rlim_max: int +proc getrlimit(resource: cint, rlp: var rlimit): cint + {.importc: "getrlimit",header: "<sys/resource.h>".} + +when hasThreadSupport: + template withPollLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withPollLock(s, body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + var a = rlimit() + if getrlimit(RLIMIT_NOFILE, a) != 0: + raiseOsError(osLastError()) + var maxFD = int(a.rlim_max) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.maxFD = maxFD + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + result.pollfds = allocSharedArray[TPollFd](maxFD) + initLock(result.lock) + else: + result = Selector[T]() + result.maxFD = maxFD + result.fds = newSeq[SelectorKey[T]](maxFD) + result.pollfds = newSeq[TPollFd](maxFD) + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deinitLock(s.lock) + deallocSharedArray(s.fds) + deallocSharedArray(s.pollfds) + deallocShared(cast[pointer](s)) + +template pollAdd[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + s.pollfds[s.pollcnt].fd = cint(sock) + s.pollfds[s.pollcnt].events = pollev + inc(s.count) + inc(s.pollcnt) + +template pollUpdate[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var i = 0 + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + s.pollfds[i].events = pollev + break + inc(i) + + if i == s.pollcnt: + raise newException(ValueError, "Descriptor is not registered in queue") + +template pollRemove[T](s: Selector[T], sock: cint) = + withPollLock(s): + var i = 0 + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + if i == s.pollcnt - 1: + s.pollfds[i].fd = 0 + s.pollfds[i].events = 0 + s.pollfds[i].revents = 0 + else: + while i < (s.pollcnt - 1): + s.pollfds[i].fd = s.pollfds[i + 1].fd + s.pollfds[i].events = s.pollfds[i + 1].events + inc(i) + break + inc(i) + dec(s.pollcnt) + dec(s.count) + +template checkFd(s, f) = + if f >= s.maxFD: + raise newException(ValueError, "Maximum file descriptors exceeded") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + var fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) + s.setKey(fdi, fdi, events, 0, data) + if events != {}: s.pollAdd(fdi.cint, events) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if pkey.events == {}: + s.pollAdd(fd.cint, events) + else: + if events != {}: + s.pollUpdate(fd.cint, events) + else: + s.pollRemove(fd.cint) + pkey.events = events + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + var fdi = int(ev.rfd) + doAssert(s.fds[fdi].ident == 0) + var events = {Event.User} + setKey(s, fdi, fdi, events, 0, data) + events.incl(Event.Read) + s.pollAdd(fdi.cint, events) + +proc flush*[T](s: Selector[T]) = discard + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + pkey.ident = 0 + pkey.events = {} + s.pollRemove(fdi.cint) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != 0) + doAssert(Event.User in pkey.events) + pkey.ident = 0 + pkey.events = {} + s.pollRemove(fdi.cint) + +proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + +proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + +proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rfd)) + discard posix.close(cint(ev.wfd)) + deallocShared(cast[pointer](ev)) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var maxres = MAX_POLL_RESULT_EVENTS + if maxres > len(results): + maxres = len(results) + + s.withPollLock(): + let count = posix.poll(addr(s.pollfds[0]), Tnfds(s.pollcnt), timeout) + if count < 0: + result = 0 + let err = osLastError() + if err.cint == EINTR: + discard + else: + raiseOSError(osLastError()) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 + var rindex = 0 + while (i < s.pollcnt) and (k < count) and (rindex < maxres): + let revents = s.pollfds[i].revents + if revents != 0: + let fd = s.pollfds[i].fd + var skey = addr(s.fds[fd]) + skey.key.events = {} + + if (revents and POLLIN) != 0: + skey.key.events.incl(Event.Read) + if Event.User in skey.events: + var data: uint64 = 0 + if posix.read(fd, addr data, sizeof(int)) != sizeof(int): + let err = osLastError() + if err != OSErrorCode(EAGAIN): + raiseOSError(osLastError()) + else: + # someone already consumed event data + inc(i) + continue + skey.key.events = {Event.User} + if (revents and POLLOUT) != 0: + skey.key.events.incl(Event.Write) + if (revents and POLLERR) != 0 or (revents and POLLHUP) != 0 or + (revents and POLLNVAL) != 0: + skey.key.events.incl(Event.Error) + results[rindex] = skey.key + s.pollfds[i].revents = 0 + inc(rindex) + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](MAX_POLL_RESULT_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if s.fds[fdi].ident != 0: + var value = addr(s.fds[fdi].key.data) + body1 + else: + body2 diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim new file mode 100644 index 000000000..f8099f9a0 --- /dev/null +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -0,0 +1,416 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix and Windows select(). + +import times, nativesockets + +when defined(windows): + import winlean + when defined(gcc): + {.passL: "-lws2_32".} + elif defined(vcc): + {.passL: "ws2_32.lib".} + const platformHeaders = """#include <winsock2.h> + #include <windows.h>""" + const EAGAIN = WSAEWOULDBLOCK +else: + const platformHeaders = """#include <sys/select.h> + #include <sys/time.h> + #include <sys/types.h> + #include <unistd.h>""" +type + Fdset {.importc: "fd_set", header: platformHeaders, pure, final.} = object +var + FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint + +proc IOFD_SET(fd: SocketHandle, fdset: ptr Fdset) + {.cdecl, importc: "FD_SET", header: platformHeaders, inline.} +proc IOFD_CLR(fd: SocketHandle, fdset: ptr Fdset) + {.cdecl, importc: "FD_CLR", header: platformHeaders, inline.} +proc IOFD_ZERO(fdset: ptr Fdset) + {.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.} + +when defined(windows): + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + {.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + timeout: ptr Timeval): cint + {.stdcall, importc: "select", header: platformHeaders.} +else: + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + {.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + timeout: ptr Timeval): cint + {.cdecl, importc: "select", header: platformHeaders.} + +when hasThreadSupport: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: ptr SharedArray[SelectorKey[T]] + count: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rsock: SocketHandle + wsock: SocketHandle + SelectEvent* = ptr SelectEventImpl + +when hasThreadSupport: + template withSelectLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withSelectLock[T](s: Selector[T], body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.fds = allocSharedArray[SelectorKey[T]](FD_SETSIZE) + initLock result.lock + else: + result = Selector[T]() + result.fds = newSeq[SelectorKey[T]](FD_SETSIZE) + + IOFD_ZERO(addr result.rSet) + IOFD_ZERO(addr result.wSet) + IOFD_ZERO(addr result.eSet) + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +when defined(windows): + proc newSelectEvent*(): SelectEvent = + var ssock = newNativeSocket() + var wsock = newNativeSocket() + var rsock: SocketHandle = INVALID_SOCKET + var saddr = Sockaddr_in() + + saddr.sin_family = winlean.AF_INET + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(ssock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) < 0'i32: + raiseOSError(osLastError()) + + if winlean.listen(ssock, 1) == -1: + raiseOSError(osLastError()) + + var namelen = sizeof(saddr).SockLen + if getsockname(ssock, cast[ptr SockAddr](addr(saddr)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + + saddr.sin_addr.s_addr = 0x0100007F + if winlean.connect(wsock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) == -1: + raiseOSError(osLastError()) + namelen = sizeof(saddr).SockLen + rsock = winlean.accept(ssock, cast[ptr SockAddr](addr(saddr)), + cast[ptr SockLen](addr(namelen))) + if rsock == SocketHandle(-1): + raiseOSError(osLastError()) + + if winlean.closesocket(ssock) == -1: + raiseOSError(osLastError()) + + var mode = clong(1) + if ioctlsocket(rsock, FIONBIO, addr(mode)) == -1: + raiseOSError(osLastError()) + mode = clong(1) + if ioctlsocket(wsock, FIONBIO, addr(mode)) == -1: + raiseOSError(osLastError()) + + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = rsock + result.wsock = wsock + + proc setEvent*(ev: SelectEvent) = + var data: int = 1 + if winlean.send(ev.wsock, cast[pointer](addr data), + cint(sizeof(int)), 0) != sizeof(int): + raiseOSError(osLastError()) + + proc close*(ev: SelectEvent) = + discard winlean.closesocket(ev.rsock) + discard winlean.closesocket(ev.wsock) + deallocShared(cast[pointer](ev)) + +else: + proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) == -1: + raiseOSError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = SocketHandle(fds[0]) + result.wsock = SocketHandle(fds[1]) + + proc setEvent*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(cint(ev.wsock), addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + + proc close*(ev: SelectEvent) = + discard posix.close(cint(ev.rsock)) + discard posix.close(cint(ev.wsock)) + deallocShared(cast[pointer](ev)) + +proc setKey[T](s: Selector[T], fd: SocketHandle, events: set[Event], data: T) = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == 0: + var pkey = addr(s.fds[i]) + pkey.ident = fdi + pkey.events = events + pkey.key.fd = fd.int + pkey.key.events = {} + pkey.key.data = data + break + inc(i) + if i == FD_SETSIZE: + raise newException(ValueError, "Maximum numbers of fds exceeded") + +proc getKey[T](s: Selector[T], fd: SocketHandle): ptr SelectorKey[T] = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + result = addr(s.fds[i]) + break + inc(i) + doAssert(i < FD_SETSIZE, "Descriptor not registered in queue") + +proc delKey[T](s: Selector[T], fd: SocketHandle) = + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fd.int: + s.fds[i].ident = 0 + s.fds[i].events = {} + break + inc(i) + doAssert(i < FD_SETSIZE, "Descriptor not registered in queue") + +proc registerHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event], data: T) = + when not defined(windows): + let fdi = int(fd) + s.withSelectLock(): + s.setKey(fd, events, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + if Event.Read in events: + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if Event.Write in events: + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + when not defined(windows): + let fdi = int(ev.rsock) + s.withSelectLock(): + s.setKey(ev.rsock, {Event.User}, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + IOFD_SET(ev.rsock, addr s.rSet) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + s.withSelectLock(): + var pkey = s.getKey(fd) + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: SocketHandle) = + s.withSelectLock(): + var pkey = s.getKey(fd) + if Event.Read in pkey.events: + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if Event.Write in pkey.events: + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + s.delKey(fd) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fd = ev.rsock + s.withSelectLock(): + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + s.delKey(fd) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey[T]]): int = + var tv = Timeval() + var ptv = addr tv + var rset, wset, eset: FdSet + + if timeout != -1: + tv.tv_sec = timeout.int32 div 1_000 + tv.tv_usec = (timeout.int32 %% 1_000) * 1_000 + else: + ptv = nil + + s.withSelectLock(): + rset = s.rSet + wset = s.wSet + eset = s.eSet + + var count = ioselect(cint(s.maxFD) + 1, addr(rset), addr(wset), + addr(eset), ptv) + if count < 0: + result = 0 + when defined(windows): + raiseOSError(osLastError()) + else: + let err = osLastError() + if cint(err) != EINTR: + raiseOSError(err) + elif count == 0: + result = 0 + else: + var rindex = 0 + var i = 0 + var k = 0 + + while (i < FD_SETSIZE) and (k < count): + if s.fds[i].ident != 0: + var flag = false + var pkey = addr(s.fds[i]) + pkey.key.events = {} + let fd = SocketHandle(pkey.ident) + if IOFD_ISSET(fd, addr rset) != 0: + if Event.User in pkey.events: + var data: uint64 = 0 + if recv(fd, cast[pointer](addr(data)), + sizeof(uint64).cint, 0) != sizeof(uint64): + let err = osLastError() + if cint(err) != EAGAIN: + raiseOSError(err) + else: + inc(i) + inc(k) + continue + else: + flag = true + pkey.key.events = {Event.User} + else: + flag = true + pkey.key.events = {Event.Read} + if IOFD_ISSET(fd, addr wset) != 0: + pkey.key.events.incl(Event.Write) + if IOFD_ISSET(fd, addr eset) != 0: + pkey.key.events.incl(Event.Error) + flag = true + if flag: + results[rindex] = pkey.key + inc(rindex) + inc(k) + inc(i) + result = rindex + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey[T]] = + result = newSeq[ReadyKey[T]](FD_SETSIZE) + var count = selectInto(s, timeout, result) + result.setLen(count) + +proc flush*[T](s: Selector[T]) = discard + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +when hasThreadSupport: + template withSelectLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withSelectLock[T](s: Selector[T], body: untyped) = + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body: untyped) = + mixin withSelectLock + s.withSelectLock(): + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].key.data) + break + inc(i) + if i != FD_SETSIZE: + body + +template withData*[T](s: Selector[T], fd: SocketHandle, value, + body1, body2: untyped) = + mixin withSelectLock + s.withSelectLock(): + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].key.data) + break + inc(i) + if i != FD_SETSIZE: + body1 + else: + body2 diff --git a/lib/pure/json.nim b/lib/pure/json.nim index b0179cd82..19947fbc2 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -20,7 +20,8 @@ ## let ## small_json = """{"test": 1.3, "key2": true}""" ## jobj = parseJson(small_json) -## assert (jobj.kind == JObject) +## assert (jobj.kind == JObject)\ +## jobj["test"] = newJFloat(0.7) # create or update ## echo($jobj["test"].fnum) ## echo($jobj["key2"].bval) ## @@ -49,6 +50,10 @@ ## "age": herAge ## } ## ] +## +## var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]} +## j2["details"] = %* {"age":35, "pi":3.1415} +## echo j2 import hashes, tables, strutils, lexbase, streams, unicode, macros @@ -56,6 +61,11 @@ import export tables.`$` +when defined(nimJsonGet): + {.pragma: deprecatedGet, deprecated.} +else: + {.pragma: deprecatedGet.} + type JsonEventKind* = enum ## enumeration of all events that may occur when parsing jsonError, ## an error occurred during parsing @@ -116,7 +126,7 @@ type TJsonParser: JsonParser, TTokKind: TokKind].} const - errorMessages: array [JsonError, string] = [ + errorMessages: array[JsonError, string] = [ "no error", "invalid token", "string expected", @@ -129,7 +139,7 @@ const "EOF expected", "expression expected" ] - tokToStr: array [TokKind, string] = [ + tokToStr: array[TokKind, string] = [ "invalid token", "EOF", "string literal", @@ -702,17 +712,28 @@ proc `%`*(b: bool): JsonNode = proc `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode = ## Generic constructor for JSON data. Creates a new `JObject JsonNode` - new(result) - result.kind = JObject - result.fields = initTable[string, JsonNode](4) + if keyvals.len == 0: return newJArray() + result = newJObject() for key, val in items(keyVals): result.fields[key] = val -proc `%`*(elements: openArray[JsonNode]): JsonNode = +template `%`*(j: JsonNode): JsonNode = j + +proc `%`*[T](elements: openArray[T]): JsonNode = ## Generic constructor for JSON data. Creates a new `JArray JsonNode` - new(result) - result.kind = JArray - newSeq(result.elems, elements.len) - for i, p in pairs(elements): result.elems[i] = p + result = newJArray() + for elem in elements: result.add(%elem) + +proc `%`*(o: object): JsonNode = + ## Generic constructor for JSON data. Creates a new `JObject JsonNode` + result = newJObject() + for k, v in o.fieldPairs: result[k] = %v + +proc `%`*(o: ref object): JsonNode = + ## Generic constructor for JSON data. Creates a new `JObject JsonNode` + if o.isNil: + result = newJNull() + else: + result = %(o[]) proc toJson(x: NimNode): NimNode {.compiletime.} = case x.kind @@ -731,6 +752,9 @@ proc toJson(x: NimNode): NimNode {.compiletime.} = result = newNimNode(nnkTableConstr) x.expectLen(0) + of nnkNilLit: + result = newCall("newJNull") + else: result = x @@ -799,16 +823,23 @@ proc len*(n: JsonNode): int = of JObject: result = n.fields.len else: discard -proc `[]`*(node: JsonNode, name: string): JsonNode {.inline.} = +proc `[]`*(node: JsonNode, name: string): JsonNode {.inline, deprecatedGet.} = ## Gets a field from a `JObject`, which must not be nil. - ## If the value at `name` does not exist, returns nil + ## If the value at `name` does not exist, raises KeyError. + ## + ## **Note:** The behaviour of this procedure changed in version 0.14.0. To + ## get a list of usages and to restore the old behaviour of this procedure, + ## compile with the ``-d:nimJsonGet`` flag. assert(not isNil(node)) assert(node.kind == JObject) - result = node.fields.getOrDefault(name) + when defined(nimJsonGet): + if not node.fields.hasKey(name): return nil + result = node.fields[name] proc `[]`*(node: JsonNode, index: int): JsonNode {.inline.} = ## Gets the node at `index` in an Array. Result is undefined if `index` - ## is out of bounds + ## is out of bounds, but as long as array bound checks are enabled it will + ## result in an exception. assert(not isNil(node)) assert(node.kind == JArray) return node.elems[index] @@ -818,6 +849,16 @@ proc hasKey*(node: JsonNode, key: string): bool = assert(node.kind == JObject) result = node.fields.hasKey(key) +proc contains*(node: JsonNode, key: string): bool = + ## Checks if `key` exists in `node`. + assert(node.kind == JObject) + node.fields.hasKey(key) + +proc contains*(node: JsonNode, val: JsonNode): bool = + ## Checks if `val` exists in array `node`. + assert(node.kind == JArray) + find(node.elems, val) >= 0 + proc existsKey*(node: JsonNode, key: string): bool {.deprecated.} = node.hasKey(key) ## Deprecated for `hasKey` @@ -838,20 +879,28 @@ proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = ## Traverses the node and gets the given value. If any of the - ## keys do not exist, returns nil. Also returns nil if one of the - ## intermediate data structures is not an object + ## keys do not exist, returns ``nil``. Also returns ``nil`` if one of the + ## intermediate data structures is not an object. result = node for key in keys: - if isNil(result) or result.kind!=JObject: + if isNil(result) or result.kind != JObject: return nil - result=result[key] + result = result.fields.getOrDefault(key) + +proc getOrDefault*(node: JsonNode, key: string): JsonNode = + ## Gets a field from a `node`. If `node` is nil or not an object or + ## value at `key` does not exist, returns nil + if not isNil(node) and node.kind == JObject: + result = node.fields.getOrDefault(key) + +template simpleGetOrDefault*{`{}`(node, [key])}(node: JsonNode, key: string): JsonNode = node.getOrDefault(key) proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) = ## Traverses the node and tries to set the value at the given location - ## to `value` If any of the keys are missing, they are added + ## to ``value``. If any of the keys are missing, they are added. var node = node for i in 0..(keys.len-2): - if isNil(node[keys[i]]): + if not node.hasKey(keys[i]): node[keys[i]] = newJObject() node = node[keys[i]] node[keys[keys.len-1]] = value @@ -972,7 +1021,7 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, result.add("null") proc pretty*(node: JsonNode, indent = 2): string = - ## Converts `node` to its JSON Representation, with indentation and + ## Returns a JSON Representation of `node`, with indentation and ## on multiple lines. result = "" toPretty(result, node, indent) @@ -982,7 +1031,9 @@ proc toUgly*(result: var string, node: JsonNode) = ## regard for human readability. Meant to improve ``$`` string ## conversion performance. ## - ## This provides higher efficiency than the ``toPretty`` procedure as it + ## JSON representation is stored in the passed `result` + ## + ## This provides higher efficiency than the ``pretty`` procedure as it ## does **not** attempt to format the resulting JSON to make it human readable. var comma = false case node.kind: @@ -1076,7 +1127,7 @@ proc parseJson(p: var JsonParser): JsonNode = discard getTok(p) while p.tok != tkCurlyRi: if p.tok != tkString: - raiseParseErr(p, "string literal as key expected") + raiseParseErr(p, "string literal as key") var key = p.a discard getTok(p) eat(p, tkColon) @@ -1217,16 +1268,6 @@ when false: # To get that we shall use, obj["json"] when isMainModule: - when not defined(js): - var parsed = parseFile("tests/testdata/jsontest.json") - - try: - discard parsed["key2"][12123] - doAssert(false) - except IndexError: doAssert(true) - - var parsed2 = parseFile("tests/testdata/jsontest2.json") - doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd", "c": "\ud83c\udf83", "d": "\u00E6"}""" # nil passthrough @@ -1314,3 +1355,35 @@ when isMainModule: var j4 = %*{"test": nil} doAssert j4 == %{"test": newJNull()} + + let seqOfNodes = @[%1, %2] + let jSeqOfNodes = %seqOfNodes + doAssert(jSeqOfNodes[1].num == 2) + + type MyObj = object + a, b: int + s: string + f32: float32 + f64: float64 + next: ref MyObj + var m: MyObj + m.s = "hi" + m.a = 5 + let jMyObj = %m + doAssert(jMyObj["a"].num == 5) + doAssert(jMyObj["s"].str == "hi") + + # Test loading of file. + when not defined(js): + echo("99% of tests finished. Going to try loading file.") + var parsed = parseFile("tests/testdata/jsontest.json") + + try: + discard parsed["key2"][12123] + doAssert(false) + except IndexError: doAssert(true) + + var parsed2 = parseFile("tests/testdata/jsontest2.json") + doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") + + echo("Tests succeeded!") diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index f602ce31d..6a27e6af8 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -54,14 +54,15 @@ type lvlAll, ## all levels active lvlDebug, ## debug level (and any above) active lvlInfo, ## info level (and any above) active + lvlNotice, ## info notice (and any above) active lvlWarn, ## warn level (and any above) active lvlError, ## error level (and any above) active lvlFatal, ## fatal level (and any above) active lvlNone ## no levels active const - LevelNames*: array [Level, string] = [ - "DEBUG", "DEBUG", "INFO", "WARN", "ERROR", "FATAL", "NONE" + LevelNames*: array[Level, string] = [ + "DEBUG", "DEBUG", "INFO", "NOTICE", "WARN", "ERROR", "FATAL", "NONE" ] defaultFmtStr* = "$levelname " ## default format string @@ -258,22 +259,47 @@ template log*(level: Level, args: varargs[string, `$`]) = template debug*(args: varargs[string, `$`]) = ## Logs a debug message to all registered handlers. + ## + ## Messages that are useful to the application developer only and are usually + ## turned off in release. log(lvlDebug, args) template info*(args: varargs[string, `$`]) = ## Logs an info message to all registered handlers. + ## + ## Messages that are generated during the normal operation of an application + ## and are of no particular importance. Useful to aggregate for potential + ## later analysis. log(lvlInfo, args) +template notice*(args: varargs[string, `$`]) = + ## Logs an notice message to all registered handlers. + ## + ## Semantically very similar to `info`, but meant to be messages you want to + ## be actively notified about (depending on your application). + ## These could be, for example, grouped by hour and mailed out. + log(lvlNotice, args) + template warn*(args: varargs[string, `$`]) = ## Logs a warning message to all registered handlers. + ## + ## A non-error message that may indicate a potential problem rising or + ## impacted performance. log(lvlWarn, args) template error*(args: varargs[string, `$`]) = ## Logs an error message to all registered handlers. + ## + ## A application-level error condition. For example, some user input generated + ## an exception. The application will continue to run, but functionality or + ## data was impacted, possibly visible to users. log(lvlError, args) template fatal*(args: varargs[string, `$`]) = ## Logs a fatal error message to all registered handlers. + ## + ## A application-level fatal condition. FATAL usually means that the application + ## cannot go on and will exit (but this logging event will not do that for you). log(lvlFatal, args) proc addHandler*(handler: Logger) = diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim index 5c28f65a0..7022c21d9 100644 --- a/lib/pure/matchers.nim +++ b/lib/pure/matchers.nim @@ -8,6 +8,10 @@ # ## This module contains various string matchers for email addresses, etc. +## +## **Warning:** This module is deprecated since version 0.14.0. +{.deprecated.} + {.deadCodeElim: on.} {.push debugger:off .} # the user does not want to trace a part diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 84c8d3b11..4ef169b4f 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -39,8 +39,6 @@ proc fac*(n: int): int {.noSideEffect.} = when defined(Posix) and not defined(haiku): {.passl: "-lm".} -when not defined(js) and not defined(nimscript): - import times const PI* = 3.1415926535897932384626433 ## the circle constant PI (Ludolph's number) @@ -119,192 +117,247 @@ proc sum*[T](x: openArray[T]): T {.noSideEffect.} = ## If `x` is empty, 0 is returned. for i in items(x): result = result + i -proc random*(max: int): int {.benign.} - ## Returns a random number in the range 0..max-1. The sequence of - ## random number is always the same, unless `randomize` is called - ## which initializes the random number generator with a "random" - ## number, i.e. a tickcount. - -proc random*(max: float): float {.benign.} - ## Returns a random number in the range 0..<max. The sequence of - ## random number is always the same, unless `randomize` is called - ## which initializes the random number generator with a "random" - ## number, i.e. a tickcount. This has a 16-bit resolution on windows - ## and a 48-bit resolution on other platforms. - -when not defined(nimscript): - proc randomize*() {.benign.} - ## Initializes the random number generator with a "random" - ## number, i.e. a tickcount. Note: Does nothing for the JavaScript target, - ## as JavaScript does not support this. Nor does it work for NimScript. - -proc randomize*(seed: int) {.benign.} - ## Initializes the random number generator with a specific seed. - ## Note: Does nothing for the JavaScript target, - ## as JavaScript does not support this. - {.push noSideEffect.} when not defined(JS): - proc sqrt*(x: float): float {.importc: "sqrt", header: "<math.h>".} + proc sqrt*(x: float32): float32 {.importc: "sqrtf", header: "<math.h>".} + proc sqrt*(x: float64): float64 {.importc: "sqrt", header: "<math.h>".} ## Computes the square root of `x`. - proc cbrt*(x: float): float {.importc: "cbrt", header: "<math.h>".} + proc cbrt*(x: float32): float32 {.importc: "cbrtf", header: "<math.h>".} + proc cbrt*(x: float64): float64 {.importc: "cbrt", header: "<math.h>".} ## Computes the cubic root of `x` - proc ln*(x: float): float {.importc: "log", header: "<math.h>".} + proc ln*(x: float32): float32 {.importc: "logf", header: "<math.h>".} + proc ln*(x: float64): float64 {.importc: "log", header: "<math.h>".} ## Computes the natural log of `x` - proc log10*(x: float): float {.importc: "log10", header: "<math.h>".} + proc log10*(x: float32): float32 {.importc: "log10f", header: "<math.h>".} + proc log10*(x: float64): float64 {.importc: "log10", header: "<math.h>".} ## Computes the common logarithm (base 10) of `x` - proc log2*(x: float): float = return ln(x) / ln(2.0) + proc log2*[T: float32|float64](x: T): T = return ln(x) / ln(2.0) ## Computes the binary logarithm (base 2) of `x` - proc exp*(x: float): float {.importc: "exp", header: "<math.h>".} + proc exp*(x: float32): float32 {.importc: "expf", header: "<math.h>".} + proc exp*(x: float64): float64 {.importc: "exp", header: "<math.h>".} ## Computes the exponential function of `x` (pow(E, x)) - proc frexp*(x: float, exponent: var int): float {. - importc: "frexp", header: "<math.h>".} - ## Split a number into mantissa and exponent. - ## `frexp` calculates the mantissa m (a float greater than or equal to 0.5 - ## and less than 1) and the integer value n such that `x` (the original - ## float value) equals m * 2**n. frexp stores n in `exponent` and returns - ## m. - - proc round*(x: float): int {.importc: "lrint", header: "<math.h>".} - ## Converts a float to an int by rounding. - - proc arccos*(x: float): float {.importc: "acos", header: "<math.h>".} + proc arccos*(x: float32): float32 {.importc: "acosf", header: "<math.h>".} + proc arccos*(x: float64): float64 {.importc: "acos", header: "<math.h>".} ## Computes the arc cosine of `x` - proc arcsin*(x: float): float {.importc: "asin", header: "<math.h>".} + proc arcsin*(x: float32): float32 {.importc: "asinf", header: "<math.h>".} + proc arcsin*(x: float64): float64 {.importc: "asin", header: "<math.h>".} ## Computes the arc sine of `x` - proc arctan*(x: float): float {.importc: "atan", header: "<math.h>".} + proc arctan*(x: float32): float32 {.importc: "atanf", header: "<math.h>".} + proc arctan*(x: float64): float64 {.importc: "atan", header: "<math.h>".} ## Calculate the arc tangent of `y` / `x` - proc arctan2*(y, x: float): float {.importc: "atan2", header: "<math.h>".} + proc arctan2*(y, x: float32): float32 {.importc: "atan2f", header: "<math.h>".} + proc arctan2*(y, x: float64): float64 {.importc: "atan2", header: "<math.h>".} ## Calculate the arc tangent of `y` / `x`. ## `atan2` returns the arc tangent of `y` / `x`; it produces correct ## results even when the resulting angle is near pi/2 or -pi/2 ## (`x` near 0). - proc cos*(x: float): float {.importc: "cos", header: "<math.h>".} + proc cos*(x: float32): float32 {.importc: "cosf", header: "<math.h>".} + proc cos*(x: float64): float64 {.importc: "cos", header: "<math.h>".} ## Computes the cosine of `x` - proc cosh*(x: float): float {.importc: "cosh", header: "<math.h>".} + + proc cosh*(x: float32): float32 {.importc: "coshf", header: "<math.h>".} + proc cosh*(x: float64): float64 {.importc: "cosh", header: "<math.h>".} ## Computes the hyperbolic cosine of `x` - proc hypot*(x, y: float): float {.importc: "hypot", header: "<math.h>".} + + proc hypot*(x, y: float32): float32 {.importc: "hypotf", header: "<math.h>".} + proc hypot*(x, y: float64): float64 {.importc: "hypot", header: "<math.h>".} ## Computes the hypotenuse of a right-angle triangle with `x` and ## `y` as its base and height. Equivalent to ``sqrt(x*x + y*y)``. - proc sinh*(x: float): float {.importc: "sinh", header: "<math.h>".} + proc sinh*(x: float32): float32 {.importc: "sinhf", header: "<math.h>".} + proc sinh*(x: float64): float64 {.importc: "sinh", header: "<math.h>".} ## Computes the hyperbolic sine of `x` - proc sin*(x: float): float {.importc: "sin", header: "<math.h>".} + proc sin*(x: float32): float32 {.importc: "sinf", header: "<math.h>".} + proc sin*(x: float64): float64 {.importc: "sin", header: "<math.h>".} ## Computes the sine of `x` - proc tan*(x: float): float {.importc: "tan", header: "<math.h>".} + + proc tan*(x: float32): float32 {.importc: "tanf", header: "<math.h>".} + proc tan*(x: float64): float64 {.importc: "tan", header: "<math.h>".} ## Computes the tangent of `x` - proc tanh*(x: float): float {.importc: "tanh", header: "<math.h>".} + proc tanh*(x: float32): float32 {.importc: "tanhf", header: "<math.h>".} + proc tanh*(x: float64): float64 {.importc: "tanh", header: "<math.h>".} ## Computes the hyperbolic tangent of `x` - proc pow*(x, y: float): float {.importc: "pow", header: "<math.h>".} - ## Computes `x` to power of `y`. - proc erf*(x: float): float {.importc: "erf", header: "<math.h>".} + proc pow*(x, y: float32): float32 {.importc: "powf", header: "<math.h>".} + proc pow*(x, y: float64): float64 {.importc: "pow", header: "<math.h>".} + ## computes x to power raised of y. + + proc erf*(x: float32): float32 {.importc: "erff", header: "<math.h>".} + proc erf*(x: float64): float64 {.importc: "erf", header: "<math.h>".} ## The error function - proc erfc*(x: float): float {.importc: "erfc", header: "<math.h>".} + proc erfc*(x: float32): float32 {.importc: "erfcf", header: "<math.h>".} + proc erfc*(x: float64): float64 {.importc: "erfc", header: "<math.h>".} ## The complementary error function - proc lgamma*(x: float): float {.importc: "lgamma", header: "<math.h>".} + proc lgamma*(x: float32): float32 {.importc: "lgammaf", header: "<math.h>".} + proc lgamma*(x: float64): float64 {.importc: "lgamma", header: "<math.h>".} ## Natural log of the gamma function - proc tgamma*(x: float): float {.importc: "tgamma", header: "<math.h>".} + proc tgamma*(x: float32): float32 {.importc: "tgammaf", header: "<math.h>".} + proc tgamma*(x: float64): float64 {.importc: "tgamma", header: "<math.h>".} ## The gamma function - # C procs: - when defined(vcc) and false: - # The "secure" random, available from Windows XP - # https://msdn.microsoft.com/en-us/library/sxtz2fa8.aspx - # Present in some variants of MinGW but not enough to justify - # `when defined(windows)` yet - proc rand_s(val: var cuint) {.importc: "rand_s", header: "<stdlib.h>".} - # To behave like the normal version - proc rand(): cuint = rand_s(result) - else: - proc srand(seed: cint) {.importc: "srand", header: "<stdlib.h>".} - proc rand(): cint {.importc: "rand", header: "<stdlib.h>".} - - when not defined(windows): - proc srand48(seed: clong) {.importc: "srand48", header: "<stdlib.h>".} - proc drand48(): float {.importc: "drand48", header: "<stdlib.h>".} - proc random(max: float): float = - result = drand48() * max - else: - when defined(vcc): # Windows with Visual C - proc random(max: float): float = - # we are hardcoding this because - # importc-ing macros is extremely problematic - # and because the value is publicly documented - # on MSDN and very unlikely to change - # See https://msdn.microsoft.com/en-us/library/296az74e.aspx - const rand_max = 4294967295 # UINT_MAX - result = (float(rand()) / float(rand_max)) * max - proc randomize() = discard - proc randomize(seed: int) = discard - else: # Windows with another compiler - proc random(max: float): float = - # we are hardcoding this because - # importc-ing macros is extremely problematic - # and because the value is publicly documented - # on MSDN and very unlikely to change - const rand_max = 32767 - result = (float(rand()) / float(rand_max)) * max - - when not defined(vcc): # the above code for vcc uses `discard` instead - # this is either not Windows or is Windows without vcc - when not defined(nimscript): - proc randomize() = - randomize(cast[int](epochTime())) - proc randomize(seed: int) = - srand(cint(seed)) # rand_s doesn't use srand - when declared(srand48): srand48(seed) - - proc random(max: int): int = - result = int(rand()) mod max - - proc trunc*(x: float): float {.importc: "trunc", header: "<math.h>".} - ## Truncates `x` to the decimal point - ## - ## .. code-block:: nim - ## echo trunc(PI) # 3.0 - proc floor*(x: float): float {.importc: "floor", header: "<math.h>".} + proc floor*(x: float32): float32 {.importc: "floorf", header: "<math.h>".} + proc floor*(x: float64): float64 {.importc: "floor", header: "<math.h>".} ## Computes the floor function (i.e., the largest integer not greater than `x`) ## ## .. code-block:: nim ## echo floor(-3.5) ## -4.0 - proc ceil*(x: float): float {.importc: "ceil", header: "<math.h>".} + + proc ceil*(x: float32): float32 {.importc: "ceilf", header: "<math.h>".} + proc ceil*(x: float64): float64 {.importc: "ceil", header: "<math.h>".} ## Computes the ceiling function (i.e., the smallest integer not less than `x`) ## ## .. code-block:: nim ## echo ceil(-2.1) ## -2.0 - proc fmod*(x, y: float): float {.importc: "fmod", header: "<math.h>".} + when defined(windows) and defined(vcc): + # MSVC 2010 don't have trunc/truncf + # this implementation was inspired by Go-lang Math.Trunc + proc truncImpl(f: float64): float64 = + const + mask : uint64 = 0x7FF + shift: uint64 = 64 - 12 + bias : uint64 = 0x3FF + + if f < 1: + if f < 0: return -truncImpl(-f) + elif f == 0: return f # Return -0 when f == -0 + else: return 0 + + var x = cast[uint64](f) + let e = (x shr shift) and mask - bias + + # Keep the top 12+e bits, the integer part; clear the rest. + if e < 64-12: + x = x and (not (1'u64 shl (64'u64-12'u64-e) - 1'u64)) + + result = cast[float64](x) + + proc truncImpl(f: float32): float32 = + const + mask : uint32 = 0xFF + shift: uint32 = 32 - 9 + bias : uint32 = 0x7F + + if f < 1: + if f < 0: return -truncImpl(-f) + elif f == 0: return f # Return -0 when f == -0 + else: return 0 + + var x = cast[uint32](f) + let e = (x shr shift) and mask - bias + + # Keep the top 9+e bits, the integer part; clear the rest. + if e < 32-9: + x = x and (not (1'u32 shl (32'u32-9'u32-e) - 1'u32)) + + result = cast[float32](x) + + proc trunc*(x: float64): float64 = + if classify(x) in {fcZero, fcNegZero, fcNan, fcInf, fcNegInf}: return x + result = truncImpl(x) + + proc trunc*(x: float32): float32 = + if classify(x) in {fcZero, fcNegZero, fcNan, fcInf, fcNegInf}: return x + result = truncImpl(x) + + proc round0[T: float32|float64](x: T): T = + ## Windows compilers prior to MSVC 2012 do not implement 'round', + ## 'roundl' or 'roundf'. + result = if x < 0.0: ceil(x - T(0.5)) else: floor(x + T(0.5)) + else: + proc round0(x: float32): float32 {.importc: "roundf", header: "<math.h>".} + proc round0(x: float64): float64 {.importc: "round", header: "<math.h>".} + ## Rounds a float to zero decimal places. Used internally by the round + ## function when the specified number of places is 0. + + proc trunc*(x: float32): float32 {.importc: "truncf", header: "<math.h>".} + proc trunc*(x: float64): float64 {.importc: "trunc", header: "<math.h>".} + ## Truncates `x` to the decimal point + ## + ## .. code-block:: nim + ## echo trunc(PI) # 3.0 + + proc fmod*(x, y: float32): float32 {.importc: "fmodf", header: "<math.h>".} + proc fmod*(x, y: float64): float64 {.importc: "fmod", header: "<math.h>".} ## Computes the remainder of `x` divided by `y` ## ## .. code-block:: nim ## echo fmod(-2.5, 0.3) ## -0.1 else: - proc mathrandom(): float {.importc: "Math.random", nodecl.} - proc floor*(x: float): float {.importc: "Math.floor", nodecl.} - proc ceil*(x: float): float {.importc: "Math.ceil", nodecl.} - proc random(max: int): int = - result = int(floor(mathrandom() * float(max))) - proc random(max: float): float = - result = float(mathrandom() * float(max)) - proc randomize() = discard - proc randomize(seed: int) = discard - - proc sqrt*(x: float): float {.importc: "Math.sqrt", nodecl.} - proc ln*(x: float): float {.importc: "Math.log", nodecl.} - proc log10*(x: float): float = return ln(x) / ln(10.0) - proc log2*(x: float): float = return ln(x) / ln(2.0) - - proc exp*(x: float): float {.importc: "Math.exp", nodecl.} - proc round*(x: float): int {.importc: "Math.round", nodecl.} - proc pow*(x, y: float): float {.importc: "Math.pow", nodecl.} - - proc frexp*(x: float, exponent: var int): float = + proc floor*(x: float32): float32 {.importc: "Math.floor", nodecl.} + proc floor*(x: float64): float64 {.importc: "Math.floor", nodecl.} + proc ceil*(x: float32): float32 {.importc: "Math.ceil", nodecl.} + proc ceil*(x: float64): float64 {.importc: "Math.ceil", nodecl.} + + proc sqrt*(x: float32): float32 {.importc: "Math.sqrt", nodecl.} + proc sqrt*(x: float64): float64 {.importc: "Math.sqrt", nodecl.} + proc ln*(x: float32): float32 {.importc: "Math.log", nodecl.} + proc ln*(x: float64): float64 {.importc: "Math.log", nodecl.} + proc log10*[T: float32|float64](x: T): T = return ln(x) / ln(10.0) + proc log2*[T: float32|float64](x: T): T = return ln(x) / ln(2.0) + + proc exp*(x: float32): float32 {.importc: "Math.exp", nodecl.} + proc exp*(x: float64): float64 {.importc: "Math.exp", nodecl.} + proc round0(x: float): float {.importc: "Math.round", nodecl.} + + proc pow*(x, y: float32): float32 {.importC: "Math.pow", nodecl.} + proc pow*(x, y: float64): float64 {.importc: "Math.pow", nodecl.} + + proc arccos*(x: float32): float32 {.importc: "Math.acos", nodecl.} + proc arccos*(x: float64): float64 {.importc: "Math.acos", nodecl.} + proc arcsin*(x: float32): float32 {.importc: "Math.asin", nodecl.} + proc arcsin*(x: float64): float64 {.importc: "Math.asin", nodecl.} + proc arctan*(x: float32): float32 {.importc: "Math.atan", nodecl.} + proc arctan*(x: float64): float64 {.importc: "Math.atan", nodecl.} + proc arctan2*(y, x: float32): float32 {.importC: "Math.atan2", nodecl.} + proc arctan2*(y, x: float64): float64 {.importc: "Math.atan2", nodecl.} + + proc cos*(x: float32): float32 {.importc: "Math.cos", nodecl.} + proc cos*(x: float64): float64 {.importc: "Math.cos", nodecl.} + proc cosh*(x: float32): float32 = return (exp(x)+exp(-x))*0.5 + proc cosh*(x: float64): float64 = return (exp(x)+exp(-x))*0.5 + proc hypot*[T: float32|float64](x, y: T): T = return sqrt(x*x + y*y) + proc sinh*[T: float32|float64](x: T): T = return (exp(x)-exp(-x))*0.5 + proc sin*(x: float32): float32 {.importc: "Math.sin", nodecl.} + proc sin*(x: float64): float64 {.importc: "Math.sin", nodecl.} + proc tan*(x: float32): float32 {.importc: "Math.tan", nodecl.} + proc tan*(x: float64): float64 {.importc: "Math.tan", nodecl.} + proc tanh*[T: float32|float64](x: T): T = + var y = exp(2.0*x) + return (y-1.0)/(y+1.0) + +proc round*[T: float32|float64](x: T, places: int = 0): T = + ## Round a floating point number. + ## + ## If `places` is 0 (or omitted), round to the nearest integral value + ## following normal mathematical rounding rules (e.g. `round(54.5) -> 55.0`). + ## If `places` is greater than 0, round to the given number of decimal + ## places, e.g. `round(54.346, 2) -> 54.35`. + ## If `places` is negative, round to the left of the decimal place, e.g. + ## `round(537.345, -1) -> 540.0` + if places == 0: + result = round0(x) + else: + var mult = pow(10.0, places.T) + result = round0(x*mult)/mult + +when not defined(JS): + proc frexp*(x: float32, exponent: var int): float32 {. + importc: "frexp", header: "<math.h>".} + proc frexp*(x: float64, exponent: var int): float64 {. + importc: "frexp", header: "<math.h>".} + ## Split a number into mantissa and exponent. + ## `frexp` calculates the mantissa m (a float greater than or equal to 0.5 + ## and less than 1) and the integer value n such that `x` (the original + ## float value) equals m * 2**n. frexp stores n in `exponent` and returns + ## m. +else: + proc frexp*[T: float32|float64](x: T, exponent: var int): T = if x == 0.0: exponent = 0 result = 0.0 @@ -315,20 +368,22 @@ else: exponent = round(ex) result = x / pow(2.0, ex) - proc arccos*(x: float): float {.importc: "Math.acos", nodecl.} - proc arcsin*(x: float): float {.importc: "Math.asin", nodecl.} - proc arctan*(x: float): float {.importc: "Math.atan", nodecl.} - proc arctan2*(y, x: float): float {.importc: "Math.atan2", nodecl.} - - proc cos*(x: float): float {.importc: "Math.cos", nodecl.} - proc cosh*(x: float): float = return (exp(x)+exp(-x))*0.5 - proc hypot*(x, y: float): float = return sqrt(x*x + y*y) - proc sinh*(x: float): float = return (exp(x)-exp(-x))*0.5 - proc sin*(x: float): float {.importc: "Math.sin", nodecl.} - proc tan*(x: float): float {.importc: "Math.tan", nodecl.} - proc tanh*(x: float): float = - var y = exp(2.0*x) - return (y-1.0)/(y+1.0) +proc splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] = + ## Breaks `x` into an integral and a fractional part. + ## + ## Returns a tuple containing intpart and floatpart representing + ## the integer part and the fractional part respectively. + ## + ## Both parts have the same sign as `x`. Analogous to the `modf` + ## function in C. + var + absolute: T + absolute = abs(x) + result.intpart = floor(absolute) + result.floatpart = absolute - result.intpart + if x < 0: + result.intpart = -result.intpart + result.floatpart = -result.floatpart {.pop.} @@ -340,7 +395,7 @@ proc radToDeg*[T: float32|float64](d: T): T {.inline.} = ## Convert from radians to degrees result = T(d) / RadPerDeg -proc `mod`*(x, y: float): float = +proc `mod`*[T: float32|float64](x, y: T): T = ## Computes the modulo operation for float operators. Equivalent ## to ``x - y * floor(x/y)``. Note that the remainder will always ## have the same sign as the divisor. @@ -349,21 +404,13 @@ proc `mod`*(x, y: float): float = ## echo (4.0 mod -3.1) # -2.2 result = if y == 0.0: x else: x - y * (x/y).floor -proc random*[T](x: Slice[T]): T = - ## For a slice `a .. b` returns a value in the range `a .. b-1`. - result = random(x.b - x.a) + x.a - -proc random*[T](a: openArray[T]): T = - ## returns a random element from the openarray `a`. - result = a[random(a.low..a.len)] - {.pop.} {.pop.} proc `^`*[T](x, y: T): T = ## Computes ``x`` to the power ``y`. ``x`` must be non-negative, use ## `pow <#pow,float,float>` for negative exponents. - assert y >= 0 + assert y >= T(0) var (x, y) = (x, y) result = 1 @@ -391,24 +438,6 @@ proc lcm*[T](x, y: T): T = x div gcd(x, y) * y when isMainModule and not defined(JS): - proc gettime(dummy: ptr cint): cint {.importc: "time", header: "<time.h>".} - - # Verifies random seed initialization. - let seed = gettime(nil) - randomize(seed) - const SIZE = 10 - var buf : array[0..SIZE, int] - # Fill the buffer with random values - for i in 0..SIZE-1: - buf[i] = random(high(int)) - # Check that the second random calls are the same for each position. - randomize(seed) - for i in 0..SIZE-1: - assert buf[i] == random(high(int)), "non deterministic random seeding" - - when not defined(testing): - echo "random values equal after reseeding" - # Check for no side effect annotation proc mySqrt(num: float): float {.noSideEffect.} = return sqrt(num) @@ -418,3 +447,65 @@ when isMainModule and not defined(JS): assert(lgamma(1.0) == 0.0) # ln(1.0) == 0.0 assert(erf(6.0) > erf(5.0)) assert(erfc(6.0) < erfc(5.0)) +when isMainModule: + # Function for approximate comparison of floats + proc `==~`(x, y: float): bool = (abs(x-y) < 1e-9) + + block: # round() tests + # Round to 0 decimal places + doAssert round(54.652) ==~ 55.0 + doAssert round(54.352) ==~ 54.0 + doAssert round(-54.652) ==~ -55.0 + doAssert round(-54.352) ==~ -54.0 + doAssert round(0.0) ==~ 0.0 + # Round to positive decimal places + doAssert round(-547.652, 1) ==~ -547.7 + doAssert round(547.652, 1) ==~ 547.7 + doAssert round(-547.652, 2) ==~ -547.65 + doAssert round(547.652, 2) ==~ 547.65 + # Round to negative decimal places + doAssert round(547.652, -1) ==~ 550.0 + doAssert round(547.652, -2) ==~ 500.0 + doAssert round(547.652, -3) ==~ 1000.0 + doAssert round(547.652, -4) ==~ 0.0 + doAssert round(-547.652, -1) ==~ -550.0 + doAssert round(-547.652, -2) ==~ -500.0 + doAssert round(-547.652, -3) ==~ -1000.0 + doAssert round(-547.652, -4) ==~ 0.0 + + block: # splitDecimal() tests + doAssert splitDecimal(54.674).intpart ==~ 54.0 + doAssert splitDecimal(54.674).floatpart ==~ 0.674 + doAssert splitDecimal(-693.4356).intpart ==~ -693.0 + doAssert splitDecimal(-693.4356).floatpart ==~ -0.4356 + doAssert splitDecimal(0.0).intpart ==~ 0.0 + doAssert splitDecimal(0.0).floatpart ==~ 0.0 + + block: # trunc tests for vcc + doAssert(trunc(-1.1) == -1) + doAssert(trunc(1.1) == 1) + doAssert(trunc(-0.1) == -0) + doAssert(trunc(0.1) == 0) + + #special case + doAssert(classify(trunc(1e1000000)) == fcInf) + doAssert(classify(trunc(-1e1000000)) == fcNegInf) + doAssert(classify(trunc(0.0/0.0)) == fcNan) + doAssert(classify(trunc(0.0)) == fcZero) + + #trick the compiler to produce signed zero + let + f_neg_one = -1.0 + f_zero = 0.0 + f_nan = f_zero / f_zero + + doAssert(classify(trunc(f_neg_one*f_zero)) == fcNegZero) + + doAssert(trunc(-1.1'f32) == -1) + doAssert(trunc(1.1'f32) == 1) + doAssert(trunc(-0.1'f32) == -0) + doAssert(trunc(0.1'f32) == 0) + doAssert(classify(trunc(1e1000000'f32)) == fcInf) + doAssert(classify(trunc(-1e1000000'f32)) == fcNegInf) + doAssert(classify(trunc(f_nan.float32)) == fcNan) + doAssert(classify(trunc(0.0'f32)) == fcZero) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index b9c574944..aa32778c5 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -42,6 +42,10 @@ type proc mapMem*(m: var MemFile, mode: FileMode = fmRead, mappedSize = -1, offset = 0): pointer = + ## returns a pointer to a mapped portion of MemFile `m` + ## + ## ``mappedSize`` of ``-1`` maps to the whole file, and + ## ``offset`` must be multiples of the PAGE SIZE of your OS var readonly = mode == fmRead when defined(windows): result = mapViewOfFileEx( @@ -68,7 +72,9 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead, proc unmapMem*(f: var MemFile, p: pointer, size: int) = ## unmaps the memory region ``(p, <p+size)`` of the mapped file `f`. ## All changes are written back to the file system, if `f` was opened - ## with write access. ``size`` must be of exactly the size that was requested + ## with write access. + ## + ## ``size`` must be of exactly the size that was requested ## via ``mapMem``. when defined(windows): if unmapViewOfFile(p) == 0: raiseOSError(osLastError()) @@ -79,9 +85,17 @@ proc unmapMem*(f: var MemFile, p: pointer, size: int) = proc open*(filename: string, mode: FileMode = fmRead, mappedSize = -1, offset = 0, newFileSize = -1): MemFile = ## opens a memory mapped file. If this fails, ``EOS`` is raised. - ## `newFileSize` can only be set if the file does not exist and is opened - ## with write access (e.g., with fmReadWrite). `mappedSize` and `offset` - ## can be used to map only a slice of the file. Example: + ## + ## ``newFileSize`` can only be set if the file does not exist and is opened + ## with write access (e.g., with fmReadWrite). + ## + ##``mappedSize`` and ``offset`` + ## can be used to map only a slice of the file. + ## + ## ``offset`` must be multiples of the PAGE SIZE of your OS + ## (usually 4K or 8K but is unique to your OS) + ## + ## Example: ## ## .. code-block:: nim ## var @@ -257,12 +271,10 @@ type MemSlice* = object ## represent slice of a MemFile for iteration over deli data*: pointer size*: int -proc c_memcpy(a, b: pointer, n: int) {.importc: "memcpy", header: "<string.h>".} - proc `$`*(ms: MemSlice): string {.inline.} = ## Return a Nim string built from a MemSlice. var buf = newString(ms.size) - c_memcpy(addr(buf[0]), ms.data, ms.size) + copyMem(addr(buf[0]), ms.data, ms.size) buf[ms.size] = '\0' result = buf @@ -287,7 +299,9 @@ iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} = ## iterate over line-like records in a file. However, returned (data,size) ## objects are not Nim strings, bounds checked Nim arrays, or even terminated ## C strings. So, care is required to access the data (e.g., think C mem* - ## functions, not str* functions). Example: + ## functions, not str* functions). + ## + ## Example: ## ## .. code-block:: nim ## var count = 0 @@ -320,7 +334,9 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T ## Replace contents of passed buffer with each new line, like ## `readLine(File) <system.html#readLine,File,TaintedString>`_. ## `delim`, `eat`, and delimiting logic is exactly as for - ## `memSlices <#memSlices>`_, but Nim strings are returned. Example: + ## `memSlices <#memSlices>`_, but Nim strings are returned. + ## + ## Example: ## ## .. code-block:: nim ## var buffer: TaintedString = "" @@ -329,7 +345,7 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T for ms in memSlices(mfile, delim, eat): buf.setLen(ms.size) - c_memcpy(addr(buf[0]), ms.data, ms.size) + copyMem(addr(buf[0]), ms.data, ms.size) buf[ms.size] = '\0' yield buf @@ -337,7 +353,9 @@ iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} ## Return each line in a file as a Nim string, like ## `lines(File) <system.html#lines.i,File>`_. ## `delim`, `eat`, and delimiting logic is exactly as for - ## `memSlices <#memSlices>`_, but Nim strings are returned. Example: + ## `memSlices <#memSlices>`_, but Nim strings are returned. + ## + ## Example: ## ## .. code-block:: nim ## for line in lines(memfiles.open("foo")): diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index ae0845714..36b597767 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -1,3 +1,12 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + type MersenneTwister* = object mt: array[0..623, uint32] @@ -5,29 +14,31 @@ type {.deprecated: [TMersenneTwister: MersenneTwister].} -proc newMersenneTwister*(seed: int): MersenneTwister = +proc newMersenneTwister*(seed: uint32): MersenneTwister = result.index = 0 - result.mt[0]= uint32(seed) + result.mt[0] = seed for i in 1..623'u32: - result.mt[i]= (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) + result.mt[i] = (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) proc generateNumbers(m: var MersenneTwister) = for i in 0..623: - var y = (m.mt[i] and 0x80000000'u32) + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) + var y = (m.mt[i] and 0x80000000'u32) + + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) m.mt[i] = m.mt[(i+397) mod 624] xor uint32(y shr 1'u32) if (y mod 2'u32) != 0: - m.mt[i] = m.mt[i] xor 0x9908b0df'u32 + m.mt[i] = m.mt[i] xor 0x9908b0df'u32 -proc getNum*(m: var MersenneTwister): int = +proc getNum*(m: var MersenneTwister): uint32 = + ## Returns the next pseudo random number ranging from 0 to high(uint32) if m.index == 0: generateNumbers(m) - var y = m.mt[m.index] - y = y xor (y shr 11'u32) - y = y xor ((7'u32 shl y) and 0x9d2c5680'u32) - y = y xor ((15'u32 shl y) and 0xefc60000'u32) - y = y xor (y shr 18'u32) - m.index = (m.index+1) mod 624 - return int(y) + result = m.mt[m.index] + m.index = (m.index + 1) mod m.mt.len + + result = result xor (result shr 11'u32) + result = result xor ((7'u32 shl result) and 0x9d2c5680'u32) + result = result xor ((15'u32 shl result) and 0xefc60000'u32) + result = result xor (result shr 18'u32) # Test when not defined(testing) and isMainModule: diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 043d6d80a..4526afa49 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -27,7 +27,7 @@ else: import posix export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL, EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET - export Sockaddr_storage + export Sockaddr_storage, Sockaddr_un, Sockaddr_un_path_length export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, Sockaddr_in6, @@ -38,7 +38,7 @@ export SOL_SOCKET, SOMAXCONN, SO_ACCEPTCONN, SO_BROADCAST, SO_DEBUG, SO_DONTROUTE, - SO_KEEPALIVE, SO_OOBINLINE, SO_REUSEADDR, + SO_KEEPALIVE, SO_OOBINLINE, SO_REUSEADDR, SO_REUSEPORT, MSG_PEEK when defined(macosx) and not defined(nimdoc): @@ -326,8 +326,13 @@ proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = cint(AF_INET)) if s == nil: raiseOSError(osLastError()) else: - var s = posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, - cint(posix.AF_INET)) + var s = + when defined(android4): + posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, + cint(posix.AF_INET)) + else: + posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).Socklen, + cint(posix.AF_INET)) if s == nil: raiseOSError(osLastError(), $hstrerror(h_errno)) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 330682ca9..bd208761b 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -8,19 +8,77 @@ # ## This module implements a high-level cross-platform sockets interface. +## The procedures implemented in this module are primarily for blocking sockets. +## For asynchronous non-blocking sockets use the ``asyncnet`` module together +## with the ``asyncdispatch`` module. +## +## The first thing you will always need to do in order to start using sockets, +## is to create a new instance of the ``Socket`` type using the ``newSocket`` +## procedure. +## +## SSL +## ==== +## +## In order to use the SSL procedures defined in this module, you will need to +## compile your application with the ``-d:ssl`` flag. +## +## Examples +## ======== +## +## Connecting to a server +## ---------------------- +## +## After you create a socket with the ``newSocket`` procedure, you can easily +## connect it to a server running at a known hostname (or IP address) and port. +## To do so over TCP, use the example below. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.connect("google.com", Port(80)) +## +## UDP is a connectionless protocol, so UDP sockets don't have to explicitly +## call the ``connect`` procedure. They can simply start sending data +## immediately. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.sendTo("192.168.0.1", Port(27960), "status\n") +## +## Creating a server +## ----------------- +## +## After you create a socket with the ``newSocket`` procedure, you can create a +## TCP server by calling the ``bindAddr`` and ``listen`` procedures. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.bindAddr(Port(1234)) +## socket.listen() +## +## You can then begin accepting connections using the ``accept`` procedure. +## +## .. code-block:: Nim +## var client = newSocket() +## var address = "" +## while true: +## socket.acceptAddr(client, address) +## echo("Client connected from: ", address) +## {.deadCodeElim: on.} -import nativesockets, os, strutils, parseutils, times +import nativesockets, os, strutils, parseutils, times, sets export Port, `$`, `==` +export Domain, SockType, Protocol const useWinVersion = defined(Windows) or defined(nimdoc) +const defineSsl = defined(ssl) or defined(nimdoc) -when defined(ssl): +when defineSsl: import openssl # Note: The enumerations are mapped to Window's constants. -when defined(ssl): +when defineSsl: type SslError* = object of Exception @@ -30,7 +88,10 @@ when defined(ssl): SslProtVersion* = enum protSSLv2, protSSLv3, protTLSv1, protSSLv23 - SslContext* = distinct SslCtx + SslContext* = ref object + context*: SslCtx + extraInternalIndex: int + referencedData: HashSet[int] SslAcceptResult* = enum AcceptNoClient = 0, AcceptNoHandshake, AcceptSuccess @@ -38,6 +99,10 @@ when defined(ssl): SslHandshakeType* = enum handshakeAsClient, handshakeAsServer + SslClientGetPskFunc* = proc(hint: string): tuple[identity: string, psk: string] + + SslServerGetPskFunc* = proc(identity: string): string + {.deprecated: [ESSL: SSLError, TSSLCVerifyMode: SSLCVerifyMode, TSSLProtVersion: SSLProtVersion, PSSLContext: SSLContext, TSSLAcceptResult: SSLAcceptResult].} @@ -54,7 +119,7 @@ type currPos: int # current index in buffer bufLen: int # current length of buffer of false: nil - when defined(ssl): + when defineSsl: case isSsl: bool of true: sslHandle: SSLPtr @@ -72,7 +137,7 @@ type SOBool* = enum ## Boolean socket options. OptAcceptConn, OptBroadcast, OptDebug, OptDontRoute, OptKeepAlive, - OptOOBInline, OptReuseAddr + OptOOBInline, OptReuseAddr, OptReusePort ReadLineResult* = enum ## result for readLineAsync ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone @@ -140,6 +205,10 @@ proc newSocket*(fd: SocketHandle, domain: Domain = AF_INET, if buffered: result.currPos = 0 + # Set SO_NOSIGPIPE on OS X. + when defined(macosx): + setSockOptInt(fd, SOL_SOCKET, SO_NOSIGPIPE, 1) + proc newSocket*(domain, sockType, protocol: cint, buffered = true): Socket = ## Creates a new socket. ## @@ -160,13 +229,18 @@ proc newSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, raiseOSError(osLastError()) result = newSocket(fd, domain, sockType, protocol, buffered) -when defined(ssl): +when defineSsl: CRYPTO_malloc_init() SslLibraryInit() SslLoadErrorStrings() ErrLoadBioStrings() OpenSSL_add_all_algorithms() + type + SslContextExtraInternal = ref object of RootRef + serverGetPskFunc: SslServerGetPskFunc + clientGetPskFunc: SslClientGetPskFunc + proc raiseSSLError*(s = "") = ## Raises a new SSL error. if s != "": @@ -179,6 +253,34 @@ when defined(ssl): var errStr = ErrErrorString(err, nil) raise newException(SSLError, $errStr) + proc getExtraDataIndex*(ctx: SSLContext): int = + ## Retrieves unique index for storing extra data in SSLContext. + result = SSL_CTX_get_ex_new_index(0, nil, nil, nil, nil).int + if result < 0: + raiseSSLError() + + proc getExtraData*(ctx: SSLContext, index: int): RootRef = + ## Retrieves arbitrary data stored inside SSLContext. + if index notin ctx.referencedData: + raise newException(IndexError, "No data with that index.") + let res = ctx.context.SSL_CTX_get_ex_data(index.cint) + if cast[int](res) == 0: + raiseSSLError() + return cast[RootRef](res) + + proc setExtraData*(ctx: SSLContext, index: int, data: RootRef) = + ## Stores arbitrary data inside SSLContext. The unique `index` + ## should be retrieved using getSslContextExtraDataIndex. + if index in ctx.referencedData: + GC_unref(getExtraData(ctx, index)) + + if ctx.context.SSL_CTX_set_ex_data(index.cint, cast[pointer](data)) == -1: + raiseSSLError() + + if index notin ctx.referencedData: + ctx.referencedData.incl(index) + GC_ref(data) + # http://simplestcodings.blogspot.co.uk/2010/08/secure-server-client-using-openssl-in-c.html proc loadCertificates(ctx: SSL_CTX, certFile, keyFile: string) = if certFile != "" and not existsFile(certFile): @@ -201,7 +303,7 @@ when defined(ssl): raiseSSLError("Verification of private key file failed.") proc newContext*(protVersion = protSSLv23, verifyMode = CVerifyPeer, - certFile = "", keyFile = ""): SSLContext = + certFile = "", keyFile = "", cipherList = "ALL"): SSLContext = ## Creates an SSL context. ## ## Protocol version specifies the protocol to use. SSLv2, SSLv3, TLSv1 @@ -222,13 +324,13 @@ when defined(ssl): of protSSLv23: newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. of protSSLv2: - raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv3") + raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") of protSSLv3: - newCTX = SSL_CTX_new(SSLv3_method()) + raiseSslError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") of protTLSv1: newCTX = SSL_CTX_new(TLSv1_method()) - if newCTX.SSLCTXSetCipherList("ALL") != 1: + if newCTX.SSLCTXSetCipherList(cipherList) != 1: raiseSSLError() case verifyMode of CVerifyPeer: @@ -240,7 +342,87 @@ when defined(ssl): discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY) newCTX.loadCertificates(certFile, keyFile) - return SSLContext(newCTX) + + result = SSLContext(context: newCTX, extraInternalIndex: 0, + referencedData: initSet[int]()) + result.extraInternalIndex = getExtraDataIndex(result) + + let extraInternal = new(SslContextExtraInternal) + result.setExtraData(result.extraInternalIndex, extraInternal) + + proc getExtraInternal(ctx: SSLContext): SslContextExtraInternal = + return SslContextExtraInternal(ctx.getExtraData(ctx.extraInternalIndex)) + + proc destroyContext*(ctx: SSLContext) = + ## Free memory referenced by SSLContext. + + # We assume here that OpenSSL's internal indexes increase by 1 each time. + # That means we can assume that the next internal index is the length of + # extra data indexes. + for i in ctx.referencedData: + GC_unref(getExtraData(ctx, i).RootRef) + ctx.context.SSL_CTX_free() + + proc `pskIdentityHint=`*(ctx: SSLContext, hint: string) = + ## Sets the identity hint passed to server. + ## + ## Only used in PSK ciphersuites. + if ctx.context.SSL_CTX_use_psk_identity_hint(hint) <= 0: + raiseSSLError() + + proc clientGetPskFunc*(ctx: SSLContext): SslClientGetPskFunc = + return ctx.getExtraInternal().clientGetPskFunc + + proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar; + max_psk_len: cuint): cuint {.cdecl.} = + let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX, extraInternalIndex: 0) + let hintString = if hint == nil: nil else: $hint + let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString) + if psk.len.cuint > max_psk_len: + return 0 + if identityString.len.cuint >= max_identity_len: + return 0 + + copyMem(identity, identityString.cstring, pskString.len + 1) # with the last zero byte + copyMem(psk, pskString.cstring, pskString.len) + + return pskString.len.cuint + + proc `clientGetPskFunc=`*(ctx: SSLContext, fun: SslClientGetPskFunc) = + ## Sets function that returns the client identity and the PSK based on identity + ## hint from the server. + ## + ## Only used in PSK ciphersuites. + ctx.getExtraInternal().clientGetPskFunc = fun + assert ctx.extraInternalIndex == 0, + "The pskClientCallback assumes the extraInternalIndex is 0" + ctx.context.SSL_CTX_set_psk_client_callback( + if fun == nil: nil else: pskClientCallback) + + proc serverGetPskFunc*(ctx: SSLContext): SslServerGetPskFunc = + return ctx.getExtraInternal().serverGetPskFunc + + proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr cuchar; max_psk_len: cint): cuint {.cdecl.} = + let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX, extraInternalIndex: 0) + let pskString = (ctx.serverGetPskFunc)($identity) + if psk.len.cint > max_psk_len: + return 0 + copyMem(psk, pskString.cstring, pskString.len) + + return pskString.len.cuint + + proc `serverGetPskFunc=`*(ctx: SSLContext, fun: SslServerGetPskFunc) = + ## Sets function that returns PSK based on the client identity. + ## + ## Only used in PSK ciphersuites. + ctx.getExtraInternal().serverGetPskFunc = fun + ctx.context.SSL_CTX_set_psk_server_callback(if fun == nil: nil + else: pskServerCallback) + + proc getPskIdentity*(socket: Socket): string = + ## Gets the PSK identity provided by the client. + assert socket.isSSL + return $(socket.sslHandle.SSL_get_psk_identity) proc wrapSocket*(ctx: SSLContext, socket: Socket) = ## Wraps a socket in an SSL context. This function effectively turns @@ -255,7 +437,7 @@ when defined(ssl): assert (not socket.isSSL) socket.isSSL = true socket.sslContext = ctx - socket.sslHandle = SSLNew(SSLCTX(socket.sslContext)) + socket.sslHandle = SSLNew(socket.sslContext.context) socket.sslNoHandshake = false socket.sslHasPeekChar = false if socket.sslHandle == nil: @@ -301,7 +483,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, ## error was caused by no data being available to be read. ## ## If ``err`` is not lower than 0 no exception will be raised. - when defined(ssl): + when defineSsl: if socket.isSSL: if err <= 0: var ret = SSLGetError(socket.sslHandle, err.cint) @@ -334,7 +516,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, raiseSSLError() else: raiseSSLError("Unknown Error") - if err == -1 and not (when defined(ssl): socket.isSSL else: false): + if err == -1 and not (when defineSsl: socket.isSSL else: false): var lastE = if lastError.int == -1: getSocketError(socket) else: lastError if async: when useWinVersion: @@ -414,7 +596,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, client.isBuffered = server.isBuffered # Handle SSL. - when defined(ssl): + when defineSsl: if server.isSSL: # We must wrap the client sock in a ssl context. @@ -425,7 +607,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, # Client socket is set above. address = $inet_ntoa(sockAddress.sin_addr) -when false: #defined(ssl): +when false: #defineSsl: proc acceptAddrSSL*(server: Socket, client: var Socket, address: var string): SSLAcceptResult {. tags: [ReadIOEffect].} = @@ -444,7 +626,7 @@ when false: #defined(ssl): ## ``AcceptNoClient`` will be returned when no client is currently attempting ## to connect. template doHandshake(): stmt = - when defined(ssl): + when defineSsl: if server.isSSL: client.setBlocking(false) # We must wrap the client sock in a ssl context. @@ -495,7 +677,7 @@ proc accept*(server: Socket, client: var Socket, proc close*(socket: Socket) = ## Closes a socket. try: - when defined(ssl): + when defineSsl: if socket.isSSL: ErrClearError() # As we are closing the underlying socket immediately afterwards, @@ -522,6 +704,7 @@ proc toCInt*(opt: SOBool): cint = of OptKeepAlive: SO_KEEPALIVE of OptOOBInline: SO_OOBINLINE of OptReuseAddr: SO_REUSEADDR + of OptReusePort: SO_REUSEPORT proc getSockOpt*(socket: Socket, opt: SOBool, level = SOL_SOCKET): bool {. tags: [ReadIOEffect].} = @@ -547,8 +730,35 @@ proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) { var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) +when defined(posix) and not defined(nimdoc): + proc makeUnixAddr(path: string): Sockaddr_un = + result.sun_family = AF_UNIX.toInt + if path.len >= Sockaddr_un_path_length: + raise newException(ValueError, "socket path too long") + copyMem(addr result.sun_path, path.cstring, path.len + 1) + +when defined(posix): + proc connectUnix*(socket: Socket, path: string) = + ## Connects to Unix socket on `path`. + ## This only works on Unix-style systems: Mac OS X, BSD and Linux + when not defined(nimdoc): + var socketAddr = makeUnixAddr(path) + if socket.fd.connect(cast[ptr SockAddr](addr socketAddr), + sizeof(socketAddr).Socklen) != 0'i32: + raiseOSError(osLastError()) + + proc bindUnix*(socket: Socket, path: string) = + ## Binds Unix socket to `path`. + ## This only works on Unix-style systems: Mac OS X, BSD and Linux + when not defined(nimdoc): + var socketAddr = makeUnixAddr(path) + if socket.fd.bindAddr(cast[ptr SockAddr](addr socketAddr), + sizeof(socketAddr).Socklen) != 0'i32: + raiseOSError(osLastError()) + when defined(ssl): - proc handshake*(socket: Socket): bool {.tags: [ReadIOEffect, WriteIOEffect].} = + proc handshake*(socket: Socket): bool + {.tags: [ReadIOEffect, WriteIOEffect], deprecated.} = ## This proc needs to be called on a socket after it connects. This is ## only applicable when using ``connectAsync``. ## This proc performs the SSL handshake. @@ -557,6 +767,8 @@ when defined(ssl): ## ``True`` whenever handshake completed successfully. ## ## A ESSL error is raised on any other errors. + ## + ## **Note:** This procedure is deprecated since version 0.14.0. result = true if socket.isSSL: var ret = SSLConnect(socket.sslHandle) @@ -594,7 +806,7 @@ proc hasDataBuffered*(s: Socket): bool = if s.isBuffered: result = s.bufLen > 0 and s.currPos != s.bufLen - when defined(ssl): + when defineSsl: if s.isSSL and not result: result = s.sslHasPeekChar @@ -608,7 +820,7 @@ proc select(readfd: Socket, timeout = 500): int = proc readIntoBuf(socket: Socket, flags: int32): int = result = 0 - when defined(ssl): + when defineSsl: if socket.isSSL: result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high)) else: @@ -658,7 +870,7 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect] result = read else: - when defined(ssl): + when defineSsl: if socket.isSSL: if socket.sslHasPeekChar: copyMem(data, addr(socket.sslPeekChar), 1) @@ -696,7 +908,7 @@ proc waitFor(socket: Socket, waited: var float, timeout, size: int, if timeout - int(waited * 1000.0) < 1: raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") - when defined(ssl): + when defineSsl: if socket.isSSL: if socket.hasDataBuffered: # sslPeekChar is present. @@ -764,7 +976,7 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = c = socket.buffer[socket.currPos] else: - when defined(ssl): + when defineSsl: if socket.isSSL: if not socket.sslHasPeekChar: result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) @@ -792,11 +1004,11 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, ## ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. - template addNLIfEmpty(): stmt = + template addNLIfEmpty() = if line.len == 0: line.string.add("\c\L") - template raiseSockError(): stmt {.dirty, immediate.} = + template raiseSockError() {.dirty.} = let lastError = getSocketError(socket) if flags.isDisconnectionError(lastError): setLen(line.string, 0); return socket.socketError(n, lastError = lastError) @@ -872,7 +1084,7 @@ proc send*(socket: Socket, data: pointer, size: int): int {. ## ## **Note**: This is a low-level version of ``send``. You likely should use ## the version below. - when defined(ssl): + when defineSsl: if socket.isSSL: return SSLWrite(socket.sslHandle, cast[cstring](data), size) @@ -943,7 +1155,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, proc isSsl*(socket: Socket): bool = ## Determines whether ``socket`` is a SSL socket. - when defined(ssl): + when defineSsl: result = socket.isSSL else: result = false @@ -1253,7 +1465,7 @@ proc connect*(socket: Socket, address: string, dealloc(aiList) if not success: raiseOSError(lastError) - when defined(ssl): + when defineSsl: if socket.isSSL: # RFC3546 for SNI specifies that IP addresses are not allowed. if not isIpAddress(address): @@ -1314,8 +1526,10 @@ proc connect*(socket: Socket, address: string, port = Port(0), if selectWrite(s, timeout) != 1: raise newException(TimeoutError, "Call to 'connect' timed out.") else: - when defined(ssl): + when defineSsl and not defined(nimdoc): if socket.isSSL: socket.fd.setBlocking(true) + {.warning[Deprecated]: off.} doAssert socket.handshake() + {.warning[Deprecated]: on.} socket.fd.setBlocking(true) diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 5a7deaab0..4289eb049 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -27,19 +27,22 @@ const tickCountCorrection = 50_000 when not declared(system.StackTrace): - type StackTrace = array [0..20, cstring] + type StackTrace = object + lines: array[0..20, cstring] + files: array[0..20, cstring] {.deprecated: [TStackTrace: StackTrace].} + proc `[]`*(st: StackTrace, i: int): cstring = st.lines[i] # We use a simple hash table of bounded size to keep track of the stack traces: type ProfileEntry = object total: int st: StackTrace - ProfileData = array [0..64*1024-1, ptr ProfileEntry] + ProfileData = array[0..64*1024-1, ptr ProfileEntry] {.deprecated: [TProfileEntry: ProfileEntry, TProfileData: ProfileData].} proc `==`(a, b: StackTrace): bool = - for i in 0 .. high(a): + for i in 0 .. high(a.lines): if a[i] != b[i]: return false result = true @@ -72,7 +75,7 @@ proc hookAux(st: StackTrace, costs: int) = # this is quite performance sensitive! when withThreads: acquire profilingLock inc totalCalls - var last = high(st) + var last = high(st.lines) while last > 0 and isNil(st[last]): dec last var h = hash(pointer(st[last])) and high(profileData) @@ -178,7 +181,7 @@ proc writeProfile() {.noconv.} = var perProc = initCountTable[string]() for i in 0..entries-1: var dups = initSet[string]() - for ii in 0..high(StackTrace): + for ii in 0..high(StackTrace.lines): let procname = profileData[i].st[ii] if isNil(procname): break let p = $procname @@ -193,10 +196,11 @@ proc writeProfile() {.noconv.} = writeLine(f, "Entry: ", i+1, "/", entries, " Calls: ", profileData[i].total // totalCalls, " [sum: ", sum, "; ", sum // totalCalls, "]") - for ii in 0..high(StackTrace): + for ii in 0..high(StackTrace.lines): let procname = profileData[i].st[ii] + let filename = profileData[i].st.files[ii] if isNil(procname): break - writeLine(f, " ", procname, " ", perProc[$procname] // totalCalls) + writeLine(f, " ", $filename & ": " & $procname, " ", perProc[$procname] // totalCalls) close(f) echo "... done" else: diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index fca10dab6..e4c97b260 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -74,8 +74,7 @@ proc genOid*(): Oid = var t = gettime(nil) - var i = int32(incr) - atomicInc(incr) + var i = int32(atomicInc(incr)) if fuzz == 0: # racy, but fine semantically: diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 470559e17..1e474f4d4 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -26,7 +26,6 @@ elif defined(posix): else: {.error: "OS module not ported to your operating system!".} -include "system/ansi_c" include ospaths when defined(posix): @@ -37,6 +36,23 @@ when defined(posix): var pathMax {.importc: "PATH_MAX", header: "<stdlib.h>".}: cint +proc c_remove(filename: cstring): cint {. + importc: "remove", header: "<stdio.h>".} +proc c_rename(oldname, newname: cstring): cint {. + importc: "rename", header: "<stdio.h>".} +proc c_system(cmd: cstring): cint {. + importc: "system", header: "<stdlib.h>".} +proc c_strerror(errnum: cint): cstring {. + importc: "strerror", header: "<string.h>".} +proc c_strlen(a: cstring): cint {. + importc: "strlen", header: "<string.h>", noSideEffect.} +proc c_getenv(env: cstring): cstring {. + importc: "getenv", header: "<stdlib.h>".} +proc c_putenv(env: cstring): cint {. + importc: "putenv", header: "<stdlib.h>".} + +var errno {.importc, header: "<errno.h>".}: cint + proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} = ## Retrieves the operating system's error flag, ``errno``. ## On Windows ``GetLastError`` is checked before ``errno``. @@ -50,18 +66,18 @@ proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} = if err != 0'i32: when useWinUnicode: var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: result = $msgbuf if msgbuf != nil: localFree(cast[pointer](msgbuf)) else: var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200, + if formatMessageA(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF, nil, err, 0, addr(msgbuf), 0, nil) != 0'i32: result = $msgbuf if msgbuf != nil: localFree(msgbuf) if errno != 0'i32: - result = $os.strerror(errno) + result = $os.c_strerror(errno) {.push warning[deprecated]: off.} proc raiseOSError*(msg: string = "") {.noinline, rtl, extern: "nos$1", @@ -114,7 +130,7 @@ proc osErrorMsg*(errorCode: OSErrorCode): string = if msgbuf != nil: localFree(msgbuf) else: if errorCode != OSErrorCode(0'i32): - result = $os.strerror(errorCode.int32) + result = $os.c_strerror(errorCode.int32) proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} = ## Raises an ``OSError`` exception. The ``errorCode`` will determine the @@ -129,7 +145,7 @@ proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} = if additionalInfo.len == 0: e.msg = osErrorMsg(errorCode) else: - e.msg = additionalInfo & " " & osErrorMsg(errorCode) + e.msg = osErrorMsg(errorCode) & "\nAdditional info: " & additionalInfo if e.msg == "": e.msg = "unknown OS error" raise e @@ -157,24 +173,24 @@ proc osLastError*(): OSErrorCode = when defined(windows): when useWinUnicode: - template wrapUnary(varname, winApiProc, arg: expr) {.immediate.} = + template wrapUnary(varname, winApiProc, arg: untyped) = var varname = winApiProc(newWideCString(arg)) - template wrapBinary(varname, winApiProc, arg, arg2: expr) {.immediate.} = + template wrapBinary(varname, winApiProc, arg, arg2: untyped) = var varname = winApiProc(newWideCString(arg), arg2) proc findFirstFile(a: string, b: var WIN32_FIND_DATA): Handle = result = findFirstFileW(newWideCString(a), b) - template findNextFile(a, b: expr): expr = findNextFileW(a, b) - template getCommandLine(): expr = getCommandLineW() + template findNextFile(a, b: untyped): untyped = findNextFileW(a, b) + template getCommandLine(): untyped = getCommandLineW() - template getFilename(f: expr): expr = + template getFilename(f: untyped): untyped = $cast[WideCString](addr(f.cFilename[0])) else: - template findFirstFile(a, b: expr): expr = findFirstFileA(a, b) - template findNextFile(a, b: expr): expr = findNextFileA(a, b) - template getCommandLine(): expr = getCommandLineA() + template findFirstFile(a, b: untyped): untyped = findFirstFileA(a, b) + template findNextFile(a, b: untyped): untyped = findNextFileA(a, b) + template getCommandLine(): untyped = getCommandLineA() - template getFilename(f: expr): expr = $f.cFilename + template getFilename(f: untyped): untyped = $f.cFilename proc skipFindData(f: WIN32_FIND_DATA): bool {.inline.} = # Note - takes advantage of null delimiter in the cstring @@ -320,7 +336,7 @@ proc setCurrentDir*(newDir: string) {.inline, tags: [].} = proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", tags: [ReadDirEffect].} = - ## Returns the full path of `filename`, raises OSError in case of an error. + ## Returns the full (`absolute`:idx:) path of the file `filename`, raises OSError in case of an error. when defined(windows): const bufsize = 3072'i32 when useWinUnicode: @@ -762,12 +778,26 @@ iterator envPairs*(): tuple[key, value: TaintedString] {.tags: [ReadEnvEffect].} yield (TaintedString(substr(environment[i], 0, p-1)), TaintedString(substr(environment[i], p+1))) -iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = - ## Iterate over all the files that match the `pattern`. On POSIX this uses - ## the `glob`:idx: call. - ## - ## `pattern` is OS dependent, but at least the "\*.ext" - ## notation is supported. +# Templates for filtering directories and files +when defined(windows): + template isDir(f: WIN32_FIND_DATA): bool = + (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + template isFile(f: WIN32_FIND_DATA): bool = + not isDir(f) +else: + template isDir(f: string): bool = + dirExists(f) + template isFile(f: string): bool = + fileExists(f) + +template defaultWalkFilter(item): bool = + ## Walk filter used to return true on both + ## files and directories + true + +template walkCommon(pattern: string, filter) = + ## Common code for getting the files and directories with the + ## specified `pattern` when defined(windows): var f: WIN32_FIND_DATA @@ -776,8 +806,7 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = if res != -1: defer: findClose(res) while true: - if not skipFindData(f) and - (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) == 0'i32: + if not skipFindData(f) and filter(f): # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check # that the file extensions have the same length ... let ff = getFilename(f) @@ -799,7 +828,33 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = if res == 0: for i in 0.. f.gl_pathc - 1: assert(f.gl_pathv[i] != nil) - yield $f.gl_pathv[i] + let path = $f.gl_pathv[i] + if filter(path): + yield path + +iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect].} = + ## Iterate over all the files and directories that match the `pattern`. + ## On POSIX this uses the `glob`:idx: call. + ## + ## `pattern` is OS dependent, but at least the "\*.ext" + ## notation is supported. + walkCommon(pattern, defaultWalkFilter) + +iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} = + ## Iterate over all the files that match the `pattern`. On POSIX this uses + ## the `glob`:idx: call. + ## + ## `pattern` is OS dependent, but at least the "\*.ext" + ## notation is supported. + walkCommon(pattern, isFile) + +iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect].} = + ## Iterate over all the directories that match the `pattern`. + ## On POSIX this uses the `glob`:idx: call. + ## + ## `pattern` is OS dependent, but at least the "\*.ext" + ## notation is supported. + walkCommon(pattern, isDir) type PathComponent* = enum ## Enumeration specifying a path component. @@ -1067,7 +1122,7 @@ proc parseCmdLine*(c: string): seq[string] {. while true: setLen(a, 0) # eat all delimiting whitespace - while c[i] == ' ' or c[i] == '\t' or c [i] == '\l' or c [i] == '\r' : inc(i) + while c[i] == ' ' or c[i] == '\t' or c[i] == '\l' or c[i] == '\r' : inc(i) when defined(windows): # parse a single argument according to the above rules: if c[i] == '\0': break @@ -1447,13 +1502,13 @@ type lastWriteTime*: Time # Time file was last modified/written to. creationTime*: Time # Time file was created. Not supported on all systems! -template rawToFormalFileInfo(rawInfo, formalInfo): expr = +template rawToFormalFileInfo(rawInfo, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(Windows): - template toTime(e): expr = winTimeToUnixTime(rdFileTime(e)) - template merge(a, b): expr = a or (b shl 32) + template toTime(e: FILETIME): untyped {.gensym.} = winTimeToUnixTime(rdFileTime(e)) # local templates default to bind semantics + template merge(a, b): untyped = a or (b shl 32) formalInfo.id.device = rawInfo.dwVolumeSerialNumber formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) formalInfo.size = merge(rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) @@ -1478,7 +1533,7 @@ template rawToFormalFileInfo(rawInfo, formalInfo): expr = else: - template checkAndIncludeMode(rawMode, formalMode: expr) = + template checkAndIncludeMode(rawMode, formalMode: untyped) = if (rawInfo.st_mode and rawMode) != 0'i32: formalInfo.permissions.incl(formalMode) formalInfo.id = (rawInfo.st_dev, rawInfo.st_ino) diff --git a/lib/pure/ospaths.nim b/lib/pure/ospaths.nim index 9fc816f2f..56671ee62 100644 --- a/lib/pure/ospaths.nim +++ b/lib/pure/ospaths.nim @@ -10,7 +10,7 @@ # Included by the ``os`` module but a module in its own right for NimScript # support. -when isMainModule: +when not declared(os): {.pragma: rtl.} import strutils @@ -556,12 +556,20 @@ when declared(getEnv) or defined(nimscript): yield substr(s, first, last-1) inc(last) - proc findExe*(exe: string): string {. + when not defined(windows) and declared(os): + proc checkSymlink(path: string): bool = + var rawInfo: Stat + if lstat(path, rawInfo) < 0'i32: result = false + else: result = S_ISLNK(rawInfo.st_mode) + + proc findExe*(exe: string, followSymlinks: bool = true): string {. tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect].} = ## Searches for `exe` in the current working directory and then ## in directories listed in the ``PATH`` environment variable. ## Returns "" if the `exe` cannot be found. On DOS-like platforms, `exe` ## is added the `ExeExt <#ExeExt>`_ file extension if it has none. + ## If the system supports symlinks it also resolves them until it + ## meets the actual file. This behavior can be disabled if desired. result = addFileExt(exe, ExeExt) if existsFile(result): return var path = string(getEnv("PATH")) @@ -572,7 +580,25 @@ when declared(getEnv) or defined(nimscript): result else: var x = expandTilde(candidate) / result - if existsFile(x): return x + if existsFile(x): + when not defined(windows) and declared(os): + while followSymlinks: # doubles as if here + if x.checkSymlink: + var r = newString(256) + var len = readlink(x, r, 256) + if len < 0: + raiseOSError(osLastError()) + if len > 256: + r = newString(len+1) + len = readlink(x, r, len) + setLen(r, len) + if isAbsolute(r): + x = r + else: + x = parentDir(x) / r + else: + break + return x result = "" when defined(nimscript) or (defined(nimdoc) and not declared(os)): diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 38b0ed4a3..7378520e3 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -89,7 +89,7 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" result.add("\"") proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote s, so it can be safely passed to POSIX shell. + ## Quote ``s``, so it can be safely passed to POSIX shell. ## Based on Python's pipes.quote const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', '0'..'9', 'A'..'Z', 'a'..'z'} @@ -104,7 +104,7 @@ proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} return "'" & s.replace("'", "'\"'\"'") & "'" proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote s, so it can be safely passed to shell. + ## Quote ``s``, so it can be safely passed to shell. when defined(Windows): return quoteShellWindows(s) elif defined(posix): @@ -175,7 +175,11 @@ proc startCmd*(command: string, options: set[ProcessOption] = { result = startProcess(command=command, options=options + {poEvalCommand}) proc close*(p: Process) {.rtl, extern: "nosp$1", tags: [].} - ## When the process has finished executing, cleanup related handles + ## When the process has finished executing, cleanup related handles. + ## + ## **Warning:** If the process has not finished executing, this will forcibly + ## terminate the process. Doing so may result in zombie processes and + ## `pty leaks <http://stackoverflow.com/questions/27021641/how-to-fix-request-failed-on-channel-0>`_. proc suspend*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Suspends the process `p`. @@ -400,15 +404,16 @@ when defined(Windows) and not defined(useNimRtl): result = cast[cstring](alloc0(res.len+1)) copyMem(result, cstring(res), res.len) - proc buildEnv(env: StringTableRef): cstring = + proc buildEnv(env: StringTableRef): tuple[str: cstring, len: int] = var L = 0 for key, val in pairs(env): inc(L, key.len + val.len + 2) - result = cast[cstring](alloc0(L+2)) + var str = cast[cstring](alloc0(L+2)) L = 0 for key, val in pairs(env): var x = key & "=" & val - copyMem(addr(result[L]), cstring(x), x.len+1) # copy \0 + copyMem(addr(str[L]), cstring(x), x.len+1) # copy \0 inc(L, x.len+1) + (str, L) #proc open_osfhandle(osh: Handle, mode: int): int {. # importc: "_open_osfhandle", header: "<fcntl.h>".} @@ -526,13 +531,15 @@ when defined(Windows) and not defined(useNimRtl): else: cmdl = buildCommandLine(command, args) var wd: cstring = nil - var e: cstring = nil + var e = (str: nil.cstring, len: -1) if len(workingDir) > 0: wd = workingDir if env != nil: e = buildEnv(env) if poEchoCmd in options: echo($cmdl) when useWinUnicode: var tmp = newWideCString(cmdl) - var ee = newWideCString(e) + var ee = + if e.str.isNil: nil + else: newWideCString(e.str, e.len) var wwd = newWideCString(wd) var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT if poDemon in options: flags = flags or CREATE_NO_WINDOW @@ -549,7 +556,7 @@ when defined(Windows) and not defined(useNimRtl): if poStdErrToStdOut notin options: fileClose(si.hStdError) - if e != nil: dealloc(e) + if e.str != nil: dealloc(e.str) if success == 0: if poInteractive in result.options: close(result) const errInvalidParameter = 87.int @@ -721,7 +728,7 @@ elif not defined(useNimRtl): env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): Process = var - pStdin, pStdout, pStderr: array [0..1, cint] + pStdin, pStdout, pStderr: array[0..1, cint] new(result) result.options = options result.exitCode = -3 # for ``waitForExit`` @@ -875,8 +882,9 @@ elif not defined(useNimRtl): var error: cint let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error)) if sizeRead == sizeof(error): - raiseOSError("Could not find command: '$1'. OS error: $2" % - [$data.sysCommand, $strerror(error)]) + raiseOSError(osLastError(), + "Could not find command: '$1'. OS error: $2" % + [$data.sysCommand, $strerror(error)]) return pid @@ -967,16 +975,168 @@ elif not defined(useNimRtl): if kill(p.id, SIGKILL) != 0'i32: raiseOsError(osLastError()) - proc waitForExit(p: Process, timeout: int = -1): int = - #if waitPid(p.id, p.exitCode, 0) == int(p.id): - # ``waitPid`` fails if the process is not running anymore. But then - # ``running`` probably set ``p.exitCode`` for us. Since ``p.exitCode`` is - # initialized with -3, wrong success exit codes are prevented. - if p.exitCode != -3: return p.exitCode - if waitpid(p.id, p.exitCode, 0) < 0: - p.exitCode = -3 - raiseOSError(osLastError()) - result = int(p.exitCode) shr 8 + when defined(macosx) or defined(freebsd) or defined(netbsd) or + defined(openbsd): + import kqueue, times + + proc waitForExit(p: Process, timeout: int = -1): int = + if p.exitCode != -3: return p.exitCode + if timeout == -1: + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + else: + var kqFD = kqueue() + if kqFD == -1: + raiseOSError(osLastError()) + + var kevIn = KEvent(ident: p.id.uint, filter: EVFILT_PROC, + flags: EV_ADD, fflags: NOTE_EXIT) + var kevOut: KEvent + var tmspec: Timespec + + if timeout >= 1000: + tmspec.tv_sec = (timeout div 1_000).Time + tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 + else: + tmspec.tv_sec = 0.Time + tmspec.tv_nsec = (timeout * 1_000_000) + + try: + while true: + var count = kevent(kqFD, addr(kevIn), 1, addr(kevOut), 1, + addr(tmspec)) + if count < 0: + let err = osLastError() + if err.cint != EINTR: + raiseOSError(osLastError()) + elif count == 0: + # timeout expired, so we trying to kill process + if posix.kill(p.id, SIGKILL) == -1: + raiseOSError(osLastError()) + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + break + else: + if kevOut.ident == p.id.uint and kevOut.filter == EVFILT_PROC: + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + break + else: + raiseOSError(osLastError()) + finally: + discard posix.close(kqFD) + + result = int(p.exitCode) shr 8 + else: + import times + + const + hasThreadSupport = compileOption("threads") and not defined(nimscript) + + proc waitForExit(p: Process, timeout: int = -1): int = + template adjustTimeout(t, s, e: Timespec) = + var diff: int + var b: Timespec + b.tv_sec = e.tv_sec + b.tv_nsec = e.tv_nsec + e.tv_sec = (e.tv_sec - s.tv_sec).Time + if e.tv_nsec >= s.tv_nsec: + e.tv_nsec -= s.tv_nsec + else: + if e.tv_sec == 0.Time: + raise newException(ValueError, "System time was modified") + else: + diff = s.tv_nsec - e.tv_nsec + e.tv_nsec = 1_000_000_000 - diff + t.tv_sec = (t.tv_sec - e.tv_sec).Time + if t.tv_nsec >= e.tv_nsec: + t.tv_nsec -= e.tv_nsec + else: + t.tv_sec = (int(t.tv_sec) - 1).Time + diff = e.tv_nsec - t.tv_nsec + t.tv_nsec = 1_000_000_000 - diff + s.tv_sec = b.tv_sec + s.tv_nsec = b.tv_nsec + + #if waitPid(p.id, p.exitCode, 0) == int(p.id): + # ``waitPid`` fails if the process is not running anymore. But then + # ``running`` probably set ``p.exitCode`` for us. Since ``p.exitCode`` is + # initialized with -3, wrong success exit codes are prevented. + if p.exitCode != -3: return p.exitCode + if timeout == -1: + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + else: + var nmask, omask: Sigset + var sinfo: SigInfo + var stspec, enspec, tmspec: Timespec + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, SIGCHLD) + + when hasThreadSupport: + if pthread_sigmask(SIG_BLOCK, nmask, omask) == -1: + raiseOSError(osLastError()) + else: + if sigprocmask(SIG_BLOCK, nmask, omask) == -1: + raiseOSError(osLastError()) + + if timeout >= 1000: + tmspec.tv_sec = (timeout div 1_000).Time + tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 + else: + tmspec.tv_sec = 0.Time + tmspec.tv_nsec = (timeout * 1_000_000) + + try: + if clock_gettime(CLOCK_REALTIME, stspec) == -1: + raiseOSError(osLastError()) + while true: + let res = sigtimedwait(nmask, sinfo, tmspec) + if res == SIGCHLD: + if sinfo.si_pid == p.id: + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + break + else: + # we have SIGCHLD, but not for process we are waiting, + # so we need to adjust timeout value and continue + if clock_gettime(CLOCK_REALTIME, enspec) == -1: + raiseOSError(osLastError()) + adjustTimeout(tmspec, stspec, enspec) + elif res < 0: + let err = osLastError() + if err.cint == EINTR: + # we have received another signal, so we need to + # adjust timeout and continue + if clock_gettime(CLOCK_REALTIME, enspec) == -1: + raiseOSError(osLastError()) + adjustTimeout(tmspec, stspec, enspec) + elif err.cint == EAGAIN: + # timeout expired, so we trying to kill process + if posix.kill(p.id, SIGKILL) == -1: + raiseOSError(osLastError()) + if waitpid(p.id, p.exitCode, 0) < 0: + p.exitCode = -3 + raiseOSError(osLastError()) + break + else: + raiseOSError(err) + finally: + when hasThreadSupport: + if pthread_sigmask(SIG_UNBLOCK, nmask, omask) == -1: + raiseOSError(osLastError()) + else: + if sigprocmask(SIG_UNBLOCK, nmask, omask) == -1: + raiseOSError(osLastError()) + + result = int(p.exitCode) shr 8 proc peekExitCode(p: Process): int = if p.exitCode != -3: return p.exitCode diff --git a/lib/pure/oswalkdir.nim b/lib/pure/oswalkdir.nim index 000fe25a3..23ca0566a 100644 --- a/lib/pure/oswalkdir.nim +++ b/lib/pure/oswalkdir.nim @@ -1,3 +1,11 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# ## Compile-time only version for walkDir if you need it at compile-time ## for JavaScript. diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 9bcac0a50..c648b0703 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -15,17 +15,78 @@ ## This is an example of how a configuration file may look like: ## -## .. include:: doc/mytest.cfg +## .. include:: ../../doc/mytest.cfg ## :literal: ## The file ``examples/parsecfgex.nim`` demonstrates how to use the ## configuration file parser: ## ## .. code-block:: nim -## :file: examples/parsecfgex.nim - +## :file: ../../examples/parsecfgex.nim +## +## Examples +## -------- +## +## This is an example of a configuration file. +## +## :: +## +## charset = "utf-8" +## [Package] +## name = "hello" +## --threads:on +## [Author] +## name = "lihf8515" +## qq = "10214028" +## email = "lihaifeng@wxm.com" +## +## Creating a configuration file. +## ============================== +## .. code-block:: nim +## +## import parsecfg +## var dict=newConfig() +## dict.setSectionKey("","charset","utf-8") +## dict.setSectionKey("Package","name","hello") +## dict.setSectionKey("Package","--threads","on") +## dict.setSectionKey("Author","name","lihf8515") +## dict.setSectionKey("Author","qq","10214028") +## dict.setSectionKey("Author","email","lihaifeng@wxm.com") +## dict.writeConfig("config.ini") +## +## Reading a configuration file. +## ============================= +## .. code-block:: nim +## +## import parsecfg +## var dict = loadConfig("config.ini") +## var charset = dict.getSectionValue("","charset") +## var threads = dict.getSectionValue("Package","--threads") +## var pname = dict.getSectionValue("Package","name") +## var name = dict.getSectionValue("Author","name") +## var qq = dict.getSectionValue("Author","qq") +## var email = dict.getSectionValue("Author","email") +## echo pname & "\n" & name & "\n" & qq & "\n" & email +## +## Modifying a configuration file. +## =============================== +## .. code-block:: nim +## +## import parsecfg +## var dict = loadConfig("config.ini") +## dict.setSectionKey("Author","name","lhf") +## dict.writeConfig("config.ini") +## +## Deleting a section key in a configuration file. +## =============================================== +## .. code-block:: nim +## +## import parsecfg +## var dict = loadConfig("config.ini") +## dict.delSectionKey("Author","email") +## dict.writeConfig("config.ini") import - hashes, strutils, lexbase, streams + hashes, strutils, lexbase, streams, tables include "system/inclrtl" @@ -70,7 +131,7 @@ type # implementation const - SymChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.', '/', '\\'} + SymChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.', '/', '\\', '-'} proc rawGetTok(c: var CfgParser, tok: var Token) {.gcsafe.} @@ -359,3 +420,138 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = result.kind = cfgError result.msg = errorStr(c, "invalid token: " & c.tok.literal) rawGetTok(c, c.tok) + +# ---------------- Configuration file related operations ---------------- +type + Config* = OrderedTableRef[string, OrderedTableRef[string, string]] + +proc newConfig*(): Config = + ## Create a new configuration table. + ## Useful when wanting to create a configuration file. + result = newOrderedTable[string, OrderedTableRef[string, string]]() + +proc loadConfig*(filename: string): Config = + ## Load the specified configuration file into a new Config instance. + var dict = newOrderedTable[string, OrderedTableRef[string, string]]() + var curSection = "" ## Current section, + ## the default value of the current section is "", + ## which means that the current section is a common + var p: CfgParser + var fileStream = newFileStream(filename, fmRead) + if fileStream != nil: + open(p, fileStream, filename) + while true: + var e = next(p) + case e.kind + of cfgEof: + break + of cfgSectionStart: # Only look for the first time the Section + curSection = e.section + of cfgKeyValuePair: + var t = newOrderedTable[string, string]() + if dict.hasKey(curSection): + t = dict[curSection] + t[e.key] = e.value + dict[curSection] = t + of cfgOption: + var c = newOrderedTable[string, string]() + if dict.hasKey(curSection): + c = dict[curSection] + c["--" & e.key] = e.value + dict[curSection] = c + of cfgError: + break + close(p) + result = dict + +proc replace(s: string): string = + var d = "" + var i = 0 + while i < s.len(): + if s[i] == '\\': + d.add(r"\\") + elif s[i] == '\c' and s[i+1] == '\L': + d.add(r"\n") + inc(i) + elif s[i] == '\c': + d.add(r"\n") + elif s[i] == '\L': + d.add(r"\n") + else: + d.add(s[i]) + inc(i) + result = d + +proc writeConfig*(dict: Config, filename: string) = + ## Writes the contents of the table to the specified configuration file. + ## Note: Comment statement will be ignored. + var file: File + if file.open(filename, fmWrite): + try: + var section, key, value, kv, segmentChar:string + for pair in dict.pairs(): + section = pair[0] + if section != "": ## Not general section + if not allCharsInSet(section, SymChars): ## Non system character + file.writeLine("[\"" & section & "\"]") + else: + file.writeLine("[" & section & "]") + for pair2 in pair[1].pairs(): + key = pair2[0] + value = pair2[1] + if key[0] == '-' and key[1] == '-': ## If it is a command key + segmentChar = ":" + if not allCharsInSet(key[2..key.len()-1], SymChars): + kv.add("--\"") + kv.add(key[2..key.len()-1]) + kv.add("\"") + else: + kv = key + else: + segmentChar = "=" + kv = key + if value != "": ## If the key is not empty + if not allCharsInSet(value, SymChars): + kv.add(segmentChar) + kv.add("\"") + kv.add(replace(value)) + kv.add("\"") + else: + kv.add(segmentChar) + kv.add(value) + file.writeLine(kv) + except: + raise + finally: + file.close() + +proc getSectionValue*(dict: Config, section, key: string): string = + ## Gets the Key value of the specified Section. + if dict.haskey(section): + if dict[section].hasKey(key): + result = dict[section][key] + else: + result = "" + else: + result = "" + +proc setSectionKey*(dict: var Config, section, key, value: string) = + ## Sets the Key value of the specified Section. + var t = newOrderedTable[string, string]() + if dict.hasKey(section): + t = dict[section] + t[key] = value + dict[section] = t + +proc delSection*(dict: var Config, section: string) = + ## Deletes the specified section and all of its sub keys. + dict.del(section) + +proc delSectionKey*(dict: var Config, section, key: string) = + ## Delete the key of the specified section. + if dict.haskey(section): + if dict[section].hasKey(key): + if dict[section].len() == 1: + dict.del(section) + else: + dict[section].del(key) diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index af51e1201..77b145a73 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -25,6 +25,28 @@ ## echo "##", val, "##" ## close(x) ## +## For CSV files with a header row, the header can be read and then used as a +## reference for item access with `rowEntry <#rowEntry.CsvParser.string>`_: +## +## .. code-block:: nim +## import parsecsv +## import os +## # Prepare a file +## var csv_content = """One,Two,Three,Four +## 1,2,3,4 +## 10,20,30,40 +## 100,200,300,400 +## """ +## writeFile("temp.csv", content) +## +## var p: CsvParser +## p.open("temp.csv") +## p.readHeaderRow() +## while p.readRow(): +## echo "new row: " +## for col in items(p.headers): +## echo "##", col, ":", p.rowEntry(col), "##" +## p.close() import lexbase, streams @@ -37,6 +59,9 @@ type sep, quote, esc: char skipWhite: bool currRow: int + headers*: seq[string] ## The columns that are defined in the csv file + ## (read using `readHeaderRow <#readHeaderRow.CsvParser>`_). + ## Used with `rowEntry <#rowEntry.CsvParser.string>`_). CsvError* = object of IOError ## exception that is raised if ## a parsing error occurs @@ -77,6 +102,15 @@ proc open*(my: var CsvParser, input: Stream, filename: string, my.row = @[] my.currRow = 0 +proc open*(my: var CsvParser, filename: string, + separator = ',', quote = '"', escape = '\0', + skipInitialSpace = false) = + ## same as the other `open` but creates the file stream for you. + var s = newFileStream(filename, fmRead) + if s == nil: my.error(0, "cannot open: " & filename) + open(my, s, filename, separator, + quote, escape, skipInitialSpace) + proc parseField(my: var CsvParser, a: var string) = var pos = my.bufpos var buf = my.buf @@ -131,6 +165,8 @@ proc readRow*(my: var CsvParser, columns = 0): bool = ## reads the next row; if `columns` > 0, it expects the row to have ## exactly this many columns. Returns false if the end of the file ## has been encountered else true. + ## + ## Blank lines are skipped. var col = 0 # current column var oldpos = my.bufpos while my.buf[my.bufpos] != '\0': @@ -166,6 +202,22 @@ proc close*(my: var CsvParser) {.inline.} = ## closes the parser `my` and its associated input stream. lexbase.close(my) +proc readHeaderRow*(my: var CsvParser) = + ## Reads the first row and creates a look-up table for column numbers + ## See also `rowEntry <#rowEntry.CsvParser.string>`_. + var present = my.readRow() + if present: + my.headers = my.row + +proc rowEntry*(my: var CsvParser, entry: string): string = + ## Reads a specified `entry` from the current row. + ## + ## Assumes that `readHeaderRow <#readHeaderRow.CsvParser>`_ has already been + ## called. + var index = my.headers.find(entry) + if index >= 0: + result = my.row[index] + when not defined(testing) and isMainModule: import os var s = newFileStream(paramStr(1), fmRead) @@ -178,3 +230,35 @@ when not defined(testing) and isMainModule: echo "##", val, "##" close(x) +when isMainModule: + import os + import strutils + block: # Tests for reading the header row + var content = "One,Two,Three,Four\n1,2,3,4\n10,20,30,40,\n100,200,300,400\n" + writeFile("temp.csv", content) + + var p: CsvParser + p.open("temp.csv") + p.readHeaderRow() + while p.readRow(): + var zeros = repeat('0', p.currRow-2) + doAssert p.rowEntry("One") == "1" & zeros + doAssert p.rowEntry("Two") == "2" & zeros + doAssert p.rowEntry("Three") == "3" & zeros + doAssert p.rowEntry("Four") == "4" & zeros + p.close() + + when not defined(testing): + var parser: CsvParser + parser.open("temp.csv") + parser.readHeaderRow() + while parser.readRow(): + echo "new row: " + for col in items(parser.headers): + echo "##", col, ":", parser.rowEntry(col), "##" + parser.close() + removeFile("temp.csv") + + # Tidy up + removeFile("temp.csv") + diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 698bde42a..fb7d72182 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -173,6 +173,22 @@ proc parseUntil*(s: string, token: var string, until: char, result = i-start token = substr(s, start, i-1) +proc parseUntil*(s: string, token: var string, until: string, + start = 0): int {.inline.} = + ## parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that comes before the `until` token. + var i = start + while i < s.len: + if s[i] == until[0]: + var u = 1 + while i+u < s.len and u < until.len and s[i+u] == until[u]: + inc u + if u >= until.len: break + inc(i) + result = i-start + token = substr(s, start, i-1) + proc parseWhile*(s: string, token: var string, validChars: set[char], start = 0): int {.inline.} = ## parses a token and stores it in ``token``. Returns @@ -234,6 +250,51 @@ proc parseInt*(s: string, number: var int, start = 0): int {. elif result != 0: number = int(res) +# overflowChecks doesn't work with uint64 +proc rawParseUInt(s: string, b: var uint64, start = 0): int = + var + res = 0'u64 + prev = 0'u64 + i = start + if s[i] == '+': inc(i) # Allow + if s[i] in {'0'..'9'}: + b = 0 + while s[i] in {'0'..'9'}: + prev = res + res = res * 10 + (ord(s[i]) - ord('0')).uint64 + if prev > res: + return 0 # overflowChecks emulation + inc(i) + while s[i] == '_': inc(i) # underscores are allowed and ignored + b = res + result = i - start + +proc parseBiggestUInt*(s: string, number: var uint64, start = 0): int {. + rtl, extern: "npuParseBiggestUInt", noSideEffect.} = + ## parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## Result is the number of processed chars or 0 if there is no integer + ## or overflow detected. + var res: uint64 + # use 'res' for exception safety (don't write to 'number' in case of an + # overflow exception): + result = rawParseUInt(s, res, start) + number = res + +proc parseUInt*(s: string, number: var uint, start = 0): int {. + rtl, extern: "npuParseUInt", noSideEffect.} = + ## parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## Result is the number of processed chars or 0 if there is no integer or + ## overflow detected. + var res: uint64 + result = parseBiggestUInt(s, res, start) + if (sizeof(uint) <= 4) and + (res > 0xFFFF_FFFF'u64): + raise newException(OverflowError, "overflow") + elif result != 0: + number = uint(res) + proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. magic: "ParseBiggestFloat", importc: "nimParseBiggestFloat", noSideEffect.} ## parses a float starting at `start` and stores the value into `number`. diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index f8b2c3d8d..d16a55302 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -34,7 +34,7 @@ ## document. ## ## .. code-block:: nim -## :file: examples/htmltitle.nim +## :file: ../../examples/htmltitle.nim ## ## ## Example 2: Retrieve all HTML links @@ -45,7 +45,7 @@ ## an HTML document contains. ## ## .. code-block:: nim -## :file: examples/htmlrefs.nim +## :file: ../../examples/htmlrefs.nim ## import @@ -142,6 +142,9 @@ proc kind*(my: XmlParser): XmlEventKind {.inline.} = template charData*(my: XmlParser): string = ## returns the character data for the events: ``xmlCharData``, ## ``xmlWhitespace``, ``xmlComment``, ``xmlCData``, ``xmlSpecial`` + ## Raises an assertion in debug mode if ``my.kind`` is not one + ## of those events. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind in {xmlCharData, xmlWhitespace, xmlComment, xmlCData, xmlSpecial}) my.a @@ -149,31 +152,49 @@ template charData*(my: XmlParser): string = template elementName*(my: XmlParser): string = ## returns the element name for the events: ``xmlElementStart``, ## ``xmlElementEnd``, ``xmlElementOpen`` + ## Raises an assertion in debug mode if ``my.kind`` is not one + ## of those events. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind in {xmlElementStart, xmlElementEnd, xmlElementOpen}) my.a template entityName*(my: XmlParser): string = ## returns the entity name for the event: ``xmlEntity`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlEntity``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlEntity) my.a template attrKey*(my: XmlParser): string = ## returns the attribute key for the event ``xmlAttribute`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlAttribute``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.a template attrValue*(my: XmlParser): string = ## returns the attribute value for the event ``xmlAttribute`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlAttribute``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.b template piName*(my: XmlParser): string = ## returns the processing instruction name for the event ``xmlPI`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlPI``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.a template piRest*(my: XmlParser): string = ## returns the rest of the processing instruction for the event ``xmlPI`` + ## Raises an assertion in debug mode if ``my.kind`` is not + ## ``xmlPI``. In release mode, this will not trigger an error + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.b diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index fead66de2..5c978a2f8 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -12,7 +12,7 @@ ## Matching performance is hopefully competitive with optimized regular ## expression engines. ## -## .. include:: ../doc/pegdocs.txt +## .. include:: ../../doc/pegdocs.txt ## include "system/inclrtl" @@ -659,7 +659,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. of pkSearch: var oldMl = c.ml result = 0 - while start+result < s.len: + while start+result <= s.len: var x = rawMatch(s, p.sons[0], start+result, c) if x >= 0: inc(result, x) @@ -671,7 +671,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. var idx = c.ml # reserve a slot for the subpattern inc(c.ml) result = 0 - while start+result < s.len: + while start+result <= s.len: var x = rawMatch(s, p.sons[0], start+result, c) if x >= 0: if idx < MaxSubpatterns: @@ -962,6 +962,50 @@ proc parallelReplace*(s: string, subs: varargs[ # copy the rest: add(result, substr(s, i)) +proc replace*(s: string, sub: Peg, cb: proc( + match: int, cnt: int, caps: openArray[string]): string): string {. + rtl, extern: "npegs$1cb".}= + ## Replaces `sub` in `s` by the resulting strings from the callback. + ## The callback proc receives the index of the current match (starting with 0), + ## the count of captures and an open array with the captures of each match. Examples: + ## + ## .. code-block:: nim + ## + ## proc handleMatches*(m: int, n: int, c: openArray[string]): string = + ## result = "" + ## if m > 0: + ## result.add ", " + ## result.add case n: + ## of 2: c[0].toLower & ": '" & c[1] & "'" + ## of 1: c[0].toLower & ": ''" + ## else: "" + ## + ## let s = "Var1=key1;var2=Key2; VAR3" + ## echo s.replace(peg"{\ident}('='{\ident})* ';'* \s*", handleMatches) + ## + ## Results in: + ## + ## .. code-block:: nim + ## + ## "var1: 'key1', var2: 'Key2', var3: ''" + result = "" + var i = 0 + var caps: array[0..MaxSubpatterns-1, string] + var c: Captures + var m = 0 + while i < s.len: + c.ml = 0 + var x = rawMatch(s, sub, i, c) + if x <= 0: + add(result, s[i]) + inc(i) + else: + fillMatches(s, caps, c) + add(result, cb(m, c.ml, caps)) + inc(i, x) + inc(m) + add(result, substr(s, i)) + proc transformFile*(infile, outfile: string, subs: varargs[tuple[pattern: Peg, repl: string]]) {. rtl, extern: "npegs$1".} = @@ -1789,3 +1833,22 @@ when isMainModule: assert(str.find(empty_test) == 0) assert(str.match(empty_test)) + + proc handleMatches*(m: int, n: int, c: openArray[string]): string = + result = "" + + if m > 0: + result.add ", " + + result.add case n: + of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'" + of 1: toLowerAscii(c[0]) & ": ''" + else: "" + + assert("Var1=key1;var2=Key2; VAR3". + replace(peg"{\ident}('='{\ident})* ';'* \s*", + handleMatches)=="var1: 'key1', var2: 'Key2', var3: ''") + + + doAssert "test1".match(peg"""{@}$""") + doAssert "test2".match(peg"""{(!$ .)*} $""") diff --git a/lib/pure/punycode.nim b/lib/pure/punycode.nim new file mode 100644 index 000000000..ab6501ed1 --- /dev/null +++ b/lib/pure/punycode.nim @@ -0,0 +1,174 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import strutils +import unicode + +# issue #3045 + +const + Base = 36 + TMin = 1 + TMax = 26 + Skew = 38 + Damp = 700 + InitialBias = 72 + InitialN = 128 + Delimiter = '-' + +type + PunyError* = object of Exception + +proc decodeDigit(x: char): int {.raises: [PunyError].} = + if '0' <= x and x <= '9': + result = ord(x) - (ord('0') - 26) + elif 'A' <= x and x <= 'Z': + result = ord(x) - ord('A') + elif 'a' <= x and x <= 'z': + result = ord(x) - ord('a') + else: + raise newException(PunyError, "Bad input") + +proc encodeDigit(digit: int): Rune {.raises: [PunyError].} = + if 0 <= digit and digit < 26: + result = Rune(digit + ord('a')) + elif 26 <= digit and digit < 36: + result = Rune(digit + (ord('0') - 26)) + else: + raise newException(PunyError, "internal error in punycode encoding") + +proc isBasic(c: char): bool = ord(c) < 0x80 +proc isBasic(r: Rune): bool = int(r) < 0x80 + +proc adapt(delta, numPoints: int, first: bool): int = + var d = if first: delta div Damp else: delta div 2 + d += d div numPoints + var k = 0 + while d > ((Base-TMin)*TMax) div 2: + d = d div (Base - TMin) + k += Base + result = k + (Base - TMin + 1) * d div (d + Skew) + +proc encode*(prefix, s: string): string {.raises: [PunyError].} = + ## Encode a string that may contain Unicode. + ## Prepend `prefix` to the result + result = prefix + var (d, n, bias) = (0, InitialN, InitialBias) + var (b, remaining) = (0, 0) + for r in s.runes: + if r.isBasic: + # basic Ascii character + inc b + result.add($r) + else: + # special character + inc remaining + + var h = b + if b > 0: + result.add(Delimiter) # we have some Ascii chars + while remaining != 0: + var m: int = high(int32) + for r in s.runes: + if m > int(r) and int(r) >= n: + m = int(r) + d += (m - n) * (h + 1) + if d < 0: + raise newException(PunyError, "invalid label " & s) + n = m + for r in s.runes: + if int(r) < n: + inc d + if d < 0: + raise newException(PunyError, "invalid label " & s) + continue + if int(r) > n: + continue + var q = d + var k = Base + while true: + var t = k - bias + if t < TMin: + t = TMin + elif t > TMax: + t = TMax + if q < t: + break + result.add($encodeDigit(t + (q - t) mod (Base - t))) + q = (q - t) div (Base - t) + k += Base + result.add($encodeDigit(q)) + bias = adapt(d, h + 1, h == b) + d = 0 + inc h + dec remaining + inc d + inc n + +proc encode*(s: string): string {.raises: [PunyError].} = + ## Encode a string that may contain Unicode. Prefix is empty. + result = encode("", s) + +proc decode*(encoded: string): string {.raises: [PunyError].} = + ## Decode a Punycode-encoded string + var + n = InitialN + i = 0 + bias = InitialBias + var d = rfind(encoded, Delimiter) + result = "" + + if d > 0: + # found Delimiter + for j in 0..<d: + var c = encoded[j] # char + if not c.isBasic: + raise newException(PunyError, "Encoded contains a non-basic char") + result.add(c) # add the character + inc d + else: + d = 0 # set to first index + + while (d < len(encoded)): + var oldi = i + var w = 1 + var k = Base + while true: + if d == len(encoded): + raise newException(PunyError, "Bad input: " & encoded) + var c = encoded[d]; inc d + var digit = int(decodeDigit(c)) + if digit > (high(int32) - i) div w: + raise newException(PunyError, "Too large a value: " & $digit) + i += digit * w + var t: int + if k <= bias: + t = TMin + elif k >= bias + TMax: + t = TMax + else: + t = k - bias + if digit < t: + break + w *= Base - t + k += Base + bias = adapt(i - oldi, runelen(result) + 1, oldi == 0) + + if i div (runelen(result) + 1) > high(int32) - n: + raise newException(PunyError, "Value too large") + + n += i div (runelen(result) + 1) + i = i mod (runelen(result) + 1) + insert(result, $Rune(n), i) + inc i + +when isMainModule: + assert(decode(encode("", "bücher")) == "bücher") + assert(decode(encode("münchen")) == "münchen") + assert encode("xn--", "münchen") == "xn--mnchen-3ya" diff --git a/lib/pure/random.nim b/lib/pure/random.nim new file mode 100644 index 000000000..08da771dc --- /dev/null +++ b/lib/pure/random.nim @@ -0,0 +1,128 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nim's standard random number generator. Based on the ``xoroshiro128+`` (xor/rotate/shift/rotate) library. +## * More information: http://xoroshiro.di.unimi.it/ +## * C implementation: http://xoroshiro.di.unimi.it/xoroshiro128plus.c +## + +include "system/inclrtl" +{.push debugger:off.} + +# XXX Expose RandomGenState +when defined(JS): + type ui = uint32 +else: + type ui = uint64 + +type + RandomGenState = object + a0, a1: ui + +when defined(JS): + var state = RandomGenState( + a0: 0x69B4C98Cu32, + a1: 0xFED1DD30u32) # global for backwards compatibility +else: + # racy for multi-threading but good enough for now: + var state = RandomGenState( + a0: 0x69B4C98CB8530805u64, + a1: 0xFED1DD3004688D67CAu64) # global for backwards compatibility + +proc rotl(x, k: ui): ui = + result = (x shl k) or (x shr (ui(64) - k)) + +proc next(s: var RandomGenState): uint64 = + let s0 = s.a0 + var s1 = s.a1 + result = s0 + s1 + s1 = s1 xor s0 + s.a0 = rotl(s0, 55) xor s1 xor (s1 shl 14) # a, b + s.a1 = rotl(s1, 36) # c + +proc skipRandomNumbers(s: var RandomGenState) = + ## This is the jump function for the generator. It is equivalent + ## to 2^64 calls to next(); it can be used to generate 2^64 + ## non-overlapping subsequences for parallel computations. + when defined(JS): + const helper = [0xbeac0467u32, 0xd86b048bu32] + else: + const helper = [0xbeac0467eba5facbu64, 0xd86b048b86aa9922u64] + var + s0 = ui 0 + s1 = ui 0 + for i in 0..high(helper): + for b in 0..< 64: + if (helper[i] and (ui(1) shl ui(b))) != 0: + s0 = s0 xor s.a0 + s1 = s1 xor s.a1 + discard next(s) + s.a0 = s0 + s.a1 = s1 + +proc random*(max: int): int {.benign.} = + ## Returns a random number in the range 0..max-1. The sequence of + ## random number is always the same, unless `randomize` is called + ## which initializes the random number generator with a "random" + ## number, i.e. a tickcount. + result = int(next(state) mod uint64(max)) + +proc random*(max: float): float {.benign.} = + ## Returns a random number in the range 0..<max. The sequence of + ## random number is always the same, unless `randomize` is called + ## which initializes the random number generator with a "random" + ## number, i.e. a tickcount. + let x = next(state) + when defined(JS): + result = (float(x) / float(high(uint32))) * max + else: + let u = (0x3FFu64 shl 52u64) or (x shr 12u64) + result = (cast[float](u) - 1.0) * max + +proc random*[T](x: Slice[T]): T = + ## For a slice `a .. b` returns a value in the range `a .. b-1`. + result = random(x.b - x.a) + x.a + +proc random*[T](a: openArray[T]): T = + ## returns a random element from the openarray `a`. + result = a[random(a.low..a.len)] + +proc randomize*(seed: int) {.benign.} = + ## Initializes the random number generator with a specific seed. + state.a0 = ui(seed shr 16) + state.a1 = ui(seed and 0xffff) + +when not defined(nimscript): + import times + + proc randomize*() {.benign.} = + ## Initializes the random number generator with a "random" + ## number, i.e. a tickcount. Note: Does not work for NimScript. + when defined(JS): + proc getMil(t: Time): int {.importcpp: "getTime", nodecl.} + randomize(getMil times.getTime()) + else: + randomize(int times.getTime()) + +{.pop.} + +when isMainModule: + proc main = + var occur: array[1000, int] + + var x = 8234 + for i in 0..100_000: + x = random(len(occur)) # myrand(x) + inc occur[x] + for i, oc in occur: + if oc < 69: + doAssert false, "too few occurances of " & $i + elif oc > 130: + doAssert false, "too many occurances of " & $i + main() diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index 6fd05dc4b..bf134f2ae 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -41,26 +41,26 @@ proc toRational*[T:SomeInteger](x: T): Rational[T] = proc toRationalSub(x: float, n: int): Rational[int] = var - a = 0 - b, c, d = 1 + a = 0'i64 + b, c, d = 1'i64 result = 0 // 1 # rational 0 while b <= n and d <= n: let ac = (a+c) let bd = (b+d) # scale by 1000 so not overflow for high precision - let mediant = (ac/1000) / (bd/1000) + let mediant = (ac.float/1000) / (bd.float/1000) if x == mediant: if bd <= n: - result.num = ac - result.den = bd + result.num = ac.int + result.den = bd.int return result elif d > b: - result.num = c - result.den = d + result.num = c.int + result.den = d.int return result else: - result.num = a - result.den = b + result.num = a.int + result.den = b.int return result elif x > mediant: a = ac @@ -69,8 +69,8 @@ proc toRationalSub(x: float, n: int): Rational[int] = c = ac d = bd if (b > n): - return initRational(c, d) - return initRational(a, b) + return initRational(c.int, d.int) + return initRational(a.int, b.int) proc toRational*(x: float, n: int = high(int)): Rational[int] = ## Calculate the best rational numerator and denominator diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 89e92c133..098b78c95 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -132,11 +132,12 @@ elif defined(linux): s.fds[fd].events = events proc unregister*(s: var Selector, fd: SocketHandle) = - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: - let err = osLastError() - if err.cint notin {ENOENT, EBADF}: - # TODO: Why do we sometimes get an EBADF? Is this normal? - raiseOSError(err) + if s.fds[fd].events != {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: + let err = osLastError() + if err.cint notin {ENOENT, EBADF}: + # TODO: Why do we sometimes get an EBADF? Is this normal? + raiseOSError(err) s.fds.del(fd) proc close*(s: var Selector) = diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim index b2adac2f3..050712902 100644 --- a/lib/pure/smtp.nim +++ b/lib/pure/smtp.nim @@ -20,9 +20,9 @@ ## var msg = createMessage("Hello from Nim's SMTP", ## "Hello!.\n Is this awesome or what?", ## @["foo@gmail.com"]) -## var smtp = connect("smtp.gmail.com", 465, true, true) -## smtp.auth("username", "password") -## smtp.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) +## var smtpConn = connect("smtp.gmail.com", Port 465, true, true) +## smtpConn.auth("username", "password") +## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) ## ## ## For SSL support this module relies on OpenSSL. If you want to @@ -31,6 +31,8 @@ import net, strutils, strtabs, base64, os import asyncnet, asyncdispatch +export Port + type Smtp* = object sock: Socket @@ -258,8 +260,8 @@ when not defined(testing) and isMainModule: # "Hello, my name is dom96.\n What\'s yours?", @["dominik@localhost"]) #echo(msg) - #var smtp = connect("localhost", 25, False, True) - #smtp.sendmail("root@localhost", @["dominik@localhost"], $msg) + #var smtpConn = connect("localhost", Port 25, false, true) + #smtpConn.sendmail("root@localhost", @["dominik@localhost"], $msg) #echo(decode("a17sm3701420wbe.12")) proc main() {.async.} = diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index c606b4680..eea06f4ce 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -306,68 +306,68 @@ proc peekLine*(s: Stream): TaintedString = defer: setPosition(s, pos) result = readLine(s) -type - StringStream* = ref StringStreamObj ## a stream that encapsulates a string - StringStreamObj* = object of StreamObj - data*: string - pos: int - -{.deprecated: [PStringStream: StringStream, TStringStream: StringStreamObj].} - -proc ssAtEnd(s: Stream): bool = - var s = StringStream(s) - return s.pos >= s.data.len - -proc ssSetPosition(s: Stream, pos: int) = - var s = StringStream(s) - s.pos = clamp(pos, 0, s.data.len) - -proc ssGetPosition(s: Stream): int = - var s = StringStream(s) - return s.pos - -proc ssReadData(s: Stream, buffer: pointer, bufLen: int): int = - var s = StringStream(s) - result = min(bufLen, s.data.len - s.pos) - if result > 0: - copyMem(buffer, addr(s.data[s.pos]), result) - inc(s.pos, result) - -proc ssPeekData(s: Stream, buffer: pointer, bufLen: int): int = - var s = StringStream(s) - result = min(bufLen, s.data.len - s.pos) - if result > 0: - copyMem(buffer, addr(s.data[s.pos]), result) - -proc ssWriteData(s: Stream, buffer: pointer, bufLen: int) = - var s = StringStream(s) - if bufLen <= 0: - return - if s.pos + bufLen > s.data.len: - setLen(s.data, s.pos + bufLen) - copyMem(addr(s.data[s.pos]), buffer, bufLen) - inc(s.pos, bufLen) - -proc ssClose(s: Stream) = - var s = StringStream(s) - s.data = nil - -proc newStringStream*(s: string = ""): StringStream = - ## creates a new stream from the string `s`. - new(result) - result.data = s - result.pos = 0 - result.closeImpl = ssClose - result.atEndImpl = ssAtEnd - result.setPositionImpl = ssSetPosition - result.getPositionImpl = ssGetPosition - result.readDataImpl = ssReadData - result.peekDataImpl = ssPeekData - result.writeDataImpl = ssWriteData - when not defined(js): type + StringStream* = ref StringStreamObj ## a stream that encapsulates a string + StringStreamObj* = object of StreamObj + data*: string + pos: int + + {.deprecated: [PStringStream: StringStream, TStringStream: StringStreamObj].} + + proc ssAtEnd(s: Stream): bool = + var s = StringStream(s) + return s.pos >= s.data.len + + proc ssSetPosition(s: Stream, pos: int) = + var s = StringStream(s) + s.pos = clamp(pos, 0, s.data.len) + + proc ssGetPosition(s: Stream): int = + var s = StringStream(s) + return s.pos + + proc ssReadData(s: Stream, buffer: pointer, bufLen: int): int = + var s = StringStream(s) + result = min(bufLen, s.data.len - s.pos) + if result > 0: + copyMem(buffer, addr(s.data[s.pos]), result) + inc(s.pos, result) + + proc ssPeekData(s: Stream, buffer: pointer, bufLen: int): int = + var s = StringStream(s) + result = min(bufLen, s.data.len - s.pos) + if result > 0: + copyMem(buffer, addr(s.data[s.pos]), result) + + proc ssWriteData(s: Stream, buffer: pointer, bufLen: int) = + var s = StringStream(s) + if bufLen <= 0: + return + if s.pos + bufLen > s.data.len: + setLen(s.data, s.pos + bufLen) + copyMem(addr(s.data[s.pos]), buffer, bufLen) + inc(s.pos, bufLen) + + proc ssClose(s: Stream) = + var s = StringStream(s) + s.data = nil + + proc newStringStream*(s: string = ""): StringStream = + ## creates a new stream from the string `s`. + new(result) + result.data = s + result.pos = 0 + result.closeImpl = ssClose + result.atEndImpl = ssAtEnd + result.setPositionImpl = ssSetPosition + result.getPositionImpl = ssGetPosition + result.readDataImpl = ssReadData + result.peekDataImpl = ssPeekData + result.writeDataImpl = ssWriteData + + type FileStream* = ref FileStreamObj ## a stream that encapsulates a `File` FileStreamObj* = object of Stream f: File diff --git a/lib/pure/strmisc.nim b/lib/pure/strmisc.nim new file mode 100644 index 000000000..89ef2fcd2 --- /dev/null +++ b/lib/pure/strmisc.nim @@ -0,0 +1,83 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Joey Payne +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains various string utility routines that are uncommonly +## used in comparison to `strutils <strutils.html>`_. + +import strutils + +{.deadCodeElim: on.} + +proc expandTabs*(s: string, tabSize: int = 8): string {.noSideEffect, + procvar.} = + ## Expand tab characters in `s` by `tabSize` spaces + + result = newStringOfCap(s.len + s.len shr 2) + var pos = 0 + + template addSpaces(n) = + for j in 0 ..< n: + result.add(' ') + pos += 1 + + for i in 0 ..< len(s): + let c = s[i] + if c == '\t': + let + denominator = if tabSize > 0: tabSize else: 1 + numSpaces = tabSize - pos mod denominator + + addSpaces(numSpaces) + else: + result.add(c) + pos += 1 + if c == '\l': + pos = 0 + +proc partition*(s: string, sep: string, + right: bool = false): (string, string, string) + {.noSideEffect, procvar.} = + ## Split the string at the first or last occurrence of `sep` into a 3-tuple + ## + ## Returns a 3 string tuple of (beforeSep, `sep`, afterSep) or + ## (`s`, "", "") if `sep` is not found and `right` is false or + ## ("", "", `s`) if `sep` is not found and `right` is true + let position = if right: s.rfind(sep) else: s.find(sep) + if position != -1: + return (s[0 ..< position], sep, s[position + sep.len ..< s.len]) + return if right: ("", "", s) else: (s, "", "") + +proc rpartition*(s: string, sep: string): (string, string, string) + {.noSideEffect, procvar.} = + ## Split the string at the last occurrence of `sep` into a 3-tuple + ## + ## Returns a 3 string tuple of (beforeSep, `sep`, afterSep) or + ## ("", "", `s`) if `sep` is not found + return partition(s, sep, right = true) + +when isMainModule: + doAssert expandTabs("\t", 4) == " " + doAssert expandTabs("\tfoo\t", 4) == " foo " + doAssert expandTabs("\tfoo\tbar", 4) == " foo bar" + doAssert expandTabs("\tfoo\tbar\t", 4) == " foo bar " + doAssert expandTabs("", 4) == "" + doAssert expandTabs("", 0) == "" + doAssert expandTabs("\t\t\t", 0) == "" + + doAssert partition("foo:bar", ":") == ("foo", ":", "bar") + doAssert partition("foobarbar", "bar") == ("foo", "bar", "bar") + doAssert partition("foobarbar", "bank") == ("foobarbar", "", "") + doAssert partition("foobarbar", "foo") == ("", "foo", "barbar") + doAssert partition("foofoobar", "bar") == ("foofoo", "bar", "") + + doAssert rpartition("foo:bar", ":") == ("foo", ":", "bar") + doAssert rpartition("foobarbar", "bar") == ("foobar", "bar", "") + doAssert rpartition("foobarbar", "bank") == ("", "", "foobarbar") + doAssert rpartition("foobarbar", "foo") == ("", "foo", "barbar") + doAssert rpartition("foofoobar", "bar") == ("foofoo", "bar", "") diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim new file mode 100644 index 000000000..246f018c5 --- /dev/null +++ b/lib/pure/strscans.nim @@ -0,0 +1,522 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +This module contains a `scanf`:idx: macro that can be used for extracting +substrings from an input string. This is often easier than regular expressions. +Some examples as an apetizer: + +.. code-block:: nim + # check if input string matches a triple of integers: + const input = "(1,2,4)" + var x, y, z: int + if scanf(input, "($i,$i,$i)", x, y, z): + echo "matches and x is ", x, " y is ", y, " z is ", z + + # check if input string matches an ISO date followed by an identifier followed + # by whitespace and a floating point number: + var year, month, day: int + var identifier: string + var myfloat: float + if scanf(input, "$i-$i-$i $w$s$f", year, month, day, identifier, myfloat): + echo "yes, we have a match!" + +As can be seen from the examples, strings are matched verbatim except for +substrings starting with ``$``. These constructions are available: + +================= ======================================================== +``$i`` Matches an integer. This uses ``parseutils.parseInt``. +``$f`` Matches a floating pointer number. Uses ``parseFloat``. +``$w`` Matches an ASCII identifier: ``[A-Z-a-z_][A-Za-z_0-9]*``. +``$s`` Skips optional whitespace. +``$$`` Matches a single dollar sign. +``$.`` Matches if the end of the input string has been reached. +``$*`` Matches until the token following the ``$*`` was found. + The match is allowed to be of 0 length. +``$+`` Matches until the token following the ``$+`` was found. + The match must consist of at least one char. +``${foo}`` User defined matcher. Uses the proc ``foo`` to perform + the match. See below for more details. +``$[foo]`` Call user defined proc ``foo`` to **skip** some optional + parts in the input string. See below for more details. +================= ======================================================== + +Even though ``$*`` and ``$+`` look similar to the regular expressions ``.*`` +and ``.+`` they work quite differently, there is no non-deterministic +state machine involved and the matches are non-greedy. ``[$*]`` +matches ``[xyz]`` via ``parseutils.parseUntil``. + +Furthermore no backtracking is performed, if parsing fails after a value +has already been bound to a matched subexpression this value is not restored +to its original value. This rarely causes problems in practice and if it does +for you, it's easy enough to bind to a temporary variable first. + + +Startswith vs full match +======================== + +``scanf`` returns true if the input string **starts with** the specified +pattern. If instead it should only return true if theres is also nothing +left in the input, append ``$.`` to your pattern. + + +User definable matchers +======================= + +One very nice advantage over regular expressions is that ``scanf`` is +extensible with ordinary Nim procs. The proc is either enclosed in ``${}`` +or in ``$[]``. ``${}`` matches and binds the result +to a variable (that was passed to the ``scanf`` macro) while ``$[]`` merely +optional tokens. + + +In this example, we define a helper proc ``skipSep`` that skips some separators +which we then use in our scanf pattern to help us in the matching process: + +.. code-block:: nim + + proc someSep(input: string; start: int; seps: set[char] = {':','-','.'}): int = + # Note: The parameters and return value must match to what ``scanf`` requires + result = 0 + while input[start+result] in seps: inc result + + if scanf(input, "$w${someSep}$w", key, value): + ... + +It also possible to pass arguments to a user definable matcher: + +.. code-block:: nim + + proc ndigits(input: string; start: int; intVal: var int; n: int): int = + # matches exactly ``n`` digits. Matchers need to return 0 if nothing + # matched or otherwise the number of processed chars. + var x = 0 + var i = 0 + while i < n and i+start < input.len and input[i+start] in {'0'..'9'}: + x = x * 10 + input[i+start].ord - '0'.ord + inc i + # only overwrite if we had a match + if i == n: + result = n + intVal = x + + # match an ISO date extracting year, month, day at the same time. + # Also ensure the input ends after the ISO date: + var year, month, day: int + if scanf("2013-01-03", "${ndigits(4)}-${ndigits(2)}-${ndigits(2)}$.", year, month, day): + ... + +]## + + +import macros, parseutils + +proc conditionsToIfChain(n, idx, res: NimNode; start: int): NimNode = + assert n.kind == nnkStmtList + if start >= n.len: return newAssignment(res, newLit true) + var ifs: NimNode = nil + if n[start+1].kind == nnkEmpty: + ifs = conditionsToIfChain(n, idx, res, start+3) + else: + ifs = newIfStmt((n[start+1], + newTree(nnkStmtList, newCall(bindSym"inc", idx, n[start+2]), + conditionsToIfChain(n, idx, res, start+3)))) + result = newTree(nnkStmtList, n[start], ifs) + +proc notZero(x: NimNode): NimNode = newCall(bindSym"!=", x, newLit 0) + +proc buildUserCall(x: string; args: varargs[NimNode]): NimNode = + let y = parseExpr(x) + result = newTree(nnkCall) + if y.kind in nnkCallKinds: result.add y[0] + else: result.add y + for a in args: result.add a + if y.kind in nnkCallKinds: + for i in 1..<y.len: result.add y[i] + +macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): bool = + ## See top level documentation of his module of how ``scanf`` works. + template matchBind(parser) {.dirty.} = + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym(parser), input, results[i], idx)) + conds.add resLen.notZero + conds.add resLen + + var i = 0 + var p = 0 + var idx = genSym(nskVar, "idx") + var res = genSym(nskVar, "res") + result = newTree(nnkStmtListExpr, newVarStmt(idx, newLit 0), newVarStmt(res, newLit false)) + var conds = newTree(nnkStmtList) + var fullMatch = false + while p < pattern.len: + if pattern[p] == '$': + inc p + case pattern[p] + of '$': + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym"skip", input, newLit($pattern[p]), idx)) + conds.add resLen.notZero + conds.add resLen + of 'w': + if i < results.len or getType(results[i]).typeKind != ntyString: + matchBind "parseIdent" + else: + error("no string var given for $w") + inc i + of 'i': + if i < results.len or getType(results[i]).typeKind != ntyInt: + matchBind "parseInt" + else: + error("no int var given for $d") + inc i + of 'f': + if i < results.len or getType(results[i]).typeKind != ntyFloat: + matchBind "parseFloat" + else: + error("no float var given for $f") + inc i + of 's': + conds.add newCall(bindSym"inc", idx, newCall(bindSym"skipWhitespace", input, idx)) + conds.add newEmptyNode() + conds.add newEmptyNode() + of '.': + if p == pattern.len-1: + fullMatch = true + else: + error("invalid format string") + of '*', '+': + if i < results.len or getType(results[i]).typeKind != ntyString: + var min = ord(pattern[p] == '+') + var q=p+1 + var token = "" + while q < pattern.len and pattern[q] != '$': + token.add pattern[q] + inc q + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym"parseUntil", input, results[i], newLit(token), idx)) + conds.add newCall(bindSym"!=", resLen, newLit min) + conds.add resLen + else: + error("no string var given for $" & pattern[p]) + inc i + of '{': + inc p + var nesting = 0 + let start = p + while true: + case pattern[p] + of '{': inc nesting + of '}': + if nesting == 0: break + dec nesting + of '\0': error("expected closing '}'") + else: discard + inc p + let expr = pattern.substr(start, p-1) + if i < results.len: + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, buildUserCall(expr, input, results[i], idx)) + conds.add newCall(bindSym"!=", resLen, newLit 0) + conds.add resLen + else: + error("no var given for $" & expr) + inc i + of '[': + inc p + var nesting = 0 + let start = p + while true: + case pattern[p] + of '[': inc nesting + of ']': + if nesting == 0: break + dec nesting + of '\0': error("expected closing ']'") + else: discard + inc p + let expr = pattern.substr(start, p-1) + conds.add newCall(bindSym"inc", idx, buildUserCall(expr, input, idx)) + conds.add newEmptyNode() + conds.add newEmptyNode() + else: error("invalid format string") + inc p + else: + var token = "" + while p < pattern.len and pattern[p] != '$': + token.add pattern[p] + inc p + var resLen = genSym(nskLet, "resLen") + conds.add newLetStmt(resLen, newCall(bindSym"skip", input, newLit(token), idx)) + conds.add resLen.notZero + conds.add resLen + result.add conditionsToIfChain(conds, idx, res, 0) + if fullMatch: + result.add newCall(bindSym"and", res, + newCall(bindSym">=", idx, newCall(bindSym"len", input))) + else: + result.add res + +template atom*(input: string; idx: int; c: char): bool = + ## Used in scanp for the matching of atoms (usually chars). + input[idx] == c + +template atom*(input: string; idx: int; s: set[char]): bool = + input[idx] in s + +#template prepare*(input: string): int = 0 +template success*(x: int): bool = x != 0 + +template nxt*(input: string; idx, step: int = 1) = inc(idx, step) + +macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = + ## See top level documentation of his module of how ``scanp`` works. + type StmtTriple = tuple[init, cond, action: NimNode] + + template interf(x): untyped = bindSym(x, brForceOpen) + + proc toIfChain(n: seq[StmtTriple]; idx, res: NimNode; start: int): NimNode = + if start >= n.len: return newAssignment(res, newLit true) + var ifs: NimNode = nil + if n[start].cond.kind == nnkEmpty: + ifs = toIfChain(n, idx, res, start+1) + else: + ifs = newIfStmt((n[start].cond, + newTree(nnkStmtList, n[start].action, + toIfChain(n, idx, res, start+1)))) + result = newTree(nnkStmtList, n[start].init, ifs) + + proc attach(x, attached: NimNode): NimNode = + if attached == nil: x + else: newStmtList(attached, x) + + proc placeholder(n, x, j: NimNode): NimNode = + if n.kind == nnkPrefix and n[0].eqIdent("$"): + let n1 = n[1] + if n1.eqIdent"_" or n1.eqIdent"current": + result = newTree(nnkBracketExpr, x, j) + elif n1.eqIdent"input": + result = x + elif n1.eqIdent"i" or n1.eqIdent"index": + result = j + else: + error("unknown pattern " & repr(n)) + else: + result = copyNimNode(n) + for i in 0 ..< n.len: + result.add placeholder(n[i], x, j) + + proc atm(it, input, idx, attached: NimNode): StmtTriple = + template `!!`(x): untyped = attach(x, attached) + case it.kind + of nnkIdent: + var resLen = genSym(nskLet, "resLen") + result = (newLetStmt(resLen, newCall(it, input, idx)), + newCall(interf"success", resLen), + !!newCall(interf"nxt", input, idx, resLen)) + of nnkCallKinds: + # *{'A'..'Z'} !! s.add(!_) + template buildWhile(init, cond, action): untyped = + while true: + init + if not cond: break + action + + # (x) a # bind action a to (x) + if it[0].kind == nnkPar and it.len == 2: + result = atm(it[0], input, idx, placeholder(it[1], input, idx)) + elif it.kind == nnkInfix and it[0].eqIdent"->": + # bind matching to some action: + result = atm(it[1], input, idx, placeholder(it[2], input, idx)) + elif it.kind == nnkInfix and it[0].eqIdent"as": + let cond = if it[1].kind in nnkCallKinds: placeholder(it[1], input, idx) + else: newCall(it[1], input, idx) + result = (newLetStmt(it[2], cond), + newCall(interf"success", it[2]), + !!newCall(interf"nxt", input, idx, it[2])) + elif it.kind == nnkPrefix and it[0].eqIdent"*": + let (init, cond, action) = atm(it[1], input, idx, attached) + result = (getAst(buildWhile(init, cond, action)), + newEmptyNode(), newEmptyNode()) + elif it.kind == nnkPrefix and it[0].eqIdent"+": + # x+ is the same as xx* + result = atm(newTree(nnkPar, it[1], newTree(nnkPrefix, ident"*", it[1])), + input, idx, attached) + elif it.kind == nnkPrefix and it[0].eqIdent"?": + # optional. + let (init, cond, action) = atm(it[1], input, idx, attached) + if cond.kind == nnkEmpty: + error("'?' operator applied to a non-condition") + else: + result = (newTree(nnkStmtList, init, newIfStmt((cond, action))), + newEmptyNode(), newEmptyNode()) + elif it.kind == nnkPrefix and it[0].eqIdent"~": + # not operator + let (init, cond, action) = atm(it[1], input, idx, attached) + if cond.kind == nnkEmpty: + error("'~' operator applied to a non-condition") + else: + result = (init, newCall(bindSym"not", cond), action) + elif it.kind == nnkInfix and it[0].eqIdent"|": + let a = atm(it[1], input, idx, attached) + let b = atm(it[2], input, idx, attached) + if a.cond.kind == nnkEmpty or b.cond.kind == nnkEmpty: + error("'|' operator applied to a non-condition") + else: + result = (newStmtList(a.init, + newIfStmt((a.cond, a.action), (newTree(nnkStmtListExpr, b.init, b.cond), b.action))), + newEmptyNode(), newEmptyNode()) + elif it.kind == nnkInfix and it[0].eqIdent"^*": + # a ^* b is rewritten to: (a *(b a))? + #exprList = expr ^+ comma + template tmp(a, b): untyped = ?(a, *(b, a)) + result = atm(getAst(tmp(it[1], it[2])), input, idx, attached) + + elif it.kind == nnkInfix and it[0].eqIdent"^+": + # a ^* b is rewritten to: (a +(b a))? + template tmp(a, b): untyped = (a, *(b, a)) + result = atm(getAst(tmp(it[1], it[2])), input, idx, attached) + elif it.kind == nnkCommand and it.len == 2 and it[0].eqIdent"pred": + # enforce that the wrapped call is interpreted as a predicate, not a non-terminal: + result = (newEmptyNode(), placeholder(it[1], input, idx), newEmptyNode()) + else: + var resLen = genSym(nskLet, "resLen") + result = (newLetStmt(resLen, placeholder(it, input, idx)), + newCall(interf"success", resLen), !!newCall(interf"nxt", input, idx, resLen)) + of nnkStrLit..nnkTripleStrLit: + var resLen = genSym(nskLet, "resLen") + result = (newLetStmt(resLen, newCall(interf"skip", input, it, idx)), + newCall(interf"success", resLen), !!newCall(interf"nxt", input, idx, resLen)) + of nnkCurly, nnkAccQuoted, nnkCharLit: + result = (newEmptyNode(), newCall(interf"atom", input, idx, it), !!newCall(interf"nxt", input, idx)) + of nnkCurlyExpr: + if it.len == 3 and it[1].kind == nnkIntLit and it[2].kind == nnkIntLit: + var h = newTree(nnkPar, it[0]) + for count in 2..it[1].intVal: h.add(it[0]) + for count in it[1].intVal .. it[2].intVal-1: h.add(newTree(nnkPrefix, ident"?", it[0])) + result = atm(h, input, idx, attached) + elif it.len == 2 and it[1].kind == nnkIntLit: + var h = newTree(nnkPar, it[0]) + for count in 2..it[1].intVal: h.add(it[0]) + result = atm(h, input, idx, attached) + else: + error("invalid pattern") + of nnkPar: + if it.len == 1: + result = atm(it[0], input, idx, attached) + else: + # concatenation: + var conds: seq[StmtTriple] = @[] + for x in it: conds.add atm(x, input, idx, attached) + var res = genSym(nskVar, "res") + result = (newStmtList(newVarStmt(res, newLit false), + toIfChain(conds, idx, res, 0)), res, newEmptyNode()) + else: + error("invalid pattern") + + #var idx = genSym(nskVar, "idx") + var res = genSym(nskVar, "res") + result = newTree(nnkStmtListExpr, #newVarStmt(idx, newCall(interf"prepare", input)), + newVarStmt(res, newLit false)) + var conds: seq[StmtTriple] = @[] + for it in pattern: + conds.add atm(it, input, idx, nil) + result.add toIfChain(conds, idx, res, 0) + result.add res + when defined(debugScanp): + echo repr result + + +when isMainModule: + proc twoDigits(input: string; x: var int; start: int): int = + if input[start] == '0' and input[start+1] == '0': + result = 2 + x = 13 + else: + result = 0 + + proc someSep(input: string; start: int; seps: set[char] = {';',',','-','.'}): int = + result = 0 + while input[start+result] in seps: inc result + + proc demangle(s: string; res: var string; start: int): int = + while s[result+start] in {'_', '@'}: inc result + res = "" + while result+start < s.len and s[result+start] > ' ' and s[result+start] != '_': + res.add s[result+start] + inc result + while result+start < s.len and s[result+start] > ' ': + inc result + + proc parseGDB(resp: string): seq[string] = + const + digits = {'0'..'9'} + hexdigits = digits + {'a'..'f', 'A'..'F'} + whites = {' ', '\t', '\C', '\L'} + result = @[] + var idx = 0 + while true: + var prc = "" + var info = "" + if scanp(resp, idx, *`whites`, '#', *`digits`, +`whites`, ?("0x", *`hexdigits`, " in "), + demangle($input, prc, $index), *`whites`, '(', * ~ ')', ')', + *`whites`, "at ", +(~{'\C', '\L', '\0'} -> info.add($_)) ): + result.add prc & " " & info + else: + break + + var key, val: string + var intval: int + var floatval: float + doAssert scanf("abc:: xyz 89 33.25", "$w$s::$s$w$s$i $f", key, val, intval, floatVal) + doAssert key == "abc" + doAssert val == "xyz" + doAssert intval == 89 + doAssert floatVal == 33.25 + + let xx = scanf("$abc", "$$$i", intval) + doAssert xx == false + + + let xx2 = scanf("$1234", "$$$i", intval) + doAssert xx2 + + let yy = scanf(";.--Breakpoint00 [output]", "$[someSep]Breakpoint${twoDigits}$[someSep({';','.','-'})] [$+]$.", intVal, key) + doAssert yy + doAssert key == "output" + doAssert intVal == 13 + + var ident = "" + var idx = 0 + let zz = scanp("foobar x x x xWZ", idx, +{'a'..'z'} -> add(ident, $_), *(*{' ', '\t'}, "x"), ~'U', "Z") + doAssert zz + doAssert ident == "foobar" + + const digits = {'0'..'9'} + var year = 0 + var idx2 = 0 + if scanp("201655-8-9", idx2, `digits`{4,6} -> (year = year * 10 + ord($_) - ord('0')), "-8", "-9"): + doAssert year == 201655 + + const gdbOut = """ + #0 @foo_96013_1208911747@8 (x0=...) + at c:/users/anwender/projects/nim/temp.nim:11 + #1 0x00417754 in tempInit000 () at c:/users/anwender/projects/nim/temp.nim:13 + #2 0x0041768d in NimMainInner () + at c:/users/anwender/projects/nim/lib/system.nim:2605 + #3 0x004176b1 in NimMain () + at c:/users/anwender/projects/nim/lib/system.nim:2613 + #4 0x004176db in main (argc=1, args=0x712cc8, env=0x711ca8) + at c:/users/anwender/projects/nim/lib/system.nim:2620""" + const result = @["foo c:/users/anwender/projects/nim/temp.nim:11", + "tempInit000 c:/users/anwender/projects/nim/temp.nim:13", + "NimMainInner c:/users/anwender/projects/nim/lib/system.nim:2605", + "NimMain c:/users/anwender/projects/nim/lib/system.nim:2613", + "main c:/users/anwender/projects/nim/lib/system.nim:2620"] + doAssert parseGDB(gdbOut) == result diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index f2c1e77e1..bfc32bc71 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -14,6 +14,8 @@ ## <backends.html#the-javascript-target>`_. import parseutils +from math import pow, round, floor, log10 +from algorithm import reverse {.deadCodeElim: on.} @@ -24,6 +26,12 @@ include "system/inclrtl" {.pop.} +# Support old split with set[char] +when defined(nimOldSplit): + {.pragma: deprecatedSplit, deprecated.} +else: + {.pragma: deprecatedSplit.} + type CharSet* {.deprecated.} = set[char] # for compatibility with Nim {.deprecated: [TCharSet: CharSet].} @@ -62,8 +70,8 @@ const ## doAssert "01234".find(invalid) == -1 ## doAssert "01A34".find(invalid) == 2 -proc isAlpha*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaChar".}= +proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsAlphaAsciiChar".}= ## Checks whether or not `c` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -83,27 +91,27 @@ proc isDigit*(c: char): bool {.noSideEffect, procvar, ## This checks 0-9 ASCII characters only. return c in Digits -proc isSpace*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceChar".}= +proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsSpaceAsciiChar".}= ## Checks whether or not `c` is a whitespace character. return c in Whitespace -proc isLower*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerChar".}= +proc isLowerAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsLowerAsciiChar".}= ## Checks whether or not `c` is a lower case character. ## ## This checks ASCII characters only. return c in {'a'..'z'} -proc isUpper*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperChar".}= +proc isUpperAscii*(c: char): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsUpperAsciiChar".}= ## Checks whether or not `c` is an upper case character. ## ## This checks ASCII characters only. return c in {'A'..'Z'} -proc isAlpha*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaStr".}= +proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsAlphaAsciiStr".}= ## Checks whether or not `s` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -115,7 +123,7 @@ proc isAlpha*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isAlpha() and result + result = c.isAlphaAscii() and result proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsAlphaNumericStr".}= @@ -147,8 +155,8 @@ proc isDigit*(s: string): bool {.noSideEffect, procvar, for c in s: result = c.isDigit() and result -proc isSpace*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceStr".}= +proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsSpaceAsciiStr".}= ## Checks whether or not `s` is completely whitespace. ## ## Returns true if all characters in `s` are whitespace @@ -158,10 +166,11 @@ proc isSpace*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isSpace() and result + if not c.isSpaceAscii(): + return false -proc isLower*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerStr".}= +proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsLowerAsciiStr".}= ## Checks whether or not `s` contains all lower case characters. ## ## This checks ASCII characters only. @@ -172,10 +181,10 @@ proc isLower*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isLower() and result + result = c.isLowerAscii() and result -proc isUpper*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperStr".}= +proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nsuIsUpperAsciiStr".}= ## Checks whether or not `s` contains all upper case characters. ## ## This checks ASCII characters only. @@ -186,10 +195,10 @@ proc isUpper*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isUpper() and result + result = c.isUpperAscii() and result -proc toLower*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToLowerChar".} = +proc toLowerAscii*(c: char): char {.noSideEffect, procvar, + rtl, extern: "nsuToLowerAsciiChar".} = ## Converts `c` into lower case. ## ## This works only for the letters ``A-Z``. See `unicode.toLower @@ -200,8 +209,8 @@ proc toLower*(c: char): char {.noSideEffect, procvar, else: result = c -proc toLower*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToLowerStr".} = +proc toLowerAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuToLowerAsciiStr".} = ## Converts `s` into lower case. ## ## This works only for the letters ``A-Z``. See `unicode.toLower @@ -209,10 +218,10 @@ proc toLower*(s: string): string {.noSideEffect, procvar, ## character. result = newString(len(s)) for i in 0..len(s) - 1: - result[i] = toLower(s[i]) + result[i] = toLowerAscii(s[i]) -proc toUpper*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToUpperChar".} = +proc toUpperAscii*(c: char): char {.noSideEffect, procvar, + rtl, extern: "nsuToUpperAsciiChar".} = ## Converts `c` into upper case. ## ## This works only for the letters ``A-Z``. See `unicode.toUpper @@ -223,8 +232,8 @@ proc toUpper*(c: char): char {.noSideEffect, procvar, else: result = c -proc toUpper*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToUpperStr".} = +proc toUpperAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuToUpperAsciiStr".} = ## Converts `s` into upper case. ## ## This works only for the letters ``A-Z``. See `unicode.toUpper @@ -232,14 +241,145 @@ proc toUpper*(s: string): string {.noSideEffect, procvar, ## character. result = newString(len(s)) for i in 0..len(s) - 1: - result[i] = toUpper(s[i]) + result[i] = toUpperAscii(s[i]) + +proc capitalizeAscii*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nsuCapitalizeAscii".} = + ## Converts the first character of `s` into upper case. + ## + ## This works only for the letters ``A-Z``. + result = toUpperAscii(s[0]) & substr(s, 1) + +proc isSpace*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsSpaceChar".}= + ## Checks whether or not `c` is a whitespace character. + ## + ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. + isSpaceAscii(c) + +proc isLower*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsLowerChar".}= + ## Checks whether or not `c` is a lower case character. + ## + ## This checks ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. + isLowerAscii(c) + +proc isUpper*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsUpperChar".}= + ## Checks whether or not `c` is an upper case character. + ## + ## This checks ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. + isUpperAscii(c) + +proc isAlpha*(c: char): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsAlphaChar".}= + ## Checks whether or not `c` is alphabetical. + ## + ## This checks a-z, A-Z ASCII characters only. + ## + ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. + isAlphaAscii(c) + +proc isAlpha*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsAlphaStr".}= + ## Checks whether or not `s` is alphabetical. + ## + ## This checks a-z, A-Z ASCII characters only. + ## Returns true if all characters in `s` are + ## alphabetic and there is at least one character + ## in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isAlphaAscii`` instead. + isAlphaAscii(s) + +proc isSpace*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsSpaceStr".}= + ## Checks whether or not `s` is completely whitespace. + ## + ## Returns true if all characters in `s` are whitespace + ## characters and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isSpaceAscii`` instead. + isSpaceAscii(s) + +proc isLower*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsLowerStr".}= + ## Checks whether or not `s` contains all lower case characters. + ## + ## This checks ASCII characters only. + ## Returns true if all characters in `s` are lower case + ## and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isLowerAscii`` instead. + isLowerAscii(s) + +proc isUpper*(s: string): bool {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuIsUpperStr".}= + ## Checks whether or not `s` contains all upper case characters. + ## + ## This checks ASCII characters only. + ## Returns true if all characters in `s` are upper case + ## and there is at least one character in `s`. + ## + ## **Deprecated since version 0.15.0**: use ``isUpperAscii`` instead. + isUpperAscii(s) + +proc toLower*(c: char): char {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToLowerChar".} = + ## Converts `c` into lower case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toLower + ## <unicode.html#toLower>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. + toLowerAscii(c) + +proc toLower*(s: string): string {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToLowerStr".} = + ## Converts `s` into lower case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toLower + ## <unicode.html#toLower>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toLowerAscii`` instead. + toLowerAscii(s) + +proc toUpper*(c: char): char {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToUpperChar".} = + ## Converts `c` into upper case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toUpper + ## <unicode.html#toUpper>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. + toUpperAscii(c) + +proc toUpper*(s: string): string {.noSideEffect, procvar, + rtl, deprecated, extern: "nsuToUpperStr".} = + ## Converts `s` into upper case. + ## + ## This works only for the letters ``A-Z``. See `unicode.toUpper + ## <unicode.html#toUpper>`_ for a version that works for any Unicode + ## character. + ## + ## **Deprecated since version 0.15.0**: use ``toUpperAscii`` instead. + toUpperAscii(s) proc capitalize*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuCapitalize".} = + rtl, deprecated, extern: "nsuCapitalize".} = ## Converts the first character of `s` into upper case. ## ## This works only for the letters ``A-Z``. - result = toUpper(s[0]) & substr(s, 1) + ## + ## **Deprecated since version 0.15.0**: use ``capitalizeAscii`` instead. + capitalizeAscii(s) proc normalize*(s: string): string {.noSideEffect, procvar, rtl, extern: "nsuNormalize".} = @@ -268,7 +408,7 @@ proc cmpIgnoreCase*(a, b: string): int {.noSideEffect, var i = 0 var m = min(a.len, b.len) while i < m: - result = ord(toLower(a[i])) - ord(toLower(b[i])) + result = ord(toLowerAscii(a[i])) - ord(toLowerAscii(b[i])) if result != 0: return inc(i) result = a.len - b.len @@ -289,8 +429,8 @@ proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, while true: while a[i] == '_': inc(i) while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLower(a[i]) - var bb = toLower(b[j]) + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[j]) result = ord(aa) - ord(bb) if result != 0 or aa == '\0': break inc(i) @@ -324,16 +464,77 @@ proc toOctal*(c: char): string {.noSideEffect, rtl, extern: "nsuToOctal".} = result[i] = chr(val mod 8 + ord('0')) val = val div 8 -iterator split*(s: string, seps: set[char] = Whitespace): string = +proc isNilOrEmpty*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsNilOrEmpty".} = + ## Checks if `s` is nil or empty. + result = len(s) == 0 + +proc isNilOrWhitespace*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsNilOrWhitespace".} = + ## Checks if `s` is nil or consists entirely of whitespace characters. + if len(s) == 0: + return true + + result = true + for c in s: + if not c.isSpaceAscii(): + return false + +proc substrEq(s: string, pos: int, substr: string): bool = + var i = 0 + var length = substr.len + while i < length and s[pos+i] == substr[i]: + inc i + + return i == length + +# --------- Private templates for different split separators ----------- + +template stringHasSep(s: string, index: int, seps: set[char]): bool = + s[index] in seps + +template stringHasSep(s: string, index: int, sep: char): bool = + s[index] == sep + +template stringHasSep(s: string, index: int, sep: string): bool = + s.substrEq(index, sep) + +template splitCommon(s, sep, maxsplit, sepLen) = + ## Common code for split procedures + var last = 0 + var splits = maxsplit + + if len(s) > 0: + while last <= len(s): + var first = last + while last < len(s) and not stringHasSep(s, last, sep): + inc(last) + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + inc(last, sepLen) + +template oldSplit(s, seps, maxsplit) = + var last = 0 + var splits = maxsplit + assert(not ('\0' in seps)) + while last < len(s): + while s[last] in seps: inc(last) + var first = last + while last < len(s) and s[last] notin seps: inc(last) + if first <= last-1: + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + +iterator split*(s: string, seps: set[char] = Whitespace, + maxsplit: int = -1): string = ## Splits the string `s` into substrings using a group of separators. ## - ## Substrings are separated by a substring containing only `seps`. Note - ## that whole sequences of characters found in ``seps`` will be counted as - ## a single split point and leading/trailing separators will be ignored. - ## The following example: + ## Substrings are separated by a substring containing only `seps`. ## ## .. code-block:: nim - ## for word in split(" this is an example "): + ## for word in split("this\lis an\texample"): ## writeLine(stdout, word) ## ## ...generates this output: @@ -347,7 +548,7 @@ iterator split*(s: string, seps: set[char] = Whitespace): string = ## And the following code: ## ## .. code-block:: nim - ## for word in split(";;this;is;an;;example;;;", {';'}): + ## for word in split("this:is;an$example", {';', ':', '$'}): ## writeLine(stdout, word) ## ## ...produces the same output as the first example. The code: @@ -368,22 +569,26 @@ iterator split*(s: string, seps: set[char] = Whitespace): string = ## "08" ## "08.398990" ## - var last = 0 - assert(not ('\0' in seps)) - while last < len(s): - while s[last] in seps: inc(last) - var first = last - while last < len(s) and s[last] notin seps: inc(last) # BUGFIX! - if first <= last-1: - yield substr(s, first, last-1) + when defined(nimOldSplit): + oldSplit(s, seps, maxsplit) + else: + splitCommon(s, seps, maxsplit, 1) + +iterator splitWhitespace*(s: string): string = + ## Splits at whitespace. + oldSplit(s, Whitespace, -1) -iterator split*(s: string, sep: char): string = +proc splitWhitespace*(s: string): seq[string] {.noSideEffect, + rtl, extern: "nsuSplitWhitespace".} = + ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ + ## iterator, but is a proc that returns a sequence of substrings. + accumulateResult(splitWhitespace(s)) + +iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a single separator. ## ## Substrings are separated by the character `sep`. - ## Unlike the version of the iterator which accepts a set of separator - ## characters, this proc will not coalesce groups of the - ## separator, returning a string for each found character. The code: + ## The code: ## ## .. code-block:: nim ## for word in split(";;this;is;an;;example;;;", ';'): @@ -403,28 +608,118 @@ iterator split*(s: string, sep: char): string = ## "" ## "" ## - var last = 0 - assert('\0' != sep) - if len(s) > 0: - # `<=` is correct here for the edge cases! - while last <= len(s): - var first = last - while last < len(s) and s[last] != sep: inc(last) - yield substr(s, first, last-1) - inc(last) + splitCommon(s, sep, maxsplit, 1) -iterator split*(s: string, sep: string): string = +iterator split*(s: string, sep: string, maxsplit: int = -1): string = ## Splits the string `s` into substrings using a string separator. ## ## Substrings are separated by the string `sep`. - var last = 0 + ## The code: + ## + ## .. code-block:: nim + ## for word in split("thisDATAisDATAcorrupted", "DATA"): + ## writeLine(stdout, word) + ## + ## Results in: + ## + ## .. code-block:: + ## "this" + ## "is" + ## "corrupted" + ## + + splitCommon(s, sep, maxsplit, sep.len) + +template rsplitCommon(s, sep, maxsplit, sepLen) = + ## Common code for rsplit functions + var + last = s.len - 1 + first = last + splits = maxsplit + startPos = 0 + if len(s) > 0: - while last <= len(s): - var first = last - while last < len(s) and s.substr(last, last + <sep.len) != sep: - inc(last) - yield substr(s, first, last-1) - inc(last, sep.len) + # go to -1 in order to get separators at the beginning + while first >= -1: + while first >= 0 and not stringHasSep(s, first, sep): + dec(first) + + if splits == 0: + # No more splits means set first to the beginning + first = -1 + + if first == -1: + startPos = 0 + else: + startPos = first + sepLen + + yield substr(s, startPos, last) + + if splits == 0: + break + + dec(splits) + dec(first) + + last = first + +iterator rsplit*(s: string, seps: set[char] = Whitespace, + maxsplit: int = -1): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,char>`_ except in reverse order. + ## + ## .. code-block:: nim + ## for piece in "foo bar".rsplit(WhiteSpace): + ## echo piece + ## + ## Results in: + ## + ## .. code-block:: nim + ## "bar" + ## "foo" + ## + ## Substrings are separated from the right by the set of chars `seps` + + rsplitCommon(s, seps, maxsplit, 1) + +iterator rsplit*(s: string, sep: char, + maxsplit: int = -1): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,char>`_ except in reverse order. + ## + ## .. code-block:: nim + ## for piece in "foo:bar".rsplit(':'): + ## echo piece + ## + ## Results in: + ## + ## .. code-block:: nim + ## "bar" + ## "foo" + ## + ## Substrings are separated from the right by the char `sep` + rsplitCommon(s, sep, maxsplit, 1) + +iterator rsplit*(s: string, sep: string, maxsplit: int = -1, + keepSeparators: bool = false): string = + ## Splits the string `s` into substrings from the right using a + ## string separator. Works exactly the same as `split iterator + ## <#split.i,string,string>`_ except in reverse order. + ## + ## .. code-block:: nim + ## for piece in "foothebar".rsplit("the"): + ## echo piece + ## + ## Results in: + ## + ## .. code-block:: nim + ## "bar" + ## "foo" + ## + ## Substrings are separated from the right by the string `sep` + rsplitCommon(s, sep, maxsplit, sep.len) iterator splitLines*(s: string): string = ## Splits the string `s` into its containing lines. @@ -493,25 +788,92 @@ proc countLines*(s: string): int {.noSideEffect, else: discard inc i -proc split*(s: string, seps: set[char] = Whitespace): seq[string] {. +proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] {. noSideEffect, rtl, extern: "nsuSplitCharSet".} = ## The same as the `split iterator <#split.i,string,set[char]>`_, but is a ## proc that returns a sequence of substrings. - accumulateResult(split(s, seps)) + accumulateResult(split(s, seps, maxsplit)) -proc split*(s: string, sep: char): seq[string] {.noSideEffect, +proc split*(s: string, sep: char, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuSplitChar".} = ## The same as the `split iterator <#split.i,string,char>`_, but is a proc ## that returns a sequence of substrings. - accumulateResult(split(s, sep)) + accumulateResult(split(s, sep, maxsplit)) -proc split*(s: string, sep: string): seq[string] {.noSideEffect, +proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuSplitString".} = ## Splits the string `s` into substrings using a string separator. ## ## Substrings are separated by the string `sep`. This is a wrapper around the ## `split iterator <#split.i,string,string>`_. - accumulateResult(split(s, sep)) + accumulateResult(split(s, sep, maxsplit)) + +proc rsplit*(s: string, seps: set[char] = Whitespace, + maxsplit: int = -1): seq[string] + {.noSideEffect, rtl, extern: "nsuRSplitCharSet".} = + ## The same as the `rsplit iterator <#rsplit.i,string,set[char]>`_, but is a + ## proc that returns a sequence of substrings. + ## + ## A possible common use case for `rsplit` is path manipulation, + ## particularly on systems that don't use a common delimiter. + ## + ## For example, if a system had `#` as a delimiter, you could + ## do the following to get the tail of the path: + ## + ## .. code-block:: nim + ## var tailSplit = rsplit("Root#Object#Method#Index", {'#'}, maxsplit=1) + ## + ## Results in `tailSplit` containing: + ## + ## .. code-block:: nim + ## @["Root#Object#Method", "Index"] + ## + accumulateResult(rsplit(s, seps, maxsplit)) + result.reverse() + +proc rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] + {.noSideEffect, rtl, extern: "nsuRSplitChar".} = + ## The same as the `split iterator <#rsplit.i,string,char>`_, but is a proc + ## that returns a sequence of substrings. + ## + ## A possible common use case for `rsplit` is path manipulation, + ## particularly on systems that don't use a common delimiter. + ## + ## For example, if a system had `#` as a delimiter, you could + ## do the following to get the tail of the path: + ## + ## .. code-block:: nim + ## var tailSplit = rsplit("Root#Object#Method#Index", '#', maxsplit=1) + ## + ## Results in `tailSplit` containing: + ## + ## .. code-block:: nim + ## @["Root#Object#Method", "Index"] + ## + accumulateResult(rsplit(s, sep, maxsplit)) + result.reverse() + +proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] + {.noSideEffect, rtl, extern: "nsuRSplitString".} = + ## The same as the `split iterator <#rsplit.i,string,string>`_, but is a proc + ## that returns a sequence of substrings. + ## + ## A possible common use case for `rsplit` is path manipulation, + ## particularly on systems that don't use a common delimiter. + ## + ## For example, if a system had `#` as a delimiter, you could + ## do the following to get the tail of the path: + ## + ## .. code-block:: nim + ## var tailSplit = rsplit("Root#Object#Method#Index", "#", maxsplit=1) + ## + ## Results in `tailSplit` containing: + ## + ## .. code-block:: nim + ## @["Root#Object#Method", "Index"] + ## + accumulateResult(rsplit(s, sep, maxsplit)) + result.reverse() proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect, rtl, extern: "nsuToHex".} = @@ -530,6 +892,10 @@ proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect, # handle negative overflow if n == 0 and x < 0: n = -1 +proc toHex*[T](x: T): string = + ## Shortcut for ``toHex(x, T.sizeOf * 2)`` + toHex(x, T.sizeOf * 2) + proc intToStr*(x: int, minchars: Positive = 1): string {.noSideEffect, rtl, extern: "nsuIntToStr".} = ## Converts `x` to its decimal representation. @@ -560,6 +926,24 @@ proc parseBiggestInt*(s: string): BiggestInt {.noSideEffect, procvar, if L != s.len or L == 0: raise newException(ValueError, "invalid integer: " & s) +proc parseUInt*(s: string): uint {.noSideEffect, procvar, + rtl, extern: "nsuParseUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + var L = parseutils.parseUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + +proc parseBiggestUInt*(s: string): uint64 {.noSideEffect, procvar, + rtl, extern: "nsuParseBiggestUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + var L = parseutils.parseBiggestUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + proc parseFloat*(s: string): float {.noSideEffect, procvar, rtl, extern: "nsuParseFloat".} = ## Parses a decimal floating point value contained in `s`. If `s` is not @@ -651,7 +1035,7 @@ proc repeat*(s: string, n: Natural): string {.noSideEffect, result = newStringOfCap(n * s.len) for i in 1..n: result.add(s) -template spaces*(n: Natural): string = repeat(' ',n) +template spaces*(n: Natural): string = repeat(' ', n) ## Returns a String with `n` space characters. You can use this proc ## to left align strings. Example: ## @@ -766,8 +1150,7 @@ proc indent*(s: string, count: Natural, padding: string = " "): string {.noSideEffect, rtl, extern: "nsuIndent".} = ## Indents each line in ``s`` by ``count`` amount of ``padding``. ## - ## **Note:** This currently does not preserve the specific new line characters - ## used. + ## **Note:** This does not preserve the new line characters used in ``s``. result = "" var i = 0 for line in s.splitLines(): @@ -778,32 +1161,39 @@ proc indent*(s: string, count: Natural, padding: string = " "): string result.add(line) i.inc -proc unindent*(s: string, eatAllIndent = false): string {. - noSideEffect, rtl, extern: "nsuUnindent".} = - ## Unindents `s`. - result = newStringOfCap(s.len) +proc unindent*(s: string, count: Natural, padding: string = " "): string + {.noSideEffect, rtl, extern: "nsuUnindent".} = + ## Unindents each line in ``s`` by ``count`` amount of ``padding``. + ## + ## **Note:** This does not preserve the new line characters used in ``s``. + result = "" var i = 0 - var pattern = true - var indent = 0 - while s[i] == ' ': inc i - var level = if i == 0: -1 else: i - while i < s.len: - if s[i] == ' ': - if i > 0 and s[i-1] in {'\l', '\c'}: - pattern = true - indent = 0 - if pattern: - inc(indent) - if indent > level and not eatAllIndent: - result.add(s[i]) - if level < 0: level = indent - else: - # a space somewhere: do not delete - result.add(s[i]) - else: - pattern = false - result.add(s[i]) - inc i + for line in s.splitLines(): + if i != 0: + result.add("\n") + var indentCount = 0 + for j in 0..<count.int: + indentCount.inc + if line[j .. j + <padding.len] != padding: + indentCount = j + break + result.add(line[indentCount*padding.len .. ^1]) + i.inc + +proc unindent*(s: string): string + {.noSideEffect, rtl, extern: "nsuUnindentAll".} = + ## Removes all indentation composed of whitespace from each line in ``s``. + ## + ## For example: + ## + ## .. code-block:: nim + ## const x = """ + ## Hello + ## There + ## """.unindent() + ## + ## doAssert x == "Hello\nThere\n" + unindent(s, 1000) # TODO: Passing a 1000 is a bit hackish. proc startsWith*(s, prefix: string): bool {.noSideEffect, rtl, extern: "nsuStartsWith".} = @@ -816,6 +1206,10 @@ proc startsWith*(s, prefix: string): bool {.noSideEffect, if s[i] != prefix[i]: return false inc(i) +proc startsWith*(s: string, prefix: char): bool {.noSideEffect, inline.} = + ## Returns true iff ``s`` starts with ``prefix``. + result = s[0] == prefix + proc endsWith*(s, suffix: string): bool {.noSideEffect, rtl, extern: "nsuEndsWith".} = ## Returns true iff ``s`` ends with ``suffix``. @@ -828,6 +1222,10 @@ proc endsWith*(s, suffix: string): bool {.noSideEffect, inc(i) if suffix[i] == '\0': return true +proc endsWith*(s: string, suffix: char): bool {.noSideEffect, inline.} = + ## Returns true iff ``s`` ends with ``suffix``. + result = s[s.high] == suffix + proc continuesWith*(s, substr: string, start: Natural): bool {.noSideEffect, rtl, extern: "nsuContinuesWith".} = ## Returns true iff ``s`` continues with ``substr`` at position ``start``. @@ -981,6 +1379,34 @@ proc rfind*(s: string, sub: char, start: int = -1): int {.noSideEffect, if sub == s[i]: return i return -1 +proc center*(s: string, width: int, fillChar: char = ' '): string {. + noSideEffect, rtl, extern: "nsuCenterString".} = + ## Return the contents of `s` centered in a string `width` long using + ## `fillChar` as padding. + ## + ## The original string is returned if `width` is less than or equal + ## to `s.len`. + if width <= s.len: + return s + + result = newString(width) + + # Left padding will be one fillChar + # smaller if there are an odd number + # of characters + let + charsLeft = (width - s.len) + leftPadding = charsLeft div 2 + + for i in 0 ..< width: + if i >= leftPadding and i < leftPadding + s.len: + # we are where the string should be located + result[i] = s[i-leftPadding] + else: + # we are either before or after where + # the string s should go + result[i] = fillChar + proc count*(s: string, sub: string, overlapping: bool = false): int {. noSideEffect, rtl, extern: "nsuCountString".} = ## Count the occurrences of a substring `sub` in the string `s`. @@ -1420,28 +1846,216 @@ proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, ## after the decimal point for Nim's ``float`` type. result = formatBiggestFloat(f, format, precision, decimalSep) -proc formatSize*(bytes: BiggestInt, decimalSep = '.'): string = - ## Rounds and formats `bytes`. Examples: +proc trimZeros*(x: var string) {.noSideEffect.} = + ## Trim trailing zeros from a formatted floating point + ## value (`x`). Modifies the passed value. + var spl: seq[string] + if x.contains('.') or x.contains(','): + if x.contains('e'): + spl= x.split('e') + x = spl[0] + while x[x.high] == '0': + x.setLen(x.len-1) + if x[x.high] in [',', '.']: + x.setLen(x.len-1) + if spl.len > 0: + x &= "e" & spl[1] + +type + BinaryPrefixMode* = enum ## the different names for binary prefixes + bpIEC, # use the IEC/ISO standard prefixes such as kibi + bpColloquial # use the colloquial kilo, mega etc + +proc formatSize*(bytes: int64, + decimalSep = '.', + prefix = bpIEC, + includeSpace = false): string {.noSideEffect.} = + ## Rounds and formats `bytes`. + ## + ## By default, uses the IEC/ISO standard binary prefixes, so 1024 will be + ## formatted as 1KiB. Set prefix to `bpColloquial` to use the colloquial + ## names from the SI standard (e.g. k for 1000 being reused as 1024). + ## + ## `includeSpace` can be set to true to include the (SI preferred) space + ## between the number and the unit (e.g. 1 KiB). + ## + ## Examples: ## ## .. code-block:: nim ## - ## formatSize(1'i64 shl 31 + 300'i64) == "2.204GB" - ## formatSize(4096) == "4KB" - ## - template frmt(a, b, c: expr): expr = - let bs = $b - insertSep($a) & decimalSep & bs.substr(0, 2) & c - let gigabytes = bytes shr 30 - let megabytes = bytes shr 20 - let kilobytes = bytes shr 10 - if gigabytes != 0: - result = frmt(gigabytes, megabytes, "GB") - elif megabytes != 0: - result = frmt(megabytes, kilobytes, "MB") - elif kilobytes != 0: - result = frmt(kilobytes, bytes, "KB") + ## formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" + ## formatSize((2.234*1024*1024).int) == "2.234MiB" + ## formatSize(4096, includeSpace=true) == "4 KiB" + ## formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" + ## formatSize(4096) == "4KiB" + ## formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" + ## + const iecPrefixes = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"] + const collPrefixes = ["", "k", "M", "G", "T", "P", "E", "Z", "Y"] + var + xb: int64 = bytes + fbytes: float + last_xb: int64 = bytes + matchedIndex: int + prefixes: array[9, string] + if prefix == bpColloquial: + prefixes = collPrefixes else: - result = insertSep($bytes) & "B" + prefixes = iecPrefixes + + # Iterate through prefixes seeing if value will be greater than + # 0 in each case + for index in 1..<prefixes.len: + last_xb = xb + xb = bytes div (1'i64 shl (index*10)) + matchedIndex = index + if xb == 0: + xb = last_xb + matchedIndex = index - 1 + break + # xb has the integer number for the latest value; index should be correct + fbytes = bytes.float / (1'i64 shl (matchedIndex*10)).float + result = formatFloat(fbytes, format=ffDecimal, precision=3, decimalSep=decimalSep) + result.trimZeros() + if includeSpace: + result &= " " + result &= prefixes[matchedIndex] + result &= "B" + +proc formatEng*(f: BiggestFloat, + precision: range[0..32] = 10, + trim: bool = true, + siPrefix: bool = false, + unit: string = nil, + decimalSep = '.'): string {.noSideEffect.} = + ## Converts a floating point value `f` to a string using engineering notation. + ## + ## Numbers in of the range -1000.0<f<1000.0 will be formatted without an + ## exponent. Numbers outside of this range will be formatted as a + ## significand in the range -1000.0<f<1000.0 and an exponent that will always + ## be an integer multiple of 3, corresponding with the SI prefix scale k, M, + ## G, T etc for numbers with an absolute value greater than 1 and m, μ, n, p + ## etc for numbers with an absolute value less than 1. + ## + ## The default configuration (`trim=true` and `precision=10`) shows the + ## **shortest** form that precisely (up to a maximum of 10 decimal places) + ## displays the value. For example, 4.100000 will be displayed as 4.1 (which + ## is mathematically identical) whereas 4.1000003 will be displayed as + ## 4.1000003. + ## + ## If `trim` is set to true, trailing zeros will be removed; if false, the + ## number of digits specified by `precision` will always be shown. + ## + ## `precision` can be used to set the number of digits to be shown after the + ## decimal point or (if `trim` is true) the maximum number of digits to be + ## shown. + ## + ## .. code-block:: nim + ## + ## formatEng(0, 2, trim=false) == "0.00" + ## formatEng(0, 2) == "0" + ## formatEng(0.053, 0) == "53e-3" + ## formatEng(52731234, 2) == "52.73e6" + ## formatEng(-52731234, 2) == "-52.73e6" + ## + ## If `siPrefix` is set to true, the number will be displayed with the SI + ## prefix corresponding to the exponent. For example 4100 will be displayed + ## as "4.1 k" instead of "4.1e3". Note that `u` is used for micro- in place + ## of the greek letter mu (μ) as per ISO 2955. Numbers with an absolute + ## value outside of the range 1e-18<f<1000e18 (1a<f<1000E) will be displayed + ## with an exponent rather than an SI prefix, regardless of whether + ## `siPrefix` is true. + ## + ## If `unit` is not nil, the provided unit will be appended to the string + ## (with a space as required by the SI standard). This behaviour is slightly + ## different to appending the unit to the result as the location of the space + ## is altered depending on whether there is an exponent. + ## + ## .. code-block:: nim + ## + ## formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" + ## formatEng(4.1, siPrefix=true, unit="V") == "4.1 V" + ## formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space + ## formatEng(4100, siPrefix=true) == "4.1 k" + ## formatEng(4.1, siPrefix=true, unit="") == "4.1 " # Space with unit="" + ## formatEng(4100, siPrefix=true, unit="") == "4.1 k" + ## formatEng(4100) == "4.1e3" + ## formatEng(4100, unit="V") == "4.1e3 V" + ## formatEng(4100, unit="") == "4.1e3 " # Space with unit="" + ## + ## `decimalSep` is used as the decimal separator + var + absolute: BiggestFloat + significand: BiggestFloat + fexponent: BiggestFloat + exponent: int + splitResult: seq[string] + suffix: string = "" + proc getPrefix(exp: int): char = + ## Get the SI prefix for a given exponent + ## + ## Assumes exponent is a multiple of 3; returns ' ' if no prefix found + const siPrefixes = ['a','f','p','n','u','m',' ','k','M','G','T','P','E'] + var index: int = (exp div 3) + 6 + result = ' ' + if index in low(siPrefixes)..high(siPrefixes): + result = siPrefixes[index] + + # Most of the work is done with the sign ignored, so get the absolute value + absolute = abs(f) + significand = f + + if absolute == 0.0: + # Simple case: just format it and force the exponent to 0 + exponent = 0 + result = significand.formatBiggestFloat(ffDecimal, precision, decimalSep='.') + else: + # Find the best exponent that's a multiple of 3 + fexponent = round(floor(log10(absolute))) + fexponent = 3.0 * round(floor(fexponent / 3.0)) + # Adjust the significand for the new exponent + significand /= pow(10.0, fexponent) + + # Round the significand and check whether it has affected + # the exponent + significand = round(significand, precision) + absolute = abs(significand) + if absolute >= 1000.0: + significand *= 0.001 + fexponent += 3 + # Components of the result: + result = significand.formatBiggestFloat(ffDecimal, precision, decimalSep='.') + exponent = fexponent.int() + + splitResult = result.split('.') + result = splitResult[0] + # result should have at most one decimal character + if splitResult.len() > 1: + # If trim is set, we get rid of trailing zeros. Don't use trimZeros here as + # we can be a bit more efficient through knowledge that there will never be + # an exponent in this part. + if trim: + while splitResult[1].endsWith("0"): + # Trim last character + splitResult[1].setLen(splitResult[1].len-1) + if splitResult[1].len() > 0: + result &= decimalSep & splitResult[1] + else: + result &= decimalSep & splitResult[1] + + # Combine the results accordingly + if siPrefix and exponent != 0: + var p = getPrefix(exponent) + if p != ' ': + suffix = " " & p + exponent = 0 # Exponent replaced by SI prefix + if suffix == "" and unit != nil: + suffix = " " + if unit != nil: + suffix &= unit + if exponent != 0: + result &= "e" & $exponent + result &= suffix proc findNormalized(x: string, inArray: openArray[string]): int = var i = 0 @@ -1637,9 +2251,14 @@ when isMainModule: ["1,0e-11", "1,0e-011"] doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" - when not defined(testing): - echo formatSize(1'i64 shl 31 + 300'i64) # == "4,GB" - echo formatSize(1'i64 shl 31) + + block: # formatSize tests + doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" + doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" + doAssert formatSize(4096) == "4KiB" + doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" + doAssert formatSize(4096, includeSpace=true) == "4 KiB" + doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" doAssert "$animal eats $food." % ["animal", "The cat", "food", "fish"] == "The cat eats fish." @@ -1652,6 +2271,11 @@ when isMainModule: doAssert parseEnum("invalid enum value", enC) == enC + doAssert center("foo", 13) == " foo " + doAssert center("foo", 0) == "foo" + doAssert center("foo", 3, fillChar = 'a') == "foo" + doAssert center("foo", 10, fillChar = '\t') == "\t\t\tfoo\t\t\t\t" + doAssert count("foofoofoo", "foofoo") == 1 doAssert count("foofoofoo", "foofoo", overlapping = true) == 2 doAssert count("foofoofoo", 'f') == 3 @@ -1668,13 +2292,13 @@ when isMainModule: doAssert " foo\n bar".indent(4, "Q") == "QQQQ foo\nQQQQ bar" - doAssert isAlpha('r') - doAssert isAlpha('A') - doAssert(not isAlpha('$')) + doAssert isAlphaAscii('r') + doAssert isAlphaAscii('A') + doAssert(not isAlphaAscii('$')) - doAssert isAlpha("Rasp") - doAssert isAlpha("Args") - doAssert(not isAlpha("$Tomato")) + doAssert isAlphaAscii("Rasp") + doAssert isAlphaAscii("Args") + doAssert(not isAlphaAscii("$Tomato")) doAssert isAlphaNumeric('3') doAssert isAlphaNumeric('R') @@ -1693,35 +2317,131 @@ when isMainModule: doAssert(not isDigit("12.33")) doAssert(not isDigit("A45b")) - doAssert isSpace('\t') - doAssert isSpace('\l') - doAssert(not isSpace('A')) - - doAssert isSpace("\t\l \v\r\f") - doAssert isSpace(" ") - doAssert(not isSpace("ABc \td")) + doAssert isSpaceAscii('\t') + doAssert isSpaceAscii('\l') + doAssert(not isSpaceAscii('A')) + + doAssert isSpaceAscii("\t\l \v\r\f") + doAssert isSpaceAscii(" ") + doAssert(not isSpaceAscii("ABc \td")) + + doAssert(isNilOrEmpty("")) + doAssert(isNilOrEmpty(nil)) + doAssert(not isNilOrEmpty("test")) + doAssert(not isNilOrEmpty(" ")) + + doAssert(isNilOrWhitespace("")) + doAssert(isNilOrWhitespace(nil)) + doAssert(isNilOrWhitespace(" ")) + doAssert(isNilOrWhitespace("\t\l \v\r\f")) + doAssert(not isNilOrWhitespace("ABc \td")) + + doAssert isLowerAscii('a') + doAssert isLowerAscii('z') + doAssert(not isLowerAscii('A')) + doAssert(not isLowerAscii('5')) + doAssert(not isLowerAscii('&')) + + doAssert isLowerAscii("abcd") + doAssert(not isLowerAscii("abCD")) + doAssert(not isLowerAscii("33aa")) + + doAssert isUpperAscii('A') + doAssert(not isUpperAscii('b')) + doAssert(not isUpperAscii('5')) + doAssert(not isUpperAscii('%')) + + doAssert isUpperAscii("ABC") + doAssert(not isUpperAscii("AAcc")) + doAssert(not isUpperAscii("A#$")) + + doAssert rsplit("foo bar", seps=Whitespace) == @["foo", "bar"] + doAssert rsplit(" foo bar", seps=Whitespace, maxsplit=1) == @[" foo", "bar"] + doAssert rsplit(" foo bar ", seps=Whitespace, maxsplit=1) == @[" foo bar", ""] + doAssert rsplit(":foo:bar", sep=':') == @["", "foo", "bar"] + doAssert rsplit(":foo:bar", sep=':', maxsplit=2) == @["", "foo", "bar"] + doAssert rsplit(":foo:bar", sep=':', maxsplit=3) == @["", "foo", "bar"] + doAssert rsplit("foothebar", sep="the") == @["foo", "bar"] - doAssert isLower('a') - doAssert isLower('z') - doAssert(not isLower('A')) - doAssert(not isLower('5')) - doAssert(not isLower('&')) - - doAssert isLower("abcd") - doAssert(not isLower("abCD")) - doAssert(not isLower("33aa")) - - doAssert isUpper('A') - doAssert(not isUpper('b')) - doAssert(not isUpper('5')) - doAssert(not isUpper('%')) - - doAssert isUpper("ABC") - doAssert(not isUpper("AAcc")) - doAssert(not isUpper("A#$")) doAssert(unescape(r"\x013", "", "") == "\x013") doAssert join(["foo", "bar", "baz"]) == "foobarbaz" doAssert join(@["foo", "bar", "baz"], ", ") == "foo, bar, baz" doAssert join([1, 2, 3]) == "123" doAssert join(@[1, 2, 3], ", ") == "1, 2, 3" + + doAssert """~~!!foo +~~!!bar +~~!!baz""".unindent(2, "~~!!") == "foo\nbar\nbaz" + + doAssert """~~!!foo +~~!!bar +~~!!baz""".unindent(2, "~~!!aa") == "~~!!foo\n~~!!bar\n~~!!baz" + doAssert """~~foo +~~ bar +~~ baz""".unindent(4, "~") == "foo\n bar\n baz" + doAssert """foo +bar + baz + """.unindent(4) == "foo\nbar\nbaz\n" + doAssert """foo + bar + baz + """.unindent(2) == "foo\n bar\n baz\n" + doAssert """foo + bar + baz + """.unindent(100) == "foo\nbar\nbaz\n" + + doAssert """foo + foo + bar + """.unindent() == "foo\nfoo\nbar\n" + + let s = " this is an example " + let s2 = ":this;is;an:example;;" + + doAssert s.split() == @["", "this", "is", "an", "example", "", ""] + doAssert s2.split(seps={':', ';'}) == @["", "this", "is", "an", "example", "", ""] + doAssert s.split(maxsplit=4) == @["", "this", "is", "an", "example "] + doAssert s.split(' ', maxsplit=1) == @["", "this is an example "] + doAssert s.split(" ", maxsplit=4) == @["", "this", "is", "an", "example "] + + block: # formatEng tests + doAssert formatEng(0, 2, trim=false) == "0.00" + doAssert formatEng(0, 2) == "0" + doAssert formatEng(53, 2, trim=false) == "53.00" + doAssert formatEng(0.053, 2, trim=false) == "53.00e-3" + doAssert formatEng(0.053, 4, trim=false) == "53.0000e-3" + doAssert formatEng(0.053, 4, trim=true) == "53e-3" + doAssert formatEng(0.053, 0) == "53e-3" + doAssert formatEng(52731234) == "52.731234e6" + doAssert formatEng(-52731234) == "-52.731234e6" + doAssert formatEng(52731234, 1) == "52.7e6" + doAssert formatEng(-52731234, 1) == "-52.7e6" + doAssert formatEng(52731234, 1, decimalSep=',') == "52,7e6" + doAssert formatEng(-52731234, 1, decimalSep=',') == "-52,7e6" + + doAssert formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" + doAssert formatEng(4.1, siPrefix=true, unit="V") == "4.1 V" + doAssert formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space + doAssert formatEng(4100, siPrefix=true) == "4.1 k" + doAssert formatEng(4.1, siPrefix=true, unit="") == "4.1 " # Includes space + doAssert formatEng(4100, siPrefix=true, unit="") == "4.1 k" + doAssert formatEng(4100) == "4.1e3" + doAssert formatEng(4100, unit="V") == "4.1e3 V" + doAssert formatEng(4100, unit="") == "4.1e3 " # Space with unit="" + # Don't use SI prefix as number is too big + doAssert formatEng(3.1e22, siPrefix=true, unit="a") == "31e21 a" + # Don't use SI prefix as number is too small + doAssert formatEng(3.1e-25, siPrefix=true, unit="A") == "310e-27 A" + + block: # startsWith / endsWith char tests + var s = "abcdef" + doAssert s.startsWith('a') + doAssert s.startsWith('b') == false + doAssert s.endsWith('f') + doAssert s.endsWith('a') == false + doAssert s.endsWith('\0') == false + + #echo("strutils tests passed") diff --git a/lib/pure/subexes.nim b/lib/pure/subexes.nim index 5824ace81..351b3c086 100644 --- a/lib/pure/subexes.nim +++ b/lib/pure/subexes.nim @@ -9,7 +9,7 @@ ## Nim support for `substitution expressions`:idx: (`subex`:idx:). ## -## .. include:: ../doc/subexes.txt +## .. include:: ../../doc/subexes.txt ## {.push debugger:off .} # the user does not want to trace a part @@ -390,11 +390,11 @@ when isMainModule: doAssert(("type MyEnum* = enum\n $', '2i'\n '{..}" % ["fieldA", "fieldB", "FiledClkad", "fieldD", "fieldE", "longishFieldName"]).replace(" \n", "\n") == - strutils.unindent """ + strutils.unindent(""" type MyEnum* = enum fieldA, fieldB, FiledClkad, fieldD, - fieldE, longishFieldName""") + fieldE, longishFieldName""", 6)) doAssert subex"$1($', '{2..})" % ["f", "a", "b", "c"] == "f(a, b, c)" @@ -404,8 +404,8 @@ when isMainModule: doAssert((subex("type\n Enum = enum\n $', '40c'\n '{..}") % [ "fieldNameA", "fieldNameB", "fieldNameC", "fieldNameD"]).replace(" \n", "\n") == - strutils.unindent """ + strutils.unindent(""" type Enum = enum fieldNameA, fieldNameB, fieldNameC, - fieldNameD""") + fieldNameD""", 6)) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 60f064e7c..1f34ec07e 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -384,7 +384,7 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright=false) = var old = getAttributes(h) and not 0x0007 if bright: old = old or FOREGROUND_INTENSITY - const lookup: array [ForegroundColor, int] = [ + const lookup: array[ForegroundColor, int] = [ 0, (FOREGROUND_RED), (FOREGROUND_GREEN), @@ -406,7 +406,7 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright=false) = var old = getAttributes(h) and not 0x0070 if bright: old = old or BACKGROUND_INTENSITY - const lookup: array [BackgroundColor, int] = [ + const lookup: array[BackgroundColor, int] = [ 0, (BACKGROUND_RED), (BACKGROUND_GREEN), @@ -493,17 +493,21 @@ template styledEcho*(args: varargs[expr]): expr = ## Echoes styles arguments to stdout using ``styledWriteLine``. callStyledEcho(args) -when defined(nimdoc): - proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. - ## The character is not printed to the terminal. This is not available for - ## Windows. - discard -elif not defined(windows): - proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. - ## The character is not printed to the terminal. This is not available for - ## Windows. +proc getch*(): char = + ## Read a single character from the terminal, blocking until it is entered. + ## The character is not printed to the terminal. + when defined(windows): + let fd = getStdHandle(STD_INPUT_HANDLE) + var keyEvent = KEY_EVENT_RECORD() + var numRead: cint + while true: + # Block until character is entered + doAssert(waitForSingleObject(fd, INFINITE) == WAIT_OBJECT_0) + doAssert(readConsoleInput(fd, addr(keyEvent), 1, addr(numRead)) != 0) + if numRead == 0 or keyEvent.eventType != 1 or keyEvent.bKeyDown == 0: + continue + return char(keyEvent.uChar) + else: let fd = getFileHandle(stdin) var oldMode: Termios discard fd.tcgetattr(addr oldMode) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 29ae52d47..d6eb29e1c 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -64,8 +64,9 @@ when defined(posix) and not defined(JS): proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {. importc: "gettimeofday", header: "<sys/time.h>".} + when not defined(freebsd) and not defined(netbsd) and not defined(openbsd): + var timezone {.importc, header: "<time.h>".}: int var - timezone {.importc, header: "<time.h>".}: int tzname {.importc, header: "<time.h>" .}: array[0..1, cstring] # we also need tzset() to make sure that tzname is initialized proc tzset() {.importc, header: "<time.h>".} @@ -94,7 +95,8 @@ elif defined(windows): elif defined(JS): type - Time* {.importc.} = object + Time* = ref TimeObj + TimeObj {.importc.} = object getDay: proc (): int {.tags: [], raises: [], benign.} getFullYear: proc (): int {.tags: [], raises: [], benign.} getHours: proc (): int {.tags: [], raises: [], benign.} @@ -182,7 +184,16 @@ proc getGMTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} ## converts the calendar time `t` to broken-down time representation, ## expressed in Coordinated Universal Time (UTC). -proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} +proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign, deprecated.} + ## converts a broken-down time structure to + ## calendar time representation. The function ignores the specified + ## contents of the structure members `weekday` and `yearday` and recomputes + ## them from the other information in the broken-down time structure. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTime`` instead. + +proc toTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} ## converts a broken-down time structure to ## calendar time representation. The function ignores the specified ## contents of the structure members `weekday` and `yearday` and recomputes @@ -356,16 +367,19 @@ proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## ## **Note:** This has been only briefly tested and it may not be ## very accurate. - let t = toSeconds(timeInfoToTime(a)) + let t = toSeconds(toTime(a)) let secs = toSeconds(a, interval) - result = getLocalTime(fromSeconds(t + secs)) + if a.tzname == "UTC": + result = getGMTime(fromSeconds(t + secs)) + else: + result = getLocalTime(fromSeconds(t + secs)) proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## subtracts ``interval`` time from TimeInfo ``a``. ## ## **Note:** This has been only briefly tested, it is inaccurate especially ## when you subtract so much that you reach the Julian calendar. - let t = toSeconds(timeInfoToTime(a)) + let t = toSeconds(toTime(a)) var intval: TimeInterval intval.milliseconds = - interval.milliseconds intval.seconds = - interval.seconds @@ -375,7 +389,10 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = intval.months = - interval.months intval.years = - interval.years let secs = toSeconds(a, intval) - result = getLocalTime(fromSeconds(t + secs)) + if a.tzname == "UTC": + result = getGMTime(fromSeconds(t + secs)) + else: + result = getLocalTime(fromSeconds(t + secs)) proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds @@ -407,18 +424,32 @@ when not defined(JS): when not defined(JS): # C wrapper: + when defined(freebsd) or defined(netbsd) or defined(openbsd): + type + StructTM {.importc: "struct tm", final.} = object + second {.importc: "tm_sec".}, + minute {.importc: "tm_min".}, + hour {.importc: "tm_hour".}, + monthday {.importc: "tm_mday".}, + month {.importc: "tm_mon".}, + year {.importc: "tm_year".}, + weekday {.importc: "tm_wday".}, + yearday {.importc: "tm_yday".}, + isdst {.importc: "tm_isdst".}: cint + gmtoff {.importc: "tm_gmtoff".}: clong + else: + type + StructTM {.importc: "struct tm", final.} = object + second {.importc: "tm_sec".}, + minute {.importc: "tm_min".}, + hour {.importc: "tm_hour".}, + monthday {.importc: "tm_mday".}, + month {.importc: "tm_mon".}, + year {.importc: "tm_year".}, + weekday {.importc: "tm_wday".}, + yearday {.importc: "tm_yday".}, + isdst {.importc: "tm_isdst".}: cint type - StructTM {.importc: "struct tm", final.} = object - second {.importc: "tm_sec".}, - minute {.importc: "tm_min".}, - hour {.importc: "tm_hour".}, - monthday {.importc: "tm_mday".}, - month {.importc: "tm_mon".}, - year {.importc: "tm_year".}, - weekday {.importc: "tm_wday".}, - yearday {.importc: "tm_yday".}, - isdst {.importc: "tm_isdst".}: cint - TimeInfoPtr = ptr StructTM Clock {.importc: "clock_t".} = distinct int @@ -446,30 +477,53 @@ when not defined(JS): # our own procs on top of that: proc tmToTimeInfo(tm: StructTM, local: bool): TimeInfo = const - weekDays: array [0..6, WeekDay] = [ + weekDays: array[0..6, WeekDay] = [ dSun, dMon, dTue, dWed, dThu, dFri, dSat] - TimeInfo(second: int(tm.second), - minute: int(tm.minute), - hour: int(tm.hour), - monthday: int(tm.monthday), - month: Month(tm.month), - year: tm.year + 1900'i32, - weekday: weekDays[int(tm.weekday)], - yearday: int(tm.yearday), - isDST: tm.isdst > 0, - tzname: if local: - if tm.isdst > 0: - getTzname().DST + when defined(freebsd) or defined(netbsd) or defined(openbsd): + TimeInfo(second: int(tm.second), + minute: int(tm.minute), + hour: int(tm.hour), + monthday: int(tm.monthday), + month: Month(tm.month), + year: tm.year + 1900'i32, + weekday: weekDays[int(tm.weekday)], + yearday: int(tm.yearday), + isDST: tm.isdst > 0, + tzname: if local: + if tm.isdst > 0: + getTzname().DST + else: + getTzname().nonDST + else: + "UTC", + # BSD stores in `gmtoff` offset east of UTC in seconds, + # but posix systems using west of UTC in seconds + timezone: if local: -(tm.gmtoff) else: 0 + ) + else: + TimeInfo(second: int(tm.second), + minute: int(tm.minute), + hour: int(tm.hour), + monthday: int(tm.monthday), + month: Month(tm.month), + year: tm.year + 1900'i32, + weekday: weekDays[int(tm.weekday)], + yearday: int(tm.yearday), + isDST: tm.isdst > 0, + tzname: if local: + if tm.isdst > 0: + getTzname().DST + else: + getTzname().nonDST else: - getTzname().nonDST - else: - "UTC", - timezone: if local: getTimezone() else: 0 - ) + "UTC", + timezone: if local: getTimezone() else: 0 + ) + proc timeInfoToTM(t: TimeInfo): StructTM = const - weekDays: array [WeekDay, int8] = [1'i8,2'i8,3'i8,4'i8,5'i8,6'i8,0'i8] + weekDays: array[WeekDay, int8] = [1'i8,2'i8,3'i8,4'i8,5'i8,6'i8,0'i8] result.second = t.second result.minute = t.minute result.hour = t.hour @@ -517,6 +571,11 @@ when not defined(JS): # because the header of mktime is broken in my version of libc return mktime(timeInfoToTM(cTimeInfo)) + proc toTime(timeInfo: TimeInfo): Time = + var cTimeInfo = timeInfo # for C++ we have to make a copy, + # because the header of mktime is broken in my version of libc + return mktime(timeInfoToTM(cTimeInfo)) + proc toStringTillNL(p: cstring): string = result = "" var i = 0 @@ -550,7 +609,14 @@ when not defined(JS): return ($tzname[0], $tzname[1]) proc getTimezone(): int = - return timezone + when defined(freebsd) or defined(netbsd) or defined(openbsd): + var a = timec(nil) + let lt = localtime(addr(a)) + # BSD stores in `gmtoff` offset east of UTC in seconds, + # but posix systems using west of UTC in seconds + return -(lt.gmtoff) + else: + return timezone proc fromSeconds(since1970: float): Time = Time(since1970) @@ -586,7 +652,7 @@ elif defined(JS): return newDate() const - weekDays: array [0..6, WeekDay] = [ + weekDays: array[0..6, WeekDay] = [ dSun, dMon, dTue, dWed, dThu, dFri, dSat] proc getLocalTime(t: Time): TimeInfo = @@ -618,7 +684,16 @@ elif defined(JS): result.setFullYear(timeInfo.year) result.setDate(timeInfo.monthday) - proc `$`(timeInfo: TimeInfo): string = return $(timeInfoToTime(timeInfo)) + proc toTime*(timeInfo: TimeInfo): Time = + result = internGetTime() + result.setSeconds(timeInfo.second) + result.setMinutes(timeInfo.minute) + result.setHours(timeInfo.hour) + result.setMonth(ord(timeInfo.month)) + result.setFullYear(timeInfo.year) + result.setDate(timeInfo.monthday) + + proc `$`(timeInfo: TimeInfo): string = return $(toTime(timeInfo)) proc `$`(time: Time): string = return $time.toLocaleString() proc `-` (a, b: Time): int64 = @@ -708,24 +783,24 @@ proc years*(y: int): TimeInterval {.inline.} = proc `+=`*(t: var Time, ti: TimeInterval) = ## modifies `t` by adding the interval `ti` - t = timeInfoToTime(getLocalTime(t) + ti) + t = toTime(getLocalTime(t) + ti) proc `+`*(t: Time, ti: TimeInterval): Time = ## adds the interval `ti` to Time `t` ## by converting to localTime, adding the interval, and converting back ## ## ``echo getTime() + 1.day`` - result = timeInfoToTime(getLocalTime(t) + ti) + result = toTime(getLocalTime(t) + ti) proc `-=`*(t: var Time, ti: TimeInterval) = ## modifies `t` by subtracting the interval `ti` - t = timeInfoToTime(getLocalTime(t) - ti) + t = toTime(getLocalTime(t) - ti) proc `-`*(t: Time, ti: TimeInterval): Time = ## adds the interval `ti` to Time `t` ## ## ``echo getTime() - 1.day`` - result = timeInfoToTime(getLocalTime(t) - ti) + result = toTime(getLocalTime(t) - ti) proc formatToken(info: TimeInfo, token: string, buf: var string) = ## Helper of the format proc to parse individual tokens. @@ -923,21 +998,14 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = info.monthday = value[j..j+1].parseInt() j += 2 of "ddd": - case value[j..j+2].toLower() - of "sun": - info.weekday = dSun - of "mon": - info.weekday = dMon - of "tue": - info.weekday = dTue - of "wed": - info.weekday = dWed - of "thu": - info.weekday = dThu - of "fri": - info.weekday = dFri - of "sat": - info.weekday = dSat + case value[j..j+2].toLowerAscii() + of "sun": info.weekday = dSun + of "mon": info.weekday = dMon + of "tue": info.weekday = dTue + of "wed": info.weekday = dWed + of "thu": info.weekday = dThu + of "fri": info.weekday = dFri + of "sat": info.weekday = dSat else: raise newException(ValueError, "Couldn't parse day of week (ddd), got: " & value[j..j+2]) @@ -991,31 +1059,19 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = j += 2 info.month = Month(month-1) of "MMM": - case value[j..j+2].toLower(): - of "jan": - info.month = mJan - of "feb": - info.month = mFeb - of "mar": - info.month = mMar - of "apr": - info.month = mApr - of "may": - info.month = mMay - of "jun": - info.month = mJun - of "jul": - info.month = mJul - of "aug": - info.month = mAug - of "sep": - info.month = mSep - of "oct": - info.month = mOct - of "nov": - info.month = mNov - of "dec": - info.month = mDec + case value[j..j+2].toLowerAscii(): + of "jan": info.month = mJan + of "feb": info.month = mFeb + of "mar": info.month = mMar + of "apr": info.month = mApr + of "may": info.month = mMay + of "jun": info.month = mJun + of "jul": info.month = mJul + of "aug": info.month = mAug + of "sep": info.month = mSep + of "oct": info.month = mOct + of "nov": info.month = mNov + of "dec": info.month = mDec else: raise newException(ValueError, "Couldn't parse month (MMM), got: " & value) @@ -1112,7 +1168,7 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = "Couldn't parse timezone offset (zzz), got: " & value[j]) j += 6 of "ZZZ": - info.tzname = value[j..j+2].toUpper() + info.tzname = value[j..j+2].toUpperAscii() j += 3 else: # Ignore the token and move forward in the value string by the same length @@ -1193,7 +1249,7 @@ proc parse*(value, layout: string): TimeInfo = parseToken(info, token, value, j) token = "" # Reset weekday as it might not have been provided and the default may be wrong - info.weekday = getLocalTime(timeInfoToTime(info)).weekday + info.weekday = getLocalTime(toTime(info)).weekday return info # Leap year calculations are adapted from: @@ -1204,7 +1260,7 @@ proc parse*(value, layout: string): TimeInfo = proc countLeapYears*(yearSpan: int): int = ## Returns the number of leap years spanned by a given number of years. ## - ## Note: for leap years, start date is assumed to be 1 AD. + ## **Note:** For leap years, start date is assumed to be 1 AD. ## counts the number of leap years up to January 1st of a given year. ## Keep in mind that if specified year is a leap year, the leap day ## has not happened before January 1st of that year. @@ -1239,13 +1295,14 @@ proc getDayOfWeek*(day, month, year: int): WeekDay = y = year - a m = month + (12*a) - 2 d = (day + y + (y div 4) - (y div 100) + (y div 400) + (31*m) div 12) mod 7 - # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. so we must correct - # for the WeekDay type. + # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. + # so we must correct for the WeekDay type. if d == 0: return dSun result = (d-1).WeekDay proc getDayOfWeekJulian*(day, month, year: int): WeekDay = - ## Returns the day of the week enum from day, month and year, according to the Julian calendar. + ## Returns the day of the week enum from day, month and year, + ## according to the Julian calendar. # Day & month start from one. let a = (14 - month) div 12 @@ -1254,8 +1311,11 @@ proc getDayOfWeekJulian*(day, month, year: int): WeekDay = d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 result = d.WeekDay -proc timeToTimeInfo*(t: Time): TimeInfo = +proc timeToTimeInfo*(t: Time): TimeInfo {.deprecated.} = ## Converts a Time to TimeInfo. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``getLocalTime`` or ``getGMTime`` instead. let secs = t.toSeconds().int daysSinceEpoch = secs div secondsInDay @@ -1286,34 +1346,21 @@ proc timeToTimeInfo*(t: Time): TimeInfo = s = daySeconds mod secondsInMin result = TimeInfo(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) -proc timeToTimeInterval*(t: Time): TimeInterval = +proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = ## Converts a Time to a TimeInterval. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTimeInterval`` instead. + # Milliseconds not available from Time + var tInfo = t.getLocalTime() + initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) - let - secs = t.toSeconds().int - daysSinceEpoch = secs div secondsInDay - (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch) - daySeconds = secs mod secondsInDay - - result.years = yearsSinceEpoch + epochStartYear - - var - mon = mJan - days = daysRemaining - daysInMonth = getDaysInMonth(mon, result.years) - - # calculate month and day remainder - while days > daysInMonth and mon <= mDec: - days -= daysInMonth - mon.inc - daysInMonth = getDaysInMonth(mon, result.years) - - result.months = mon.int + 1 # month is 1 indexed int - result.days = days - result.hours = daySeconds div secondsInHour + 1 - result.minutes = (daySeconds mod secondsInHour) div secondsInMin - result.seconds = daySeconds mod secondsInMin +proc toTimeInterval*(t: Time): TimeInterval = + ## Converts a Time to a TimeInterval. # Milliseconds not available from Time + var tInfo = t.getLocalTime() + initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) + when isMainModule: # this is testing non-exported function diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 45f52eb7f..0cbe8de7a 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -14,7 +14,7 @@ include "system/inclrtl" type - RuneImpl = int # underlying type of Rune + RuneImpl = int32 # underlying type of Rune Rune* = distinct RuneImpl ## type that can hold any Unicode character Rune16* = distinct int16 ## 16 bit Unicode character @@ -135,45 +135,62 @@ proc runeAt*(s: string, i: Natural): Rune = ## Returns the unicode character in ``s`` at byte index ``i`` fastRuneAt(s, i, result, false) -proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} = - ## Converts a rune into its UTF-8 representation +template fastToUTF8Copy*(c: Rune, s: var string, pos: int, doInc = true) = + ## Copies UTF-8 representation of `c` into the preallocated string `s` + ## starting at position `pos`. If `doInc == true`, `pos` is incremented + ## by the number of bytes that have been processed. + ## + ## To be the most efficient, make sure `s` is preallocated + ## with an additional amount equal to the byte length of + ## `c`. var i = RuneImpl(c) if i <=% 127: - result = newString(1) - result[0] = chr(i) + s.setLen(pos+1) + s[pos+0] = chr(i) + when doInc: inc(pos) elif i <=% 0x07FF: - result = newString(2) - result[0] = chr((i shr 6) or 0b110_00000) - result[1] = chr((i and ones(6)) or 0b10_0000_00) + s.setLen(pos+2) + s[pos+0] = chr((i shr 6) or 0b110_00000) + s[pos+1] = chr((i and ones(6)) or 0b10_0000_00) + when doInc: inc(pos, 2) elif i <=% 0xFFFF: - result = newString(3) - result[0] = chr(i shr 12 or 0b1110_0000) - result[1] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[2] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+3) + s[pos+0] = chr(i shr 12 or 0b1110_0000) + s[pos+1] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 3) elif i <=% 0x001FFFFF: - result = newString(4) - result[0] = chr(i shr 18 or 0b1111_0000) - result[1] = chr(i shr 12 and ones(6) or 0b10_0000_00) - result[2] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[3] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+4) + s[pos+0] = chr(i shr 18 or 0b1111_0000) + s[pos+1] = chr(i shr 12 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+3] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 4) elif i <=% 0x03FFFFFF: - result = newString(5) - result[0] = chr(i shr 24 or 0b111110_00) - result[1] = chr(i shr 18 and ones(6) or 0b10_0000_00) - result[2] = chr(i shr 12 and ones(6) or 0b10_0000_00) - result[3] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[4] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+5) + s[pos+0] = chr(i shr 24 or 0b111110_00) + s[pos+1] = chr(i shr 18 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i shr 12 and ones(6) or 0b10_0000_00) + s[pos+3] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+4] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 5) elif i <=% 0x7FFFFFFF: - result = newString(6) - result[0] = chr(i shr 30 or 0b1111110_0) - result[1] = chr(i shr 24 and ones(6) or 0b10_0000_00) - result[2] = chr(i shr 18 and ones(6) or 0b10_0000_00) - result[3] = chr(i shr 12 and ones(6) or 0b10_0000_00) - result[4] = chr(i shr 6 and ones(6) or 0b10_0000_00) - result[5] = chr(i and ones(6) or 0b10_0000_00) + s.setLen(pos+6) + s[pos+0] = chr(i shr 30 or 0b1111110_0) + s[pos+1] = chr(i shr 24 and ones(6) or 0b10_0000_00) + s[pos+2] = chr(i shr 18 and ones(6) or 0b10_0000_00) + s[pos+3] = chr(i shr 12 and ones(6) or 0b10_0000_00) + s[pos+4] = chr(i shr 6 and ones(6) or 0b10_0000_00) + s[pos+5] = chr(i and ones(6) or 0b10_0000_00) + when doInc: inc(pos, 6) else: discard # error, exception? +proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} = + ## Converts a rune into its UTF-8 representation + result = "" + fastToUTF8Copy(c, result, 0, false) + proc `$`*(rune: Rune): string = ## Converts a Rune to a string rune.toUTF8 @@ -183,6 +200,105 @@ proc `$`*(runes: seq[Rune]): string = result = "" for rune in runes: result.add(rune.toUTF8) +proc runeOffset*(s: string, pos:Natural, start: Natural = 0): int = + ## Returns the byte position of unicode character + ## at position pos in s with an optional start byte position. + ## returns the special value -1 if it runs out of the string + ## + ## Beware: This can lead to unoptimized code and slow execution! + ## Most problems are solve more efficient by using an iterator + ## or conversion to a seq of Rune. + var + i = 0 + o = start + while i < pos: + o += runeLenAt(s, o) + if o >= s.len: + return -1 + inc i + return o + +proc runeAtPos*(s: string, pos: int): Rune = + ## Returns the unicode character at position pos + ## + ## Beware: This can lead to unoptimized code and slow execution! + ## Most problems are solve more efficient by using an iterator + ## or conversion to a seq of Rune. + fastRuneAt(s, runeOffset(s, pos), result, false) + +proc runeStrAtPos*(s: string, pos: Natural): string = + ## Returns the unicode character at position pos as UTF8 String + ## + ## Beware: This can lead to unoptimized code and slow execution! + ## Most problems are solve more efficient by using an iterator + ## or conversion to a seq of Rune. + let o = runeOffset(s, pos) + s[o.. (o+runeLenAt(s, o)-1)] + +proc runeReverseOffset*(s: string, rev:Positive): (int, int) = + ## Returns a tuple with the the byte offset of the + ## unicode character at position ``rev`` in s counting + ## from the end (starting with 1) and the total + ## number of runes in the string. Returns a negative value + ## for offset if there are to few runes in the string to + ## satisfy the request. + ## + ## Beware: This can lead to unoptimized code and slow execution! + ## Most problems are solve more efficient by using an iterator + ## or conversion to a seq of Rune. + var + a = rev.int + o = 0 + x = 0 + while o < s.len: + let r = runeLenAt(s, o) + o += r + if a < 0: + x += r + dec a + + if a > 0: + return (-a, rev.int-a) + return (x, -a+rev.int) + +proc runeSubStr*(s: string, pos:int, len:int = int.high): string = + ## Returns the UTF-8 substring starting at codepoint pos + ## with len codepoints. If pos or len is negativ they count from + ## the end of the string. If len is not given it means the longest + ## possible string. + ## + ## (Needs some examples) + if pos < 0: + let (o, rl) = runeReverseOffset(s, -pos) + if len >= rl: + result = s[o.. s.len-1] + elif len < 0: + let e = rl + len + if e < 0: + result = "" + else: + result = s[o.. runeOffset(s, e-(rl+pos) , o)-1] + else: + result = s[o.. runeOffset(s, len, o)-1] + else: + let o = runeOffset(s, pos) + if o < 0: + result = "" + elif len == int.high: + result = s[o.. s.len-1] + elif len < 0: + let (e, rl) = runeReverseOffset(s, -len) + discard rl + if e <= 0: + result = "" + else: + result = s[o.. e-1] + else: + var e = runeOffset(s, len, o) + if e < 0: + e = s.len + result = s[o.. e-1] + const alphaRanges = [ 0x00d8, 0x00f6, # - @@ -1148,7 +1264,7 @@ const 0x01f1, 501, # 0x01f3, 499] # -proc binarySearch(c: RuneImpl, tab: openArray[RuneImpl], len, stride: int): int = +proc binarySearch(c: RuneImpl, tab: openArray[int], len, stride: int): int = var n = len var t = 0 while n > 1: @@ -1253,8 +1369,210 @@ proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = (c >= 0x20d0 and c <= 0x20ff) or (c >= 0xfe20 and c <= 0xfe2f)) +template runeCheck(s, runeProc) = + ## Common code for rune.isLower, rune.isUpper, etc + result = if len(s) == 0: false else: true + + var + i = 0 + rune: Rune + + while i < len(s) and result: + fastRuneAt(s, i, rune, doInc=true) + result = runeProc(rune) and result + +proc isUpper*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all upper case unicode characters. + runeCheck(s, isUpper) + +proc isLower*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all lower case unicode characters. + runeCheck(s, isLower) + +proc isAlpha*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all alphabetic unicode characters. + runeCheck(s, isAlpha) + +proc isSpace*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Returns true iff `s` contains all whitespace unicode characters. + runeCheck(s, isWhiteSpace) + +template convertRune(s, runeProc) = + ## Convert runes in `s` using `runeProc` as the converter. + result = newString(len(s)) + + var + i = 0 + lastIndex = 0 + rune: Rune + + while i < len(s): + lastIndex = i + fastRuneAt(s, i, rune, doInc=true) + rune = runeProc(rune) + + rune.fastToUTF8Copy(result, lastIndex) + +proc toUpper*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Converts `s` into upper-case unicode characters. + convertRune(s, toUpper) + +proc toLower*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".} = + ## Converts `s` into lower-case unicode characters. + convertRune(s, toLower) + +proc swapCase*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1".} = + ## Swaps the case of unicode characters in `s` + ## + ## Returns a new string such that the cases of all unicode characters + ## are swapped if possible + + var + i = 0 + lastIndex = 0 + rune: Rune + + result = newString(len(s)) + + while i < len(s): + lastIndex = i + + fastRuneAt(s, i, rune) + + if rune.isUpper(): + rune = rune.toLower() + elif rune.isLower(): + rune = rune.toUpper() + + rune.fastToUTF8Copy(result, lastIndex) + +proc capitalize*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1".} = + ## Converts the first character of `s` into an upper-case unicode character. + if len(s) == 0: + return s + + var + rune: Rune + i = 0 + + fastRuneAt(s, i, rune, doInc=true) + + result = $toUpper(rune) & substr(s, i) + +proc translate*(s: string, replacements: proc(key: string): string): string {. + rtl, extern: "nuc$1".} = + ## Translates words in a string using the `replacements` proc to substitute + ## words inside `s` with their replacements + ## + ## `replacements` is any proc that takes a word and returns + ## a new word to fill it's place. + + # Allocate memory for the new string based on the old one. + # If the new string length is less than the old, no allocations + # will be needed. If the new string length is greater than the + # old, then maybe only one allocation is needed + result = newStringOfCap(s.len) + + var + index = 0 + lastIndex = 0 + wordStart = 0 + inWord = false + rune: Rune + + while index < len(s): + lastIndex = index + + fastRuneAt(s, index, rune) + + let whiteSpace = rune.isWhiteSpace() + + if whiteSpace and inWord: + # If we've reached the end of a word + let word = s[wordStart ..< lastIndex] + result.add(replacements(word)) + result.add($rune) + + inWord = false + elif not whiteSpace and not inWord: + # If we've hit a non space character and + # are not currently in a word, track + # the starting index of the word + inWord = true + wordStart = lastIndex + elif whiteSpace: + result.add($rune) + + if wordStart < len(s) and inWord: + # Get the trailing word at the end + let word = s[wordStart .. ^1] + result.add(replacements(word)) + +proc title*(s: string): string {.noSideEffect, procvar, + rtl, extern: "nuc$1".} = + ## Converts `s` to a unicode title. + ## + ## Returns a new string such that the first character + ## in each word inside `s` is capitalized + + var + i = 0 + lastIndex = 0 + rune: Rune + + result = newString(len(s)) + + var firstRune = true + + while i < len(s): + lastIndex = i + + fastRuneAt(s, i, rune) + + if not rune.isWhiteSpace() and firstRune: + rune = rune.toUpper() + firstRune = false + elif rune.isWhiteSpace(): + firstRune = true + + rune.fastToUTF8Copy(result, lastIndex) + +proc isTitle*(s: string): bool {.noSideEffect, procvar, + rtl, extern: "nuc$1Str".}= + ## Checks whether or not `s` is a unicode title. + ## + ## Returns true if the first character in each word inside `s` + ## are upper case and there is at least one character in `s`. + if s.len() == 0: + return false + + result = true + + var + i = 0 + rune: Rune + + var firstRune = true + + while i < len(s) and result: + fastRuneAt(s, i, rune, doInc=true) + + if not rune.isWhiteSpace() and firstRune: + result = rune.isUpper() and result + firstRune = false + elif rune.isWhiteSpace(): + firstRune = true + iterator runes*(s: string): Rune = - ## Iterates over any unicode character of the string ``s`` + ## Iterates over any unicode character of the string ``s`` returning runes var i = 0 result: Rune @@ -1262,6 +1580,14 @@ iterator runes*(s: string): Rune = fastRuneAt(s, i, result, true) yield result +iterator utf8*(s: string): string = + ## Iterates over any unicode character of the string ``s`` returning utf8 values + var o = 0 + while o < s.len: + let n = runeLenAt(s, o) + yield s[o.. (o+n-1)] + o += n + proc toRunes*(s: string): seq[Rune] = ## Obtains a sequence containing the Runes in ``s`` result = newSeq[Rune]() @@ -1352,6 +1678,101 @@ when isMainModule: compared = (someString == $someRunes) doAssert compared == true + proc test_replacements(word: string): string = + case word + of "two": + return "2" + of "foo": + return "BAR" + of "βeta": + return "beta" + of "alpha": + return "αlpha" + else: + return "12345" + + doAssert translate("two not alpha foo βeta", test_replacements) == "2 12345 αlpha BAR beta" + doAssert translate(" two not foo βeta ", test_replacements) == " 2 12345 BAR beta " + + doAssert title("foo bar") == "Foo Bar" + doAssert title("αlpha βeta γamma") == "Αlpha Βeta Γamma" + doAssert title("") == "" + + doAssert capitalize("βeta") == "Βeta" + doAssert capitalize("foo") == "Foo" + doAssert capitalize("") == "" + + doAssert isTitle("Foo") + doAssert(not isTitle("Foo bar")) + doAssert(not isTitle("αlpha Βeta")) + doAssert(isTitle("Αlpha Βeta Γamma")) + doAssert(not isTitle("fFoo")) + + doAssert swapCase("FooBar") == "fOObAR" + doAssert swapCase(" ") == " " + doAssert swapCase("Αlpha Βeta Γamma") == "αLPHA βETA γAMMA" + doAssert swapCase("a✓B") == "A✓b" + doAssert swapCase("") == "" + + doAssert isAlpha("r") + doAssert isAlpha("α") + doAssert(not isAlpha("$")) + doAssert(not isAlpha("")) + + doAssert isAlpha("Βeta") + doAssert isAlpha("Args") + doAssert(not isAlpha("$Foo✓")) + + doAssert isSpace("\t") + doAssert isSpace("\l") + doAssert(not isSpace("Β")) + doAssert(not isSpace("Βeta")) + + doAssert isSpace("\t\l \v\r\f") + doAssert isSpace(" ") + doAssert(not isSpace("")) + doAssert(not isSpace("ΑΓc \td")) + + doAssert isLower("a") + doAssert isLower("γ") + doAssert(not isLower("Γ")) + doAssert(not isLower("4")) + doAssert(not isLower("")) + + doAssert isLower("abcdγ") + doAssert(not isLower("abCDΓ")) + doAssert(not isLower("33aaΓ")) + + doAssert isUpper("Γ") + doAssert(not isUpper("b")) + doAssert(not isUpper("α")) + doAssert(not isUpper("✓")) + doAssert(not isUpper("")) + + doAssert isUpper("ΑΒΓ") + doAssert(not isUpper("AAccβ")) + doAssert(not isUpper("A#$β")) + + doAssert toUpper("Γ") == "Γ" + doAssert toUpper("b") == "B" + doAssert toUpper("α") == "Α" + doAssert toUpper("✓") == "✓" + doAssert toUpper("") == "" + + doAssert toUpper("ΑΒΓ") == "ΑΒΓ" + doAssert toUpper("AAccβ") == "AACCΒ" + doAssert toUpper("A✓$β") == "A✓$Β" + + doAssert toLower("a") == "a" + doAssert toLower("γ") == "γ" + doAssert toLower("Γ") == "γ" + doAssert toLower("4") == "4" + doAssert toLower("") == "" + + doAssert toLower("abcdγ") == "abcdγ" + doAssert toLower("abCDΓ") == "abcdγ" + doAssert toLower("33aaΓ") == "33aaγ" + doAssert reversed("Reverse this!") == "!siht esreveR" doAssert reversed("先秦兩漢") == "漢兩秦先" doAssert reversed("as⃝df̅") == "f̅ds⃝a" @@ -1360,3 +1781,44 @@ when isMainModule: const test = "as⃝" doAssert lastRune(test, test.len-1)[1] == 3 doAssert graphemeLen("è", 0) == 2 + + # test for rune positioning and runeSubStr() + let s = "Hänsel ««: 10,00€" + + var t = "" + for c in s.utf8: + t.add c + + doAssert(s == t) + + doAssert(runeReverseOffset(s, 1) == (20, 18)) + doAssert(runeReverseOffset(s, 19) == (-1, 18)) + + doAssert(runeStrAtPos(s, 0) == "H") + doAssert(runeSubStr(s, 0, 1) == "H") + doAssert(runeStrAtPos(s, 10) == ":") + doAssert(runeSubStr(s, 10, 1) == ":") + doAssert(runeStrAtPos(s, 9) == "«") + doAssert(runeSubStr(s, 9, 1) == "«") + doAssert(runeStrAtPos(s, 17) == "€") + doAssert(runeSubStr(s, 17, 1) == "€") + # echo runeStrAtPos(s, 18) # index error + + doAssert(runeSubStr(s, 0) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, -18) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, 10) == ": 10,00€") + doAssert(runeSubStr(s, 18) == "") + doAssert(runeSubStr(s, 0, 10) == "Hänsel ««") + + doAssert(runeSubStr(s, 12) == "10,00€") + doAssert(runeSubStr(s, -6) == "10,00€") + + doAssert(runeSubStr(s, 12, 5) == "10,00") + doAssert(runeSubStr(s, 12, -1) == "10,00") + doAssert(runeSubStr(s, -6, 5) == "10,00") + doAssert(runeSubStr(s, -6, -1) == "10,00") + + doAssert(runeSubStr(s, 0, 100) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, -100, 100) == "Hänsel ««: 10,00€") + doAssert(runeSubStr(s, 0, -100) == "") + doAssert(runeSubStr(s, 100, -100) == "") diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index aca9d51e2..92ddc3e75 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -41,7 +41,11 @@ when not defined(ECMAScript): import terminal type - TestStatus* = enum OK, FAILED ## The status of a test when it is done. + TestStatus* = enum ## The status of a test when it is done. + OK, + FAILED, + SKIPPED + OutputLevel* = enum ## The output verbosity of the tests. PRINT_ALL, ## Print as much as possible. PRINT_FAILURES, ## Print only the failed tests. @@ -73,7 +77,7 @@ checkpoints = @[] proc shouldRun(testName: string): bool = result = true -template suite*(name: expr, body: stmt): stmt {.immediate, dirty.} = +template suite*(name, body) {.dirty.} = ## Declare a test suite identified by `name` with optional ``setup`` ## and/or ``teardown`` section. ## @@ -102,13 +106,13 @@ template suite*(name: expr, body: stmt): stmt {.immediate, dirty.} = ## [OK] 2 + 2 = 4 ## [OK] (2 + -2) != 4 block: - template setup(setupBody: stmt): stmt {.immediate, dirty.} = + template setup(setupBody: untyped) {.dirty.} = var testSetupIMPLFlag = true - template testSetupIMPL: stmt {.immediate, dirty.} = setupBody + template testSetupIMPL: untyped {.dirty.} = setupBody - template teardown(teardownBody: stmt): stmt {.immediate, dirty.} = + template teardown(teardownBody: untyped) {.dirty.} = var testTeardownIMPLFlag = true - template testTeardownIMPL: stmt {.immediate, dirty.} = teardownBody + template testTeardownIMPL: untyped {.dirty.} = teardownBody body @@ -120,14 +124,18 @@ proc testDone(name: string, s: TestStatus) = template rawPrint() = echo("[", $s, "] ", name) when not defined(ECMAScript): if colorOutput and not defined(ECMAScript): - var color = (if s == OK: fgGreen else: fgRed) + var color = case s + of OK: fgGreen + of FAILED: fgRed + of SKIPPED: fgYellow + else: fgWhite styledEcho styleBright, color, "[", $s, "] ", fgWhite, name else: rawPrint() else: rawPrint() -template test*(name: expr, body: stmt): stmt {.immediate, dirty.} = +template test*(name, body) {.dirty.} = ## Define a single test case identified by `name`. ## ## .. code-block:: nim @@ -203,7 +211,22 @@ template fail* = checkpoints = @[] -macro check*(conditions: stmt): stmt {.immediate.} = +template skip* = + ## Makes test to be skipped. Should be used directly + ## in case when it is not possible to perform test + ## for reasons depending on outer environment, + ## or certain application logic conditions or configurations. + ## + ## .. code-block:: nim + ## + ## if not isGLConextCreated(): + ## skip() + bind checkpoints + + testStatusIMPL = SKIPPED + checkpoints = @[] + +macro check*(conditions: untyped): untyped = ## Verify if a statement or a list of statements is true. ## A helpful error message and set checkpoints are printed out on ## failure (if ``outputLevel`` is not ``PRINT_NONE``). @@ -236,31 +259,34 @@ macro check*(conditions: stmt): stmt {.immediate.} = proc inspectArgs(exp: NimNode): NimNode = result = copyNimTree(exp) - for i in countup(1, exp.len - 1): - if exp[i].kind notin nnkLiterals: - inc counter - var arg = newIdentNode(":p" & $counter) - var argStr = exp[i].toStrLit - var paramAst = exp[i] - if exp[i].kind == nnkIdent: - argsPrintOuts.add getAst(print(argStr, paramAst)) - if exp[i].kind in nnkCallKinds: - var callVar = newIdentNode(":c" & $counter) - argsAsgns.add getAst(asgn(callVar, paramAst)) - result[i] = callVar - argsPrintOuts.add getAst(print(argStr, callVar)) - if exp[i].kind == nnkExprEqExpr: - # ExprEqExpr - # Ident !"v" - # IntLit 2 - result[i] = exp[i][1] - if exp[i].typekind notin {ntyTypeDesc}: - argsAsgns.add getAst(asgn(arg, paramAst)) - argsPrintOuts.add getAst(print(argStr, arg)) - if exp[i].kind != nnkExprEqExpr: - result[i] = arg - else: - result[i][1] = arg + if exp[0].kind == nnkIdent and + $exp[0] in ["and", "or", "not", "in", "notin", "==", "<=", + ">=", "<", ">", "!=", "is", "isnot"]: + for i in countup(1, exp.len - 1): + if exp[i].kind notin nnkLiterals: + inc counter + var arg = newIdentNode(":p" & $counter) + var argStr = exp[i].toStrLit + var paramAst = exp[i] + if exp[i].kind == nnkIdent: + argsPrintOuts.add getAst(print(argStr, paramAst)) + if exp[i].kind in nnkCallKinds: + var callVar = newIdentNode(":c" & $counter) + argsAsgns.add getAst(asgn(callVar, paramAst)) + result[i] = callVar + argsPrintOuts.add getAst(print(argStr, callVar)) + if exp[i].kind == nnkExprEqExpr: + # ExprEqExpr + # Ident !"v" + # IntLit 2 + result[i] = exp[i][1] + if exp[i].typekind notin {ntyTypeDesc}: + argsAsgns.add getAst(asgn(arg, paramAst)) + argsPrintOuts.add getAst(print(argStr, arg)) + if exp[i].kind != nnkExprEqExpr: + result[i] = arg + else: + result[i][1] = arg case checked.kind of nnkCallKinds: @@ -292,7 +318,7 @@ macro check*(conditions: stmt): stmt {.immediate.} = result = getAst(rewrite(checked, checked.lineinfo, checked.toStrLit)) -template require*(conditions: stmt): stmt {.immediate.} = +template require*(conditions: untyped) = ## Same as `check` except any failed test causes the program to quit ## immediately. Any teardown statements are not executed and the failed ## test output is not generated. @@ -302,7 +328,7 @@ template require*(conditions: stmt): stmt {.immediate.} = check conditions abortOnError = savedAbortOnError -macro expect*(exceptions: varargs[expr], body: stmt): stmt {.immediate.} = +macro expect*(exceptions: varargs[typed], body: untyped): untyped = ## Test if `body` raises an exception found in the passed `exceptions`. ## The test passes if the raised exception is part of the acceptable ## exceptions. Otherwise, it fails. @@ -310,7 +336,7 @@ macro expect*(exceptions: varargs[expr], body: stmt): stmt {.immediate.} = ## ## .. code-block:: nim ## - ## import math + ## import math, random ## proc defectiveRobot() = ## randomize() ## case random(1..4) diff --git a/lib/pure/xmldom.nim b/lib/pure/xmldom.nim index 6cf837f25..559f45348 100644 --- a/lib/pure/xmldom.nim +++ b/lib/pure/xmldom.nim @@ -51,6 +51,9 @@ const # Illegal characters illegalChars = {'>', '<', '&', '"'} + # standard xml: attribute names + # see https://www.w3.org/XML/1998/namespace + stdattrnames = ["lang", "space", "base", "id"] type Feature = tuple[name: string, version: string] @@ -229,12 +232,15 @@ proc createAttributeNS*(doc: PDocument, namespaceURI: string, qualifiedName: str raise newException(EInvalidCharacterErr, "Invalid character") # Exceptions if qualifiedName.contains(':'): + let qfnamespaces = qualifiedName.toLower().split(':') if isNil(namespaceURI): raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be nil") - elif qualifiedName.split(':')[0].toLower() == "xml" and namespaceURI != "http://www.w3.org/XML/1998/namespace": + elif qfnamespaces[0] == "xml" and + namespaceURI != "http://www.w3.org/XML/1998/namespace" and + qfnamespaces[1] notin stdattrnames: raise newException(ENamespaceErr, "When the namespace prefix is \"xml\" namespaceURI has to be \"http://www.w3.org/XML/1998/namespace\"") - elif qualifiedName.split(':')[1].toLower() == "xmlns" and namespaceURI != "http://www.w3.org/2000/xmlns/": + elif qfnamespaces[1] == "xmlns" and namespaceURI != "http://www.w3.org/2000/xmlns/": raise newException(ENamespaceErr, "When the namespace prefix is \"xmlns\" namespaceURI has to be \"http://www.w3.org/2000/xmlns/\"") @@ -305,9 +311,12 @@ proc createElement*(doc: PDocument, tagName: string): PElement = proc createElementNS*(doc: PDocument, namespaceURI: string, qualifiedName: string): PElement = ## Creates an element of the given qualified name and namespace URI. if qualifiedName.contains(':'): + let qfnamespaces = qualifiedName.toLower().split(':') if isNil(namespaceURI): raise newException(ENamespaceErr, "When qualifiedName contains a prefix namespaceURI cannot be nil") - elif qualifiedName.split(':')[0].toLower() == "xml" and namespaceURI != "http://www.w3.org/XML/1998/namespace": + elif qfnamespaces[0] == "xml" and + namespaceURI != "http://www.w3.org/XML/1998/namespace" and + qfnamespaces[1] notin stdattrnames: raise newException(ENamespaceErr, "When the namespace prefix is \"xml\" namespaceURI has to be \"http://www.w3.org/XML/1998/namespace\"") diff --git a/lib/stdlib.nimble b/lib/stdlib.nimble index e8bb364f1..4b0066ee8 100644 --- a/lib/stdlib.nimble +++ b/lib/stdlib.nimble @@ -1,6 +1,6 @@ [Package] name = "stdlib" -version = "0.13.0" +version = "0.14.3" author = "Dominik Picheta" description = "Nim's standard library." license = "MIT" diff --git a/lib/system.nim b/lib/system.nim index da37512d6..6af5dc01f 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -66,8 +66,10 @@ type `ref`* {.magic: Pointer.}[T] ## built-in generic traced pointer type `nil` {.magic: "Nil".} - expr* {.magic: Expr.} ## meta type to denote an expression (for templates) - stmt* {.magic: Stmt.} ## meta type to denote a statement (for templates) + expr* {.magic: Expr, deprecated.} ## meta type to denote an expression (for templates) + ## **Deprecated** since version 0.15. Use ``untyped`` instead. + stmt* {.magic: Stmt, deprecated.} ## meta type to denote a statement (for templates) + ## **Deprecated** since version 0.15. Use ``typed`` instead. typedesc* {.magic: TypeDesc.} ## meta type to denote a type description void* {.magic: "VoidType".} ## meta type to denote the absence of any type auto* {.magic: Expr.} ## meta type for automatic type determination @@ -302,8 +304,7 @@ proc `==` *(x, y: pointer): bool {.magic: "EqRef", noSideEffect.} ## echo (a == b) # true due to the special meaning of `nil`/0 as a pointer proc `==` *(x, y: string): bool {.magic: "EqStr", noSideEffect.} ## Checks for equality between two `string` variables -proc `==` *(x, y: cstring): bool {.magic: "EqCString", noSideEffect.} - ## Checks for equality between two `cstring` variables + proc `==` *(x, y: char): bool {.magic: "EqCh", noSideEffect.} ## Checks for equality between two `char` variables proc `==` *(x, y: bool): bool {.magic: "EqB", noSideEffect.} @@ -339,15 +340,15 @@ proc `<` *[T](x, y: ref T): bool {.magic: "LtPtr", noSideEffect.} proc `<` *[T](x, y: ptr T): bool {.magic: "LtPtr", noSideEffect.} proc `<` *(x, y: pointer): bool {.magic: "LtPtr", noSideEffect.} -template `!=` * (x, y: expr): expr {.immediate.} = +template `!=` * (x, y: untyped): untyped = ## unequals operator. This is a shorthand for ``not (x == y)``. not (x == y) -template `>=` * (x, y: expr): expr {.immediate.} = +template `>=` * (x, y: untyped): untyped = ## "is greater or equals" operator. This is the same as ``y <= x``. y <= x -template `>` * (x, y: expr): expr {.immediate.} = +template `>` * (x, y: untyped): untyped = ## "is greater" operator. This is the same as ``y < x``. y < x @@ -588,6 +589,9 @@ proc sizeof*[T](x: T): int {.magic: "SizeOf", noSideEffect.} ## that one never needs to know ``x``'s size. As a special semantic rule, ## ``x`` may also be a type identifier (``sizeof(int)`` is valid). ## + ## Limitations: If used within nim VM context ``sizeof`` will only work + ## for simple types. + ## ## .. code-block:: nim ## sizeof('A') #=> 1 ## sizeof(2) #=> 8 @@ -667,6 +671,12 @@ proc newSeq*[T](len = 0.Natural): seq[T] = ## #inputStrings[3] = "out of bounds" newSeq(result, len) +proc newSeqOfCap*[T](cap: Natural): seq[T] {. + magic: "NewSeqOfCap", noSideEffect.} = + ## creates a new sequence of type ``seq[T]`` with length 0 and capacity + ## ``cap``. + discard + proc len*[TOpenArray: openArray|varargs](x: TOpenArray): int {. magic: "LengthOpenArray", noSideEffect.} proc len*(x: string): int {.magic: "LengthStr", noSideEffect.} @@ -1090,13 +1100,13 @@ proc contains*[T](s: Slice[T], value: T): bool {.noSideEffect, inline.} = ## assert((1..3).contains(4) == false) result = s.a <= value and value <= s.b -template `in` * (x, y: expr): expr {.immediate, dirty.} = contains(y, x) +template `in` * (x, y: untyped): untyped {.dirty.} = contains(y, x) ## Sugar for contains ## ## .. code-block:: Nim ## assert(1 in (1..3) == true) ## assert(5 in (1..3) == false) -template `notin` * (x, y: expr): expr {.immediate, dirty.} = not contains(y, x) +template `notin` * (x, y: untyped): untyped {.dirty.} = not contains(y, x) ## Sugar for not containing ## ## .. code-block:: Nim @@ -1115,7 +1125,7 @@ proc `is` *[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.} ## ## assert(test[int](3) == 3) ## assert(test[string]("xyz") == 0) -template `isnot` *(x, y: expr): expr {.immediate.} = not (x is y) +template `isnot` *(x, y: untyped): untyped = not (x is y) ## Negated version of `is`. Equivalent to ``not(x is y)``. proc `of` *[T, S](x: T, y: S): bool {.magic: "Of", noSideEffect.} @@ -1291,7 +1301,7 @@ const when hasThreadSupport and defined(tcc) and not compileOption("tlsEmulation"): # tcc doesn't support TLS {.error: "``--tlsEmulation:on`` must be used when using threads with tcc backend".} - + when defined(boehmgc): when defined(windows): const boehmLib = "boehmgc.dll" @@ -1521,7 +1531,7 @@ type # these work for most platforms: ## This is the same as the type ``unsigned long long`` in *C*. cstringArray* {.importc: "char**", nodecl.} = ptr - array [0..ArrayDummySize, cstring] + array[0..ArrayDummySize, cstring] ## This is binary compatible to the type ``char**`` in *C*. The array's ## high value is large enough to disable bounds checking in practice. ## Use `cstringArrayToSeq` to convert it into a ``seq[string]``. @@ -1591,34 +1601,32 @@ proc substr*(s: string, first, last: int): string {. ## is used instead: This means ``substr`` can also be used to `cut`:idx: ## or `limit`:idx: a string's length. -when not defined(nimscript): - proc zeroMem*(p: pointer, size: Natural) {.importc, noDecl, benign.} +when not defined(nimscript) and not defined(JS): + proc zeroMem*(p: pointer, size: Natural) {.inline, benign.} ## overwrites the contents of the memory at ``p`` with the value 0. ## Exactly ``size`` bytes will be overwritten. Like any procedure ## dealing with raw memory this is *unsafe*. - proc copyMem*(dest, source: pointer, size: Natural) {. - importc: "memcpy", header: "<string.h>", benign.} + proc copyMem*(dest, source: pointer, size: Natural) {.inline, benign.} ## copies the contents from the memory at ``source`` to the memory ## at ``dest``. Exactly ``size`` bytes will be copied. The memory ## regions may not overlap. Like any procedure dealing with raw ## memory this is *unsafe*. - proc moveMem*(dest, source: pointer, size: Natural) {. - importc: "memmove", header: "<string.h>", benign.} + proc moveMem*(dest, source: pointer, size: Natural) {.inline, benign.} ## copies the contents from the memory at ``source`` to the memory ## at ``dest``. Exactly ``size`` bytes will be copied. The memory ## regions may overlap, ``moveMem`` handles this case appropriately ## and is thus somewhat more safe than ``copyMem``. Like any procedure ## dealing with raw memory this is still *unsafe*, though. - proc equalMem*(a, b: pointer, size: Natural): bool {. - importc: "equalMem", noDecl, noSideEffect.} + proc equalMem*(a, b: pointer, size: Natural): bool {.inline, noSideEffect.} ## compares the memory blocks ``a`` and ``b``. ``size`` bytes will ## be compared. If the blocks are equal, true is returned, false ## otherwise. Like any procedure dealing with raw memory this is ## *unsafe*. +when not defined(nimscript): when hasAlloc: proc alloc*(size: Natural): pointer {.noconv, rtl, tags: [], benign.} ## allocates a new memory block with at least ``size`` bytes. The @@ -1736,11 +1744,18 @@ proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.} ## swaps the values `a` and `b`. This is often more efficient than ## ``tmp = a; a = b; b = tmp``. Particularly useful for sorting algorithms. -template `>=%` *(x, y: expr): expr {.immediate.} = y <=% x +when not defined(js) and not defined(booting) and defined(nimTrMacros): + template swapRefsInArray*{swap(arr[a], arr[b])}(arr: openarray[ref], a, b: int) = + # Optimize swapping of array elements if they are refs. Default swap + # implementation will cause unsureAsgnRef to be emitted which causes + # unnecessary slow down in this case. + swap(cast[ptr pointer](addr arr[a])[], cast[ptr pointer](addr arr[b])[]) + +template `>=%` *(x, y: untyped): untyped = y <=% x ## treats `x` and `y` as unsigned and compares them. ## Returns true iff ``unsigned(x) >= unsigned(y)``. -template `>%` *(x, y: expr): expr {.immediate.} = y <% x +template `>%` *(x, y: untyped): untyped = y <% x ## treats `x` and `y` as unsigned and compares them. ## Returns true iff ``unsigned(x) > unsigned(y)``. @@ -1807,10 +1822,10 @@ const NimMajor*: int = 0 ## is the major number of Nim's version. - NimMinor*: int = 13 + NimMinor*: int = 14 ## is the minor number of Nim's version. - NimPatch*: int = 1 + NimPatch*: int = 3 ## is the patch number of Nim's version. NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch @@ -1868,7 +1883,7 @@ iterator countdown*[T](a, b: T, step = 1): T {.inline.} = yield res dec(res, step) -template countupImpl(incr: stmt) {.immediate, dirty.} = +template countupImpl(incr: untyped) {.oldimmediate, dirty.} = when T is IntLikeForCount: var res = int(a) while res <= int(b): @@ -2163,7 +2178,7 @@ proc `&` *[T](x: T, y: seq[T]): seq[T] {.noSideEffect.} = result[i+1] = y[i] when not defined(nimscript): - when not defined(JS): + when not defined(JS) or defined(nimphp): proc seqToPtr[T](x: seq[T]): pointer {.inline, nosideeffect.} = result = cast[pointer](x) else: @@ -2293,12 +2308,15 @@ proc `$`*[T: tuple|object](x: T): string = if not firstElement: result.add(", ") result.add(name) result.add(": ") - when compiles(value.isNil): - if value.isNil: result.add "nil" - else: result.add($value) + when compiles($value): + when compiles(value.isNil): + if value.isNil: result.add "nil" + else: result.add($value) + else: + result.add($value) + firstElement = false else: - result.add($value) - firstElement = false + result.add("...") result.add(")") proc collectionToString[T: set | seq](x: T, b, e: string): string = @@ -2511,7 +2529,7 @@ template newException*(exceptn: typedesc, message: string): expr = e when hostOS == "standalone": - include panicoverride + include "$projectpath/panicoverride" when not declared(sysFatal): when hostOS == "standalone": @@ -2564,15 +2582,11 @@ else: when not defined(JS): #and not defined(nimscript): {.push stack_trace: off, profiler:off.} - when not ( - defined(nimscript) or - defined(boehmgc) or - defined(gogc) or - (defined(nogc) and defined(useMalloc))): - proc initAllocator() {.inline.} - - when not defined(nimscript) and not defined(nogc): - proc initGC() + when not defined(nimscript) and (not defined(nogc) or not defined(useMalloc)): + when not defined(gcStack): + proc initGC() + when not defined(boehmgc) and not defined(gogc) and not defined(gcStack): + proc initAllocator() {.inline.} proc initStackBottom() {.inline, compilerproc.} = # WARNING: This is very fragile! An array size of 8 does not work on my @@ -2617,11 +2631,21 @@ when not defined(JS): #and not defined(nimscript): {.deprecated: [TFile: File, TFileHandle: FileHandle, TFileMode: FileMode].} - when not defined(nimscript): - include "system/ansi_c" + include "system/ansi_c" + when not defined(nimscript): proc cmp(x, y: string): int = result = int(c_strcmp(x, y)) + + proc zeroMem(p: pointer, size: Natural) = + c_memset(p, 0, size) + proc copyMem(dest, source: pointer, size: Natural) = + c_memcpy(dest, source, size) + proc moveMem(dest, source: pointer, size: Natural) = + c_memmove(dest, source, size) + proc equalMem(a, b: pointer, size: Natural): bool = + c_memcmp(a, b, size) == 0 + else: proc cmp(x, y: string): int = if x < y: result = -1 @@ -2629,28 +2653,20 @@ when not defined(JS): #and not defined(nimscript): else: result = 0 when defined(nimscript): + proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} + ## Opens a file named `filename` for reading. + ## + ## Then calls `readAll <#readAll>`_ and closes the file afterwards. + ## Returns the string. Raises an IO exception in case of an error. If + ## you need to call this inside a compile time macro you can use + ## `staticRead <#staticRead>`_. + proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} ## Opens a file named `filename` for writing. Then writes the ## `content` completely to the file and closes the file afterwards. ## Raises an IO exception in case of an error. when not defined(nimscript) and hostOS != "standalone": - when defined(windows): - # work-around C's sucking abstraction: - # BUGFIX: stdin and stdout should be binary files! - proc setmode(handle, mode: int) {.importc: "setmode", - header: "<io.h>".} - proc fileno(f: C_TextFileStar): int {.importc: "fileno", - header: "<fcntl.h>".} - var - O_BINARY {.importc: "O_BINARY", nodecl.}: int - - # we use binary mode on Windows: - setmode(fileno(c_stdin), O_BINARY) - setmode(fileno(c_stdout), O_BINARY) - - when defined(endb): - proc endbStep() # text file handling: var @@ -2661,6 +2677,21 @@ when not defined(JS): #and not defined(nimscript): stderr* {.importc: "stderr", header: "<stdio.h>".}: File ## The standard error stream. + when defined(windows): + # work-around C's sucking abstraction: + # BUGFIX: stdin and stdout should be binary files! + proc c_setmode(handle, mode: cint) {.importc: "_setmode", + header: "<io.h>".} + var + O_BINARY {.importc: "O_BINARY", nodecl.}: cint + + # we use binary mode on Windows: + c_setmode(c_fileno(stdin), O_BINARY) + c_setmode(c_fileno(stdout), O_BINARY) + + when defined(endb): + proc endbStep() + when defined(useStdoutAsStdmsg): template stdmsg*: File = stdout else: @@ -2699,17 +2730,19 @@ when not defined(JS): #and not defined(nimscript): ## ## Default mode is readonly. Returns true iff the file could be reopened. - proc close*(f: File) {.importc: "fclose", header: "<stdio.h>", tags: [].} + proc setStdIoUnbuffered*() {.tags: [], benign.} + ## Configures `stdin`, `stdout` and `stderr` to be unbuffered. + + proc close*(f: File) {.tags: [].} ## Closes the file. proc endOfFile*(f: File): bool {.tags: [], benign.} ## Returns true iff `f` is at the end. - proc readChar*(f: File): char {. - importc: "fgetc", header: "<stdio.h>", tags: [ReadIOEffect].} + proc readChar*(f: File): char {.tags: [ReadIOEffect].} ## Reads a single character from the stream `f`. - proc flushFile*(f: File) {. - importc: "fflush", header: "<stdio.h>", tags: [WriteIOEffect].} + + proc flushFile*(f: File) {.tags: [WriteIOEffect].} ## Flushes `f`'s buffer. proc readAll*(file: File): TaintedString {.tags: [ReadIOEffect], benign.} @@ -2779,6 +2812,9 @@ when not defined(JS): #and not defined(nimscript): ## reads `len` bytes into the buffer `a` starting at ``a[start]``. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. + ## + ## **Warning:** The buffer `a` must be pre-allocated. This can be done + ## using, for example, ``newString``. proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [ReadIOEffect], benign.} @@ -2812,8 +2848,7 @@ when not defined(JS): #and not defined(nimscript): ## retrieves the current position of the file pointer that is used to ## read from the file `f`. The file's first byte has the index zero. - proc getFileHandle*(f: File): FileHandle {.importc: "fileno", - header: "<stdio.h>"} + proc getFileHandle*(f: File): FileHandle ## returns the OS file handle of the file ``f``. This is only useful for ## platform specific programming. @@ -2879,6 +2914,7 @@ when not defined(JS): #and not defined(nimscript): when declared(initAllocator): initAllocator() when hasThreadSupport: + const insideRLocksModule = false include "system/syslocks" when hostOS != "standalone": include "system/threads" elif not defined(nogc) and not defined(nimscript): @@ -3031,34 +3067,6 @@ when not defined(JS): #and not defined(nimscript): {.pop.} # stacktrace when not defined(nimscript): - proc likely*(val: bool): bool {.importc: "likely", nodecl, nosideeffect.} - ## Hints the optimizer that `val` is likely going to be true. - ## - ## You can use this proc to decorate a branch condition. On certain - ## platforms this can help the processor predict better which branch is - ## going to be run. Example: - ## - ## .. code-block:: nim - ## for value in inputValues: - ## if likely(value <= 100): - ## process(value) - ## else: - ## echo "Value too big!" - - proc unlikely*(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.} - ## Hints the optimizer that `val` is likely going to be false. - ## - ## You can use this proc to decorate a branch condition. On certain - ## platforms this can help the processor predict better which branch is - ## going to be run. Example: - ## - ## .. code-block:: nim - ## for value in inputValues: - ## if unlikely(value > 100): - ## echo "Value too big!" - ## else: - ## process(value) - proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} = ## retrieves the raw proc pointer of the closure `x`. This is ## useful for interfacing closures with C. @@ -3126,11 +3134,63 @@ proc quit*(errormsg: string, errorcode = QuitFailure) {.noReturn.} = {.pop.} # checks {.pop.} # hints +when not defined(JS): + proc likely_proc(val: bool): bool {.importc: "likely", nodecl, nosideeffect.} + proc unlikely_proc(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.} + +template likely*(val: bool): bool = + ## Hints the optimizer that `val` is likely going to be true. + ## + ## You can use this template to decorate a branch condition. On certain + ## platforms this can help the processor predict better which branch is + ## going to be run. Example: + ## + ## .. code-block:: nim + ## for value in inputValues: + ## if likely(value <= 100): + ## process(value) + ## else: + ## echo "Value too big!" + ## + ## On backends without branch prediction (JS and the nimscript VM), this + ## template will not affect code execution. + when nimvm: + val + else: + when defined(JS): + val + else: + likely_proc(val) + +template unlikely*(val: bool): bool = + ## Hints the optimizer that `val` is likely going to be false. + ## + ## You can use this proc to decorate a branch condition. On certain + ## platforms this can help the processor predict better which branch is + ## going to be run. Example: + ## + ## .. code-block:: nim + ## for value in inputValues: + ## if unlikely(value > 100): + ## echo "Value too big!" + ## else: + ## process(value) + ## + ## On backends without branch prediction (JS and the nimscript VM), this + ## template will not affect code execution. + when nimvm: + val + else: + when defined(JS): + val + else: + unlikely_proc(val) + proc `/`*(x, y: int): float {.inline, noSideEffect.} = ## integer division that results in a float. result = toFloat(x) / toFloat(y) -template spliceImpl(s, a, L, b: expr): stmt {.immediate.} = +template spliceImpl(s, a, L, b: untyped): untyped = # make room for additional elements or cut: var slen = s.len var shift = b.len - L @@ -3172,7 +3232,7 @@ proc `[]`*[Idx, T](a: array[Idx, T], x: Slice[int]): seq[T] = when low(a) < 0: {.error: "Slicing for arrays with negative indices is unsupported.".} var L = x.b - x.a + 1 - newSeq(result, L) + result = newSeq[T](L) for i in 0.. <L: result[i] = a[i + x.a] proc `[]=`*[Idx, T](a: var array[Idx, T], x: Slice[int], b: openArray[T]) = @@ -3308,7 +3368,11 @@ proc astToStr*[T](x: T): string {.magic: "AstToStr", noSideEffect.} proc instantiationInfo*(index = -1, fullPaths = false): tuple[ filename: string, line: int] {. magic: "InstantiationInfo", noSideEffect.} - ## provides access to the compiler's instantiation stack line information. + ## provides access to the compiler's instantiation stack line information + ## of a template. + ## + ## While similar to the `caller info`:idx: of other languages, it is determined + ## at compile time. ## ## This proc is mostly useful for meta programming (eg. ``assert`` template) ## to retrieve information about the current filename and line number. @@ -3412,7 +3476,7 @@ iterator mitems*(a: var string): var char {.inline.} = when not defined(nimhygiene): {.pragma: inject.} -template onFailedAssert*(msg: expr, code: stmt): stmt {.dirty, immediate.} = +template onFailedAssert*(msg, code: untyped): untyped {.dirty.} = ## Sets an assertion failure handler that will intercept any assert ## statements following `onFailedAssert` in the current module scope. ## @@ -3425,7 +3489,7 @@ template onFailedAssert*(msg: expr, code: stmt): stmt {.dirty, immediate.} = ## e.lineinfo = instantiationInfo(-2) ## raise e ## - template failedAssertImpl(msgIMPL: string): stmt {.dirty.} = + template failedAssertImpl(msgIMPL: string): untyped {.dirty.} = let msg = msgIMPL code @@ -3578,7 +3642,43 @@ proc xlen*[T](x: seq[T]): int {.magic: "XLenSeq", noSideEffect.} = ## This is an optimization that rarely makes sense. discard + +proc `==` *(x, y: cstring): bool {.magic: "EqCString", noSideEffect, + inline.} = + ## Checks for equality between two `cstring` variables. + proc strcmp(a, b: cstring): cint {.noSideEffect, + importc, header: "<string.h>".} + if pointer(x) == pointer(y): result = true + elif x.isNil or y.isNil: result = false + else: result = strcmp(x, y) == 0 + +template closureScope*(body: untyped): untyped = + ## Useful when creating a closure in a loop to capture local loop variables by + ## their current iteration values. Example: + ## + ## .. code-block:: nim + ## var myClosure : proc() + ## # without closureScope: + ## for i in 0 .. 5: + ## let j = i + ## if j == 3: + ## myClosure = proc() = echo j + ## myClosure() # outputs 5. `j` is changed after closure creation + ## # with closureScope: + ## for i in 0 .. 5: + ## closureScope: # Everything in this scope is locked after closure creation + ## let j = i + ## if j == 3: + ## myClosure = proc() = echo j + ## myClosure() # outputs 3 + (proc() = body)() + {.pop.} #{.push warning[GcMem]: off, warning[Uninit]: off.} when defined(nimconfig): include "system/nimscript" + +when defined(windows) and appType == "console" and not defined(nimconfig): + proc setConsoleOutputCP(codepage: cint): cint {.stdcall, dynlib: "kernel32", + importc: "SetConsoleOutputCP".} + discard setConsoleOutputCP(65001) # 65001 - utf-8 codepage diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index e0fd53b7b..bed9fd906 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -27,15 +27,14 @@ const type PTrunk = ptr Trunk - Trunk {.final.} = object + Trunk = object next: PTrunk # all nodes are connected with this pointer key: int # start address at bit 0 bits: array[0..IntsPerTrunk-1, int] # a bit vector TrunkBuckets = array[0..255, PTrunk] - IntSet {.final.} = object + IntSet = object data: TrunkBuckets -{.deprecated: [TIntSet: IntSet, TTrunk: Trunk, TTrunkBuckets: TrunkBuckets].} type AlignType = BiggestFloat @@ -64,8 +63,6 @@ type next, prev: PBigChunk # chunks of the same (or bigger) size align: int data: AlignType # start of usable memory -{.deprecated: [TAlignType: AlignType, TFreeCell: FreeCell, TBaseChunk: BaseChunk, - TBigChunk: BigChunk, TSmallChunk: SmallChunk].} template smallChunkOverhead(): expr = sizeof(SmallChunk)-sizeof(AlignType) template bigChunkOverhead(): expr = sizeof(BigChunk)-sizeof(AlignType) @@ -79,18 +76,18 @@ template bigChunkOverhead(): expr = sizeof(BigChunk)-sizeof(AlignType) type PLLChunk = ptr LLChunk - LLChunk {.pure.} = object ## *low-level* chunk + LLChunk = object ## *low-level* chunk size: int # remaining size acc: int # accumulator next: PLLChunk # next low-level chunk; only needed for dealloc PAvlNode = ptr AvlNode - AvlNode {.pure, final.} = object + AvlNode = object link: array[0..1, PAvlNode] # Left (0) and right (1) links key, upperBound: int level: int - MemRegion {.final, pure.} = object + MemRegion = object minLargeObj, maxLargeObj: int freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk] llmem: PLLChunk @@ -99,6 +96,7 @@ type freeChunksList: PBigChunk # XXX make this a datastructure with O(1) access chunkStarts: IntSet root, deleted, last, freeAvlNodes: PAvlNode + locked: bool # if locked, we cannot free pages. {.deprecated: [TLLChunk: LLChunk, TAvlNode: AvlNode, TMemRegion: MemRegion].} # shared: @@ -234,7 +232,8 @@ proc isSmallChunk(c: PChunk): bool {.inline.} = proc chunkUnused(c: PChunk): bool {.inline.} = result = not c.used -iterator allObjects(m: MemRegion): pointer {.inline.} = +iterator allObjects(m: var MemRegion): pointer {.inline.} = + m.locked = true for s in elements(m.chunkStarts): # we need to check here again as it could have been modified: if s in m.chunkStarts: @@ -252,6 +251,7 @@ iterator allObjects(m: MemRegion): pointer {.inline.} = else: let c = cast[PBigChunk](c) yield addr(c.data) + m.locked = false proc iterToProc*(iter: typed, envType: typedesc; procName: untyped) {. magic: "Plugin", compileTime.} @@ -311,7 +311,7 @@ proc freeOsChunks(a: var MemRegion, p: pointer, size: int) = osDeallocPages(p, size) decCurrMem(a, size) dec(a.freeMem, size) - #c_fprintf(c_stdout, "[Alloc] back to OS: %ld\n", size) + #c_fprintf(stdout, "[Alloc] back to OS: %ld\n", size) proc isAccessible(a: MemRegion, p: pointer): bool {.inline.} = result = contains(a.chunkStarts, pageIndex(p)) @@ -324,9 +324,9 @@ proc contains[T](list, x: T): bool = proc writeFreeList(a: MemRegion) = var it = a.freeChunksList - c_fprintf(c_stdout, "freeChunksList: %p\n", it) + c_fprintf(stdout, "freeChunksList: %p\n", it) while it != nil: - c_fprintf(c_stdout, "it: %p, next: %p, prev: %p\n", + c_fprintf(stdout, "it: %p, next: %p, prev: %p\n", it, it.next, it.prev) it = it.next @@ -385,7 +385,7 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = excl(a.chunkStarts, pageIndex(c)) c = cast[PBigChunk](le) - if c.size < ChunkOsReturn or doNotUnmap: + if c.size < ChunkOsReturn or doNotUnmap or a.locked: incl(a, a.chunkStarts, pageIndex(c)) updatePrevSize(a, c, c.size) listAdd(a.freeChunksList, c) @@ -442,26 +442,29 @@ proc getSmallChunk(a: var MemRegion): PSmallChunk = # ----------------------------------------------------------------------------- proc isAllocatedPtr(a: MemRegion, p: pointer): bool {.benign.} -proc allocInv(a: MemRegion): bool = - ## checks some (not all yet) invariants of the allocator's data structures. - for s in low(a.freeSmallChunks)..high(a.freeSmallChunks): - var c = a.freeSmallChunks[s] - while not (c == nil): - if c.next == c: - echo "[SYSASSERT] c.next == c" - return false - if not (c.size == s * MemAlign): - echo "[SYSASSERT] c.size != s * MemAlign" - return false - var it = c.freeList - while not (it == nil): - if not (it.zeroField == 0): - echo "[SYSASSERT] it.zeroField != 0" - c_printf("%ld %p\n", it.zeroField, it) +when true: + template allocInv(a: MemRegion): bool = true +else: + proc allocInv(a: MemRegion): bool = + ## checks some (not all yet) invariants of the allocator's data structures. + for s in low(a.freeSmallChunks)..high(a.freeSmallChunks): + var c = a.freeSmallChunks[s] + while not (c == nil): + if c.next == c: + echo "[SYSASSERT] c.next == c" return false - it = it.next - c = c.next - result = true + if not (c.size == s * MemAlign): + echo "[SYSASSERT] c.size != s * MemAlign" + return false + var it = c.freeList + while not (it == nil): + if not (it.zeroField == 0): + echo "[SYSASSERT] it.zeroField != 0" + c_printf("%ld %p\n", it.zeroField, it) + return false + it = it.next + c = c.next + result = true proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(allocInv(a), "rawAlloc: begin") @@ -469,7 +472,7 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small") var size = roundup(requestedSize, MemAlign) sysAssert(size >= requestedSize, "insufficient allocated size!") - #c_fprintf(c_stdout, "alloc; size: %ld; %ld\n", requestedSize, size) + #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size) if size <= SmallChunkSize-smallChunkOverhead(): # allocate a small block: for small chunks, we use only its next pointer var s = size div MemAlign @@ -490,7 +493,7 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(allocInv(a), "rawAlloc: begin c != nil") sysAssert c.next != c, "rawAlloc 5" #if c.size != size: - # c_fprintf(c_stdout, "csize: %lld; size %lld\n", c.size, size) + # c_fprintf(stdout, "csize: %lld; size %lld\n", c.size, size) sysAssert c.size == size, "rawAlloc 6" if c.freeList == nil: sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 1abd8466d..a8e358229 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -13,60 +13,43 @@ {.push hints:off} -proc c_strcmp(a, b: cstring): cint {.header: "<string.h>", - noSideEffect, importc: "strcmp".} -proc c_memcmp(a, b: cstring, size: int): cint {.header: "<string.h>", - noSideEffect, importc: "memcmp".} -proc c_memcpy(a, b: cstring, size: int) {.header: "<string.h>", importc: "memcpy".} -proc c_strlen(a: cstring): int {.header: "<string.h>", - noSideEffect, importc: "strlen".} -proc c_memset(p: pointer, value: cint, size: int) {. - header: "<string.h>", importc: "memset".} - -when not declared(File): - type - C_TextFile {.importc: "FILE", header: "<stdio.h>", - final, incompleteStruct.} = object - C_BinaryFile {.importc: "FILE", header: "<stdio.h>", - final, incompleteStruct.} = object - C_TextFileStar = ptr C_TextFile - C_BinaryFileStar = ptr C_BinaryFile -else: - type - C_TextFileStar = File - C_BinaryFileStar = File +proc c_memchr(s: pointer, c: cint, n: csize): pointer {. + importc: "memchr", header: "<string.h>".} +proc c_memcmp(a, b: pointer, size: csize): cint {. + importc: "memcmp", header: "<string.h>", noSideEffect.} +proc c_memcpy(a, b: pointer, size: csize): pointer {. + importc: "memcpy", header: "<string.h>", discardable.} +proc c_memmove(a, b: pointer, size: csize): pointer {. + importc: "memmove", header: "<string.h>",discardable.} +proc c_memset(p: pointer, value: cint, size: csize): pointer {. + importc: "memset", header: "<string.h>", discardable.} +proc c_strcmp(a, b: cstring): cint {. + importc: "strcmp", header: "<string.h>", noSideEffect.} type C_JmpBuf {.importc: "jmp_buf", header: "<setjmp.h>".} = object -when not defined(vm): - var - c_stdin {.importc: "stdin", nodecl.}: C_TextFileStar - c_stdout {.importc: "stdout", nodecl.}: C_TextFileStar - c_stderr {.importc: "stderr", nodecl.}: C_TextFileStar - -# constants faked as variables: -when not declared(SIGINT): +when defined(windows): + const + SIGABRT = cint(22) + SIGFPE = cint(8) + SIGILL = cint(4) + SIGINT = cint(2) + SIGSEGV = cint(11) + SIGTERM = cint(15) +elif defined(macosx) or defined(linux) or defined(freebsd) or + defined(openbsd) or defined(netbsd) or defined(solaris): + const + SIGABRT = cint(6) + SIGFPE = cint(8) + SIGILL = cint(4) + SIGINT = cint(2) + SIGSEGV = cint(11) + SIGTERM = cint(15) + SIGPIPE = cint(13) +else: when NoFakeVars: - when defined(windows): - const - SIGABRT = cint(22) - SIGFPE = cint(8) - SIGILL = cint(4) - SIGINT = cint(2) - SIGSEGV = cint(11) - SIGTERM = cint(15) - elif defined(macosx) or defined(linux): - const - SIGABRT = cint(6) - SIGFPE = cint(8) - SIGILL = cint(4) - SIGINT = cint(2) - SIGSEGV = cint(11) - SIGTERM = cint(15) - SIGPIPE = cint(13) - else: - {.error: "SIGABRT not ported to your platform".} + {.error: "SIGABRT not ported to your platform".} else: var SIGINT {.importc: "SIGINT", nodecl.}: cint @@ -78,10 +61,7 @@ when not declared(SIGINT): var SIGPIPE {.importc: "SIGPIPE", nodecl.}: cint when defined(macosx): - when NoFakeVars: - const SIGBUS = cint(10) - else: - var SIGBUS {.importc: "SIGBUS", nodecl.}: cint + const SIGBUS = cint(10) else: template SIGBUS: expr = SIGSEGV @@ -103,72 +83,27 @@ else: proc c_setjmp(jmpb: C_JmpBuf): cint {. header: "<setjmp.h>", importc: "setjmp".} -proc c_signal(sign: cint, handler: proc (a: cint) {.noconv.}) {. - importc: "signal", header: "<signal.h>".} -proc c_raise(sign: cint) {.importc: "raise", header: "<signal.h>".} +type c_sighandler_t = proc (a: cint) {.noconv.} +proc c_signal(sign: cint, handler: proc (a: cint) {.noconv.}): c_sighandler_t {. + importc: "signal", header: "<signal.h>", discardable.} -proc c_fputs(c: cstring, f: C_TextFileStar) {.importc: "fputs", - header: "<stdio.h>".} -proc c_fgets(c: cstring, n: int, f: C_TextFileStar): cstring {. - importc: "fgets", header: "<stdio.h>".} -proc c_fgetc(stream: C_TextFileStar): int {.importc: "fgetc", - header: "<stdio.h>".} -proc c_ungetc(c: int, f: C_TextFileStar) {.importc: "ungetc", - header: "<stdio.h>".} -proc c_putc(c: char, stream: C_TextFileStar) {.importc: "putc", - header: "<stdio.h>".} -proc c_fprintf(f: C_TextFileStar, frmt: cstring) {. - importc: "fprintf", header: "<stdio.h>", varargs.} -proc c_printf(frmt: cstring) {. - importc: "printf", header: "<stdio.h>", varargs.} +proc c_fprintf(f: File, frmt: cstring): cint {. + importc: "fprintf", header: "<stdio.h>", varargs, discardable.} +proc c_printf(frmt: cstring): cint {. + importc: "printf", header: "<stdio.h>", varargs, discardable.} -proc c_fopen(filename, mode: cstring): C_TextFileStar {. - importc: "fopen", header: "<stdio.h>".} -proc c_fclose(f: C_TextFileStar) {.importc: "fclose", header: "<stdio.h>".} - -proc c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", - importc: "sprintf", varargs, noSideEffect.} +proc c_sprintf(buf, frmt: cstring): cint {. + importc: "sprintf", header: "<stdio.h>", varargs, noSideEffect.} # we use it only in a way that cannot lead to security issues -proc c_fread(buf: pointer, size, n: int, f: C_BinaryFileStar): int {. - importc: "fread", header: "<stdio.h>".} -proc c_fseek(f: C_BinaryFileStar, offset: clong, whence: int): int {. - importc: "fseek", header: "<stdio.h>".} - -proc c_fwrite(buf: pointer, size, n: int, f: C_BinaryFileStar): int {. - importc: "fwrite", header: "<stdio.h>".} - -proc c_exit(errorcode: cint) {.importc: "exit", header: "<stdlib.h>".} -proc c_ferror(stream: C_TextFileStar): bool {. - importc: "ferror", header: "<stdio.h>".} -proc c_fflush(stream: C_TextFileStar) {.importc: "fflush", header: "<stdio.h>".} -proc c_abort() {.importc: "abort", header: "<stdlib.h>".} -proc c_feof(stream: C_TextFileStar): bool {. - importc: "feof", header: "<stdio.h>".} +proc c_fileno(f: File): cint {. + importc: "fileno", header: "<fcntl.h>".} -proc c_malloc(size: int): pointer {.importc: "malloc", header: "<stdlib.h>".} -proc c_free(p: pointer) {.importc: "free", header: "<stdlib.h>".} -proc c_realloc(p: pointer, newsize: int): pointer {. +proc c_malloc(size: csize): pointer {. + importc: "malloc", header: "<stdlib.h>".} +proc c_free(p: pointer) {. + importc: "free", header: "<stdlib.h>".} +proc c_realloc(p: pointer, newsize: csize): pointer {. importc: "realloc", header: "<stdlib.h>".} -when hostOS != "standalone": - when not declared(errno): - when defined(NimrodVM): - var vmErrnoWrapper {.importc.}: ptr cint - template errno: expr = - bind vmErrnoWrapper - vmErrnoWrapper[] - else: - var errno {.importc, header: "<errno.h>".}: cint ## error variable -proc strerror(errnum: cint): cstring {.importc, header: "<string.h>".} - -proc c_remove(filename: cstring): cint {. - importc: "remove", header: "<stdio.h>".} -proc c_rename(oldname, newname: cstring): cint {. - importc: "rename", header: "<stdio.h>".} - -proc c_system(cmd: cstring): cint {.importc: "system", header: "<stdlib.h>".} -proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "<stdlib.h>".} -proc c_putenv(env: cstring): cint {.importc: "putenv", header: "<stdlib.h>".} - {.pop} diff --git a/lib/system/assign.nim b/lib/system/assign.nim index 55d7572e2..231a20d86 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -211,14 +211,14 @@ proc genericReset(dest: pointer, mt: PNimType) = zeroMem(dest, mt.size) # set raw bits to zero proc selectBranch(discVal, L: int, - a: ptr array [0..0x7fff, ptr TNimNode]): ptr TNimNode = + a: ptr array[0..0x7fff, ptr TNimNode]): ptr TNimNode = result = a[L] # a[L] contains the ``else`` part (but may be nil) if discVal <% L: var x = a[discVal] if x != nil: result = x proc FieldDiscriminantCheck(oldDiscVal, newDiscVal: int, - a: ptr array [0..0x7fff, ptr TNimNode], + a: ptr array[0..0x7fff, ptr TNimNode], L: int) {.compilerProc.} = var oldBranch = selectBranch(oldDiscVal, L, a) var newBranch = selectBranch(newDiscVal, L, a) diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 6caf99d27..27a3708ea 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -51,7 +51,6 @@ proc chckRangeF(x, a, b: float): float = proc chckNil(p: pointer) = if p == nil: sysFatal(ValueError, "attempt to write to a nil address") - #c_raise(SIGSEGV) proc chckObj(obj, subclass: PNimType) {.compilerproc.} = # checks if obj is of type subclass: diff --git a/lib/system/debugger.nim b/lib/system/debugger.nim index b18c61755..cc6919d36 100644 --- a/lib/system/debugger.nim +++ b/lib/system/debugger.nim @@ -108,8 +108,8 @@ proc fileMatches(c, bp: cstring): bool = # and the character for the suffix does not exist or # is one of: \ / : # depending on the OS case does not matter! - var blen: int = c_strlen(bp) - var clen: int = c_strlen(c) + var blen: int = bp.len + var clen: int = c.len if blen > clen: return false # check for \ / : if clen-blen-1 >= 0 and c[clen-blen-1] notin {'\\', '/', ':'}: @@ -159,7 +159,7 @@ type {.deprecated: [THash: Hash, TWatchpoint: Watchpoint].} var - watchpoints: array [0..99, Watchpoint] + watchpoints: array[0..99, Watchpoint] watchpointsLen: int proc `!&`(h: Hash, val: int): Hash {.inline.} = diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 03230e541..5445a067c 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -36,7 +36,7 @@ proc copyDeepString(src: NimString): NimString {.inline.} = if src != nil: result = rawNewStringNoInit(src.len) result.len = src.len - c_memcpy(result.data, src.data, src.len + 1) + copyMem(addr(result.data), addr(src.data), src.len + 1) proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) = var @@ -124,7 +124,9 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) = copyMem(dest, src, mt.size) proc genericDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} = + GC_disable() genericDeepCopyAux(dest, src, mt) + GC_enable() proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} = # also invoked for 'string' diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 3b3d1f87d..0a994efac 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -52,11 +52,14 @@ when defined(posix): # # c stuff: - var - RTLD_NOW {.importc: "RTLD_NOW", header: "<dlfcn.h>".}: int + when defined(linux) or defined(macosx): + const RTLD_NOW = cint(2) + else: + var + RTLD_NOW {.importc: "RTLD_NOW", header: "<dlfcn.h>".}: cint proc dlclose(lib: LibHandle) {.importc, header: "<dlfcn.h>".} - proc dlopen(path: cstring, mode: int): LibHandle {. + proc dlopen(path: cstring, mode: cint): LibHandle {. importc, header: "<dlfcn.h>".} proc dlsym(lib: LibHandle, name: cstring): ProcAddr {. importc, header: "<dlfcn.h>".} @@ -109,9 +112,30 @@ elif defined(windows) or defined(dos): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = getProcAddress(cast[THINSTANCE](lib), name) if result != nil: return - var decorated: array[250, char] + const decorated_length = 250 + var decorated: array[decorated_length, char] + decorated[0] = '_' + var m = 1 + while m < (decorated_length - 5): + if name[m - 1] == '\x00': break + decorated[m] = name[m - 1] + inc(m) + decorated[m] = '@' for i in countup(0, 50): - discard csprintf(decorated, "_%s@%ld", name, i*4) + var k = i * 4 + if k div 100 == 0: + if k div 10 == 0: + m = m + 1 + else: + m = m + 2 + else: + m = m + 3 + decorated[m + 1] = '\x00' + while true: + decorated[m] = chr(ord('0') + (k %% 10)) + dec(m) + k = k div 10 + if k == 0: break result = getProcAddress(cast[THINSTANCE](lib), decorated) if result != nil: return procAddrError(name) diff --git a/lib/system/endb.nim b/lib/system/endb.nim index b2cc5624b..d4d10a52c 100644 --- a/lib/system/endb.nim +++ b/lib/system/endb.nim @@ -329,14 +329,14 @@ proc dbgStackFrame(s: cstring, start: int, currFrame: PFrame) = proc readLine(f: File, line: var StaticStr): bool = while true: - var c = fgetc(f) + var c = c_fgetc(f) if c < 0'i32: if line.len > 0: break else: return false if c == 10'i32: break # LF if c == 13'i32: # CR - c = fgetc(f) # is the next char LF? - if c != 10'i32: ungetc(c, f) # no, put the character back + c = c_fgetc(f) # is the next char LF? + if c != 10'i32: discard c_ungetc(c, f) # no, put the character back break add line, chr(int(c)) result = true @@ -475,7 +475,7 @@ proc dbgWriteStackTrace(f: PFrame) = it = f i = 0 total = 0 - tempFrames: array [0..127, PFrame] + tempFrames: array[0..127, PFrame] # setup long head: while it != nil and i <= high(tempFrames)-firstCalls: tempFrames[i] = it diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 948f87410..b89729850 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -90,13 +90,13 @@ when defined(nativeStacktrace) and nativeStackTraceSupported: when not hasThreadSupport: var - tempAddresses: array [0..127, pointer] # should not be alloc'd on stack + tempAddresses: array[0..127, pointer] # should not be alloc'd on stack tempDlInfo: TDl_info proc auxWriteStackTraceWithBacktrace(s: var string) = when hasThreadSupport: var - tempAddresses: array [0..127, pointer] # but better than a threadvar + tempAddresses: array[0..127, pointer] # but better than a threadvar tempDlInfo: TDl_info # This is allowed to be expensive since it only happens during crashes # (but this way you don't need manual stack tracing) @@ -124,12 +124,12 @@ when defined(nativeStacktrace) and nativeStackTraceSupported: when not hasThreadSupport: var - tempFrames: array [0..127, PFrame] # should not be alloc'd on stack + tempFrames: array[0..127, PFrame] # should not be alloc'd on stack proc auxWriteStackTrace(f: PFrame, s: var string) = when hasThreadSupport: var - tempFrames: array [0..127, PFrame] # but better than a threadvar + tempFrames: array[0..127, PFrame] # but better than a threadvar const firstCalls = 32 var @@ -250,12 +250,12 @@ proc raiseExceptionAux(e: ref Exception) = inc L, slen template add(buf, s: expr) = xadd(buf, s, s.len) - var buf: array [0..2000, char] + var buf: array[0..2000, char] var L = 0 add(buf, "Error: unhandled exception: ") if not isNil(e.msg): add(buf, e.msg) add(buf, " [") - xadd(buf, e.name, c_strlen(e.name)) + xadd(buf, e.name, e.name.len) add(buf, "]\n") showErrorMessage(buf) quitOrDebug() diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 4f461b5c3..ba6b2fcf9 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf +# (c) Copyright 2016 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -9,13 +9,8 @@ # Garbage Collector # -# The basic algorithm is *Deferred Reference Counting* with cycle detection. -# This is achieved by combining a Deutsch-Bobrow garbage collector -# together with Christoper's partial mark-sweep garbage collector. -# -# Special care has been taken to avoid recursion as far as possible to avoid -# stack overflows when traversing deep datastructures. It is well-suited -# for soft real time applications (like games). +# Refcounting + Mark&Sweep. Complex algorithms avoided. +# Been there, done that, didn't work. when defined(nimCoroutines): import arch @@ -30,7 +25,7 @@ const # this seems to be a good value withRealTime = defined(useRealtimeGC) useMarkForDebug = defined(gcGenerational) - useBackupGc = false # use a simple M&S GC to collect + useBackupGc = true # use a simple M&S GC to collect # cycles instead of the complex # algorithm @@ -55,8 +50,8 @@ type WalkOp = enum waMarkGlobal, # part of the backup/debug mark&sweep waMarkPrecise, # part of the backup/debug mark&sweep - waZctDecRef, waPush, waCycleDecRef, waMarkGray, waScan, waScanBlack, - waCollectWhite #, waDebug + waZctDecRef, waPush + #, waDebug Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} # A ref type can have a finalizer that is called before the object's @@ -87,7 +82,6 @@ type idGenerator: int zct: CellSeq # the zero count table decStack: CellSeq # cells in the stack that are to decref again - cycleRoots: CellSet tempStack: CellSeq # temporary stack for recursion elimination recGcLock: int # prevent recursion via finalizers; no thread lock when withRealTime: @@ -96,6 +90,7 @@ type stat: GcStat when useMarkForDebug or useBackupGc: marked: CellSet + additionalRoots: CellSeq # dummy roots for GC_ref/unref when hasThreadSupport: toDispose: SharedList[pointer] @@ -136,9 +131,6 @@ proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) -proc canBeCycleRoot(c: PCell): bool {.inline.} = - result = ntfAcyclic notin c.typ.flags - proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging result = usrToCell(c).typ @@ -161,10 +153,10 @@ proc writeCell(msg: cstring, c: PCell) = var kind = -1 if c.typ != nil: kind = ord(c.typ.kind) when leakDetector: - c_fprintf(c_stdout, "[GC] %s: %p %d rc=%ld from %s(%ld)\n", + c_fprintf(stdout, "[GC] %s: %p %d rc=%ld from %s(%ld)\n", msg, c, kind, c.refcount shr rcShift, c.filename, c.line) else: - c_fprintf(c_stdout, "[GC] %s: %p %d rc=%ld; color=%ld\n", + c_fprintf(stdout, "[GC] %s: %p %d rc=%ld; color=%ld\n", msg, c, kind, c.refcount shr rcShift, c.color) template gcTrace(cell, state: expr): stmt {.immediate.} = @@ -200,14 +192,16 @@ proc prepareDealloc(cell: PCell) = (cast[Finalizer](cell.typ.finalizer))(cellToUsr(cell)) dec(gch.recGcLock) +template beforeDealloc(gch: var GcHeap; c: PCell; msg: typed) = + when false: + for i in 0..gch.decStack.len-1: + if gch.decStack.d[i] == c: + sysAssert(false, msg) + proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} = # we MUST access gch as a global here, because this crosses DLL boundaries! when hasThreadSupport and hasSharedHeap: acquireSys(HeapLock) - when cycleGC: - if c.color != rcPurple: - c.setColor(rcPurple) - incl(gch.cycleRoots, c) when hasThreadSupport and hasSharedHeap: releaseSys(HeapLock) @@ -224,22 +218,30 @@ proc decRef(c: PCell) {.inline.} = gcAssert(c.refcount >=% rcIncrement, "decRef") if --c.refcount: rtlAddZCT(c) - elif canbeCycleRoot(c): - # unfortunately this is necessary here too, because a cycle might just - # have been broken up and we could recycle it. - rtlAddCycleRoot(c) - #writeCell("decRef", c) proc incRef(c: PCell) {.inline.} = gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr") c.refcount = c.refcount +% rcIncrement # and not colorMask #writeCell("incRef", c) - if canbeCycleRoot(c): - rtlAddCycleRoot(c) -proc nimGCref(p: pointer) {.compilerProc, inline.} = incRef(usrToCell(p)) -proc nimGCunref(p: pointer) {.compilerProc, inline.} = decRef(usrToCell(p)) +proc nimGCref(p: pointer) {.compilerProc.} = + # we keep it from being collected by pretending it's not even allocated: + add(gch.additionalRoots, usrToCell(p)) + incRef(usrToCell(p)) + +proc nimGCunref(p: pointer) {.compilerProc.} = + let cell = usrToCell(p) + var L = gch.additionalRoots.len-1 + var i = L + let d = gch.additionalRoots.d + while i >= 0: + if d[i] == cell: + d[i] = d[L] + dec gch.additionalRoots.len + break + dec(i) + decRef(usrToCell(p)) proc GC_addCycleRoot*[T](p: ref T) {.inline.} = ## adds 'p' to the cycle candidate set for the cycle collector. It is @@ -306,10 +308,10 @@ proc initGC() = # init the rt init(gch.zct) init(gch.tempStack) - init(gch.cycleRoots) init(gch.decStack) when useMarkForDebug or useBackupGc: init(gch.marked) + init(gch.additionalRoots) when hasThreadSupport: gch.toDispose = initSharedList[pointer]() @@ -451,10 +453,13 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line + when leakDetector: + res.filename = nil + res.line = 0 + when not hasThreadSupport: + if framePtr != nil and framePtr.prev != nil: + res.filename = framePtr.prev.filename + res.line = framePtr.prev.line # refcount is zero, color is black, but mark it to be in the ZCT res.refcount = ZctFlag sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") @@ -502,10 +507,13 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line + when leakDetector: + res.filename = nil + res.line = 0 + when not hasThreadSupport: + if framePtr != nil and framePtr.prev != nil: + res.filename = framePtr.prev.filename + res.line = framePtr.prev.line res.refcount = rcIncrement # refcount is 1 sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") when logGC: writeCell("new cell", res) @@ -563,7 +571,7 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = d[j] = res break dec(j) - if canbeCycleRoot(ol): excl(gch.cycleRoots, ol) + beforeDealloc(gch, ol, "growObj stack trash") rawDealloc(gch.region, ol) else: # we split the old refcount in 2 parts. XXX This is still not entirely @@ -597,54 +605,12 @@ proc freeCyclicCell(gch: var GcHeap, c: PCell) = when logGC: writeCell("cycle collector dealloc cell", c) when reallyDealloc: sysAssert(allocInv(gch.region), "free cyclic cell") + beforeDealloc(gch, c, "freeCyclicCell: stack trash") rawDealloc(gch.region, c) else: gcAssert(c.typ != nil, "freeCyclicCell") zeroMem(c, sizeof(Cell)) -proc markGray(s: PCell) = - if s.color != rcGray: - setColor(s, rcGray) - forAllChildren(s, waMarkGray) - -proc scanBlack(s: PCell) = - s.setColor(rcBlack) - forAllChildren(s, waScanBlack) - -proc scan(s: PCell) = - if s.color == rcGray: - if s.refcount >=% rcIncrement: - scanBlack(s) - else: - s.setColor(rcWhite) - forAllChildren(s, waScan) - -proc collectWhite(s: PCell) = - # This is a hacky way to deal with the following problem (bug #1796) - # Consider this content in cycleRoots: - # x -> a; y -> a where 'a' is an acyclic object so not included in - # cycleRoots itself. Then 'collectWhite' used to free 'a' twice. The - # 'isAllocatedPtr' check prevents this. This also means we do not need - # to query 's notin gch.cycleRoots' at all. - if isAllocatedPtr(gch.region, s) and s.color == rcWhite: - s.setColor(rcBlack) - forAllChildren(s, waCollectWhite) - freeCyclicCell(gch, s) - -proc markRoots(gch: var GcHeap) = - var tabSize = 0 - for s in elements(gch.cycleRoots): - #writeCell("markRoot", s) - inc tabSize - if s.color == rcPurple and s.refcount >=% rcIncrement: - markGray(s) - else: - excl(gch.cycleRoots, s) - # (s.color == rcBlack and rc == 0) as 1 condition: - if s.refcount == 0: - freeCyclicCell(gch, s) - gch.stat.cycleTableSize = max(gch.stat.cycleTableSize, tabSize) - when useBackupGc: proc sweep(gch: var GcHeap) = for x in allObjects(gch.region): @@ -666,16 +632,8 @@ when useMarkForDebug or useBackupGc: proc markGlobals(gch: var GcHeap) = for i in 0 .. < globalMarkersLen: globalMarkers[i]() - - proc stackMarkS(gch: var GcHeap, p: pointer) {.inline.} = - # the addresses are not as cells on the stack, so turn them to cells: - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) - if c >% PageSize: - # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) - if objStart != nil: - markS(gch, objStart) + let d = gch.additionalRoots.d + for i in 0 .. < gch.additionalRoots.len: markS(gch, d[i]) when logGC: var @@ -697,7 +655,7 @@ when logGC: else: writeCell("cell {", s) forAllChildren(s, waDebug) - c_fprintf(c_stdout, "}\n") + c_fprintf(stdout, "}\n") proc doOperation(p: pointer, op: WalkOp) = if p == nil: return @@ -708,7 +666,7 @@ proc doOperation(p: pointer, op: WalkOp) = case op of waZctDecRef: #if not isAllocatedPtr(gch.region, c): - # c_fprintf(c_stdout, "[GC] decref bug: %p", c) + # c_fprintf(stdout, "[GC] decref bug: %p", c) gcAssert(isAllocatedPtr(gch.region, c), "decRef: waZctDecRef") gcAssert(c.refcount >=% rcIncrement, "doOperation 2") #c.refcount = c.refcount -% rcIncrement @@ -717,19 +675,6 @@ proc doOperation(p: pointer, op: WalkOp) = #if c.refcount <% rcIncrement: addZCT(gch.zct, c) of waPush: add(gch.tempStack, c) - of waCycleDecRef: - gcAssert(c.refcount >=% rcIncrement, "doOperation 3") - c.refcount = c.refcount -% rcIncrement - of waMarkGray: - gcAssert(c.refcount >=% rcIncrement, "waMarkGray") - c.refcount = c.refcount -% rcIncrement - markGray(c) - of waScan: scan(c) - of waScanBlack: - c.refcount = c.refcount +% rcIncrement - if c.color != rcBlack: - scanBlack(c) - of waCollectWhite: collectWhite(c) of waMarkGlobal: when useMarkForDebug or useBackupGc: when hasThreadSupport: @@ -748,14 +693,6 @@ proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = proc collectZCT(gch: var GcHeap): bool {.benign.} -when useMarkForDebug or useBackupGc: - proc markStackAndRegistersForSweep(gch: var GcHeap) {.noinline, cdecl, - benign.} - -proc collectRoots(gch: var GcHeap) = - for s in elements(gch.cycleRoots): - collectWhite(s) - proc collectCycles(gch: var GcHeap) = when hasThreadSupport: for c in gch.toDispose: @@ -764,33 +701,12 @@ proc collectCycles(gch: var GcHeap) = while gch.zct.len > 0: discard collectZCT(gch) when useBackupGc: cellsetReset(gch.marked) - markStackAndRegistersForSweep(gch) - markGlobals(gch) - sweep(gch) - else: - markRoots(gch) - # scanRoots: - for s in elements(gch.cycleRoots): scan(s) - collectRoots(gch) - - cellsetReset(gch.cycleRoots) - # alive cycles need to be kept in 'cycleRoots' if they are referenced - # from the stack; otherwise the write barrier will add the cycle root again - # anyway: - when false: var d = gch.decStack.d - var cycleRootsLen = 0 for i in 0..gch.decStack.len-1: - var c = d[i] - gcAssert isAllocatedPtr(gch.region, c), "addBackStackRoots" - gcAssert c.refcount >=% rcIncrement, "addBackStackRoots: dead cell" - if canBeCycleRoot(c): - #if c notin gch.cycleRoots: - inc cycleRootsLen - incl(gch.cycleRoots, c) - gcAssert c.typ != nil, "addBackStackRoots 2" - if cycleRootsLen != 0: - cfprintf(cstdout, "cycle roots: %ld\n", cycleRootsLen) + sysAssert isAllocatedPtr(gch.region, d[i]), "collectCycles" + markS(gch, d[i]) + markGlobals(gch) + sweep(gch) proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: @@ -812,31 +728,11 @@ proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = add(gch.decStack, cell) sysAssert(allocInv(gch.region), "gcMark end") -proc markThreadStacks(gch: var GcHeap) = - when hasThreadSupport and hasSharedHeap: - {.error: "not fully implemented".} - var it = threadList - while it != nil: - # mark registers: - for i in 0 .. high(it.registers): gcMark(gch, it.registers[i]) - var sp = cast[ByteAddress](it.stackBottom) - var max = cast[ByteAddress](it.stackTop) - # XXX stack direction? - # XXX unroll this loop: - while sp <=% max: - gcMark(gch, cast[ppointer](sp)[]) - sp = sp +% sizeof(pointer) - it = it.next - include gc_common proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = forEachStackSlot(gch, gcMark) -when useMarkForDebug or useBackupGc: - proc markStackAndRegistersForSweep(gch: var GcHeap) = - forEachStackSlot(gch, stackMarkS) - proc collectZCT(gch: var GcHeap): bool = # Note: Freeing may add child objects to the ZCT! So essentially we do # deep freeing, which is bad for incremental operation. In order to @@ -866,8 +762,6 @@ proc collectZCT(gch: var GcHeap): bool = # as this might be too slow. # In any case, it should be removed from the ZCT. But not # freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!** - when cycleGC: - if canbeCycleRoot(c): excl(gch.cycleRoots, c) when logGC: writeCell("zct dealloc cell", c) gcTrace(c, csZctFreed) # We are about to free the object, call the finalizer BEFORE its @@ -877,6 +771,7 @@ proc collectZCT(gch: var GcHeap): bool = forAllChildren(c, waZctDecRef) when reallyDealloc: sysAssert(allocInv(gch.region), "collectZCT: rawDealloc") + beforeDealloc(gch, c, "collectZCT: stack trash") rawDealloc(gch.region, c) else: sysAssert(c.typ != nil, "collectZCT 2") @@ -915,7 +810,6 @@ proc collectCTBody(gch: var GcHeap) = sysAssert(gch.decStack.len == 0, "collectCT") prepareForInteriorPointerChecking(gch.region) markStackAndRegisters(gch) - markThreadStacks(gch) gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len) inc(gch.stat.stackScans) if collectZCT(gch): @@ -935,12 +829,7 @@ proc collectCTBody(gch: var GcHeap) = gch.stat.maxPause = max(gch.stat.maxPause, duration) when defined(reportMissedDeadlines): if gch.maxPause > 0 and duration > gch.maxPause: - c_fprintf(c_stdout, "[GC] missed deadline: %ld\n", duration) - -when useMarkForDebug or useBackupGc: - proc markForDebug(gch: var GcHeap) = - markStackAndRegistersForSweep(gch) - markGlobals(gch) + c_fprintf(stdout, "[GC] missed deadline: %ld\n", duration) when defined(nimCoroutines): proc currentStackSizes(): int = @@ -980,7 +869,19 @@ when withRealTime: collectCTBody(gch) release(gch) - proc GC_step*(us: int, strongAdvice = false) = GC_step(gch, us, strongAdvice) + proc GC_step*(us: int, strongAdvice = false, stackSize = -1) {.noinline.} = + var stackTop {.volatile.}: pointer + let prevStackBottom = gch.stackBottom + if stackSize >= 0: + stackTop = addr(stackTop) + when stackIncreases: + gch.stackBottom = cast[pointer]( + cast[ByteAddress](stackTop) - sizeof(pointer) * 6 - stackSize) + else: + gch.stackBottom = cast[pointer]( + cast[ByteAddress](stackTop) + sizeof(pointer) * 6 + stackSize) + GC_step(gch, us, strongAdvice) + gch.stackBottom = prevStackBottom when not defined(useNimRtl): proc GC_disable() = @@ -1023,7 +924,7 @@ when not defined(useNimRtl): "[GC] max threshold: " & $gch.stat.maxThreshold & "\n" & "[GC] zct capacity: " & $gch.zct.cap & "\n" & "[GC] max cycle table size: " & $gch.stat.cycleTableSize & "\n" & - "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) + "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) & "\n" when defined(nimCoroutines): result = result & "[GC] number of stacks: " & $gch.stack.len & "\n" for stack in items(gch.stack): diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim index 6c44d509e..089c9c915 100644 --- a/lib/system/gc2.nim +++ b/lib/system/gc2.nim @@ -97,6 +97,8 @@ type additionalRoots: CellSeq # dummy roots for GC_ref/unref spaceIter: ObjectSpaceIter dumpHeapFile: File # File that is used for GC_dumpHeap + when hasThreadSupport: + toDispose: SharedList[pointer] var gch {.rtlThreadVar.}: GcHeap @@ -119,6 +121,8 @@ proc initGC() = init(gch.decStack) init(gch.additionalRoots) init(gch.greyStack) + when hasThreadSupport: + gch.toDispose = initSharedList[pointer]() # Which color to use for new objects is tricky: When we're marking, # they have to be *white* so that everything is marked that is only @@ -185,7 +189,7 @@ proc writeCell(file: File; msg: cstring, c: PCell) = msg, id, kind, c.refcount shr rcShift, col) proc writeCell(msg: cstring, c: PCell) = - c_stdout.writeCell(msg, c) + stdout.writeCell(msg, c) proc myastToStr[T](x: T): string {.magic: "AstToStr", noSideEffect.} @@ -259,7 +263,7 @@ proc nimGCunref(p: pointer) {.compilerProc.} = template markGrey(x: PCell) = if x.color != 1-gch.black and gch.phase == Phase.Marking: if not isAllocatedPtr(gch.region, x): - c_fprintf(c_stdout, "[GC] markGrey proc: %p\n", x) + c_fprintf(stdout, "[GC] markGrey proc: %p\n", x) #GC_dumpHeap() sysAssert(false, "wtf") x.setColor(rcGrey) @@ -675,7 +679,7 @@ proc sweep(gch: var GcHeap): bool = takeStartTime(100) #echo "loop start" let white = 1-gch.black - #cfprintf(cstdout, "black is %d\n", black) + #c_fprintf(stdout, "black is %d\n", black) while true: let x = allObjectsAsProc(gch.region, addr gch.spaceIter) if gch.spaceIter.state < 0: break @@ -715,7 +719,7 @@ proc markIncremental(gch: var GcHeap): bool = while L[] > 0: var c = gch.greyStack.d[0] if not isAllocatedPtr(gch.region, c): - c_fprintf(c_stdout, "[GC] not allocated anymore: %p\n", c) + c_fprintf(stdout, "[GC] not allocated anymore: %p\n", c) #GC_dumpHeap() sysAssert(false, "wtf") @@ -760,7 +764,7 @@ proc doOperation(p: pointer, op: WalkOp) = case op of waZctDecRef: #if not isAllocatedPtr(gch.region, c): - # c_fprintf(c_stdout, "[GC] decref bug: %p", c) + # c_fprintf(stdout, "[GC] decref bug: %p", c) gcAssert(isAllocatedPtr(gch.region, c), "decRef: waZctDecRef") gcAssert(c.refcount >=% rcIncrement, "doOperation 2") #c.refcount = c.refcount -% rcIncrement @@ -779,14 +783,14 @@ proc doOperation(p: pointer, op: WalkOp) = else: #gcAssert(isAllocatedPtr(gch.region, c), "doOperation: waMarkGlobal") if not isAllocatedPtr(gch.region, c): - c_fprintf(c_stdout, "[GC] not allocated anymore: MarkGlobal %p\n", c) + c_fprintf(stdout, "[GC] not allocated anymore: MarkGlobal %p\n", c) #GC_dumpHeap() sysAssert(false, "wtf") handleRoot() discard allocInv(gch.region) of waMarkGrey: if not isAllocatedPtr(gch.region, c): - c_fprintf(c_stdout, "[GC] not allocated anymore: MarkGrey %p\n", c) + c_fprintf(stdout, "[GC] not allocated anymore: MarkGrey %p\n", c) #GC_dumpHeap() sysAssert(false, "wtf") if c.color == 1-gch.black: @@ -800,6 +804,10 @@ proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = proc collectZCT(gch: var GcHeap): bool {.benign.} proc collectCycles(gch: var GcHeap): bool = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) + # ensure the ZCT 'color' is not used: while gch.zct.len > 0: discard collectZCT(gch) @@ -808,7 +816,7 @@ proc collectCycles(gch: var GcHeap): bool = gch.phase = Phase.Marking markGlobals(gch) - cfprintf(stdout, "collectCycles: introduced bug E %ld\n", gch.phase) + c_fprintf(stdout, "collectCycles: introduced bug E %ld\n", gch.phase) discard allocInv(gch.region) of Phase.Marking: # since locals do not have a write barrier, we need @@ -922,7 +930,7 @@ proc collectCTBody(gch: var GcHeap) = gch.stat.maxPause = max(gch.stat.maxPause, duration) when defined(reportMissedDeadlines): if gch.maxPause > 0 and duration > gch.maxPause: - c_fprintf(c_stdout, "[GC] missed deadline: %ld\n", duration) + c_fprintf(stdout, "[GC] missed deadline: %ld\n", duration) when defined(nimCoroutines): proc currentStackSizes(): int = @@ -956,7 +964,19 @@ when withRealTime: strongAdvice: collectCTBody(gch) - proc GC_step*(us: int, strongAdvice = false) = GC_step(gch, us, strongAdvice) + proc GC_step*(us: int, strongAdvice = false, stackSize = -1) {.noinline.} = + var stackTop {.volatile.}: pointer + let prevStackBottom = gch.stackBottom + if stackSize >= 0: + stackTop = addr(stackTop) + when stackIncreases: + gch.stackBottom = cast[pointer]( + cast[ByteAddress](stackTop) - sizeof(pointer) * 6 - stackSize) + else: + gch.stackBottom = cast[pointer]( + cast[ByteAddress](stackTop) + sizeof(pointer) * 6 + stackSize) + GC_step(gch, us, strongAdvice) + gch.stackBottom = prevStackBottom when not defined(useNimRtl): proc GC_disable() = diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index 4807bb6f8..7a1b88c84 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -63,7 +63,7 @@ when defined(nimCoroutines): stack.next = gch.stack gch.stack.prev = stack gch.stack = stack - # c_fprintf(c_stdout, "[GC] added stack 0x%016X\n", starts) + # c_fprintf(stdout, "[GC] added stack 0x%016X\n", starts) proc GC_removeStack*(starts: pointer) {.cdecl, exportc.} = var stack = gch.stack @@ -143,7 +143,7 @@ else: when not defined(useNimRtl): {.push stack_trace: off.} proc setStackBottom(theStackBottom: pointer) = - #c_fprintf(c_stdout, "stack bottom: %p;\n", theStackBottom) + #c_fprintf(stdout, "stack bottom: %p;\n", theStackBottom) # the first init must be the one that defines the stack bottom: when defined(nimCoroutines): GC_addStack(theStackBottom) @@ -152,7 +152,7 @@ when not defined(useNimRtl): else: var a = cast[ByteAddress](theStackBottom) # and not PageMask - PageSize*2 var b = cast[ByteAddress](gch.stackBottom) - #c_fprintf(c_stdout, "old: %p new: %p;\n",gch.stackBottom,theStackBottom) + #c_fprintf(stdout, "old: %p new: %p;\n",gch.stackBottom,theStackBottom) when stackIncreases: gch.stackBottom = cast[pointer](min(a, b)) else: @@ -239,7 +239,7 @@ else: # We use a jmp_buf buffer that is in the C stack. # Used to traverse the stack and registers assuming # that 'setjmp' will save registers in the C stack. - type PStackSlice = ptr array [0..7, pointer] + type PStackSlice = ptr array[0..7, pointer] var registers {.noinit.}: Registers getRegisters(registers) for i in registers.low .. registers.high: @@ -277,7 +277,7 @@ else: # We use a jmp_buf buffer that is in the C stack. # Used to traverse the stack and registers assuming # that 'setjmp' will save registers in the C stack. - type PStackSlice = ptr array [0..7, pointer] + type PStackSlice = ptr array[0..7, pointer] var registers {.noinit.}: C_JmpBuf if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. var max = cast[ByteAddress](gch.stackBottom) diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index c764571b1..ec69f6e5f 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -28,6 +28,9 @@ template mulThreshold(x): expr {.immediate.} = x * 2 when defined(memProfiler): proc nimProfile(requestedSize: int) + +when hasThreadSupport: + import sharedlist type WalkOp = enum diff --git a/lib/system/gc_stack.nim b/lib/system/gc_stack.nim index 3a5c5594a..5f72b8959 100644 --- a/lib/system/gc_stack.nim +++ b/lib/system/gc_stack.nim @@ -8,7 +8,23 @@ # "Stack GC" for embedded devices or ultra performance requirements. -include osalloc +when defined(nimphpext): + proc roundup(x, v: int): int {.inline.} = + result = (x + (v-1)) and not (v-1) + proc emalloc(size: int): pointer {.importc: "_emalloc".} + proc efree(mem: pointer) {.importc: "_efree".} + + proc osAllocPages(size: int): pointer {.inline.} = + emalloc(size) + + proc osTryAllocPages(size: int): pointer {.inline.} = + emalloc(size) + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + efree(p) + +else: + include osalloc # We manage memory as a thread local stack. Since the allocation pointer # is detached from the control flow pointer, this model is vastly more @@ -36,33 +52,34 @@ type BaseChunk = object next: Chunk size: int - head, last: ptr ObjHeader # first and last object in chunk that + head, tail: ptr ObjHeader # first and last object in chunk that # has a finalizer attached to it type StackPtr = object - chunk: pointer + bump: pointer remaining: int current: Chunk MemRegion* = object remaining: int - chunk: pointer - head, last: Chunk + bump: pointer + head, tail: Chunk nextChunkSize, totalSize: int hole: ptr Hole # we support individual freeing - lock: SysLock + when hasThreadSupport: + lock: SysLock var - region {.threadVar.}: MemRegion + tlRegion {.threadVar.}: MemRegion template withRegion*(r: MemRegion; body: untyped) = - let oldRegion = region - region = r + let oldRegion = tlRegion + tlRegion = r try: body finally: - region = oldRegion + tlRegion = oldRegion template inc(p: pointer, s: int) = p = cast[pointer](cast[int](p) +% s) @@ -71,7 +88,7 @@ template `+!`(p: pointer, s: int): pointer = cast[pointer](cast[int](p) +% s) template `-!`(p: pointer, s: int): pointer = - cast[pointer](cast[int](p) +% s) + cast[pointer](cast[int](p) -% s) proc allocSlowPath(r: var MemRegion; size: int) = # we need to ensure that the underlying linked list @@ -84,7 +101,7 @@ proc allocSlowPath(r: var MemRegion; size: int) = r.nextChunkSize = if r.totalSize < 64 * 1024: PageSize*4 else: r.nextChunkSize*2 - var s = align(size+sizeof(BaseChunk), PageSize) + var s = roundup(size+sizeof(BaseChunk), PageSize) var fresh: Chunk if s > r.nextChunkSize: fresh = cast[Chunk](osAllocPages(s)) @@ -97,22 +114,26 @@ proc allocSlowPath(r: var MemRegion; size: int) = else: s = r.nextChunkSize fresh.size = s - fresh.final = nil - r.totalSize += s - let old = r.last + fresh.head = nil + fresh.tail = nil + fresh.next = nil + inc r.totalSize, s + let old = r.tail if old == nil: r.head = fresh else: - r.last.next = fresh - r.chunk = fresh +! sizeof(BaseChunk) - r.last = fresh + r.tail.next = fresh + r.bump = fresh +! sizeof(BaseChunk) + r.tail = fresh r.remaining = s - sizeof(BaseChunk) proc alloc(r: var MemRegion; size: int): pointer {.inline.} = - if unlikely(r.remaining < size): allocSlowPath(r, size) + if size > r.remaining: + allocSlowPath(r, size) + sysAssert(size <= r.remaining, "size <= r.remaining") dec(r.remaining, size) - result = r.chunk - inc r.chunk, size + result = r.bump + inc r.bump, size proc runFinalizers(c: Chunk) = var it = c.head @@ -120,228 +141,245 @@ proc runFinalizers(c: Chunk) = # indivually freed objects with finalizer stay in the list, but # their typ is nil then: if it.typ != nil and it.typ.finalizer != nil: - (cast[Finalizer](cell.typ.finalizer))(cell+!sizeof(ObjHeader)) - it = it.next + (cast[Finalizer](it.typ.finalizer))(it+!sizeof(ObjHeader)) + it = it.nextFinal -proc dealloc(r: var MemRegion; p: pointer) = - let it = p-!sizeof(ObjHeader) - if it.typ != nil and it.typ.finalizer != nil: - (cast[Finalizer](cell.typ.finalizer))(p) - it.typ = nil +when false: + proc dealloc(r: var MemRegion; p: pointer) = + let it = cast[ptr ObjHeader](p-!sizeof(ObjHeader)) + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](it.typ.finalizer))(p) + it.typ = nil -proc deallocAll(head: Chunk) = +proc deallocAll(r: var MemRegion; head: Chunk) = var it = head while it != nil: + let nxt = it.next runFinalizers(it) + dec r.totalSize, it.size osDeallocPages(it, it.size) - it = it.next + it = nxt proc deallocAll*(r: var MemRegion) = - deallocAll(r.head) + deallocAll(r, r.head) zeroMem(addr r, sizeof r) proc obstackPtr*(r: MemRegion): StackPtr = - result.chunk = r.chunk + result.bump = r.bump result.remaining = r.remaining - result.current = r.last + result.current = r.tail + +template computeRemaining(r): untyped = + r.tail.size -% (cast[int](r.bump) -% cast[int](r.tail)) -proc setObstackPtr*(r: MemRegion; sp: StackPtr) = +proc setObstackPtr*(r: var MemRegion; sp: StackPtr) = # free everything after 'sp': if sp.current != nil: - deallocAll(sp.current.next) - r.chunk = sp.chunk + deallocAll(r, sp.current.next) + sp.current.next = nil + else: + deallocAll(r, r.head) + r.head = nil + r.bump = sp.bump + r.tail = sp.current r.remaining = sp.remaining - r.last = sp.current + +proc obstackPtr*(): StackPtr = tlRegion.obstackPtr() +proc setObstackPtr*(sp: StackPtr) = tlRegion.setObstackPtr(sp) +proc deallocAll*() = tlRegion.deallocAll() + +proc deallocOsPages(r: var MemRegion) = r.deallocAll() proc joinRegion*(dest: var MemRegion; src: MemRegion) = # merging is not hard. if dest.head.isNil: dest.head = src.head else: - dest.last.next = src.head - dest.last = src.last - dest.chunk = src.chunk + dest.tail.next = src.head + dest.tail = src.tail + dest.bump = src.bump dest.remaining = src.remaining dest.nextChunkSize = max(dest.nextChunkSize, src.nextChunkSize) - dest.totalSize += src.totalSize - if dest.hole.size < src.hole.size: - dest.hole = src.hole + inc dest.totalSize, src.totalSize proc isOnHeap*(r: MemRegion; p: pointer): bool = - # the last chunk is the largest, so check it first. It's also special + # the tail chunk is the largest, so check it first. It's also special # in that contains the current bump pointer: - if r.last >= p and p < r.chunk: + if r.tail >= p and p < r.bump: return true var it = r.head - while it != r.last: + while it != r.tail: if it >= p and p <= it+!it.size: return true it = it.next -proc isInteriorPointer(r: MemRegion; p: pointer): pointer = - discard " we cannot patch stack pointers anyway!" +when false: + # essential feature for later: copy data over from one region to another -type - PointerStackChunk = object - next, prev: ptr PointerStackChunk - len: int - data: array[128, pointer] + proc isInteriorPointer(r: MemRegion; p: pointer): pointer = + discard " we cannot patch stack pointers anyway!" -template head(s: PointerStackChunk): untyped = s.prev -template tail(s: PointerStackChunk): untyped = s.next + type + PointerStackChunk = object + next, prev: ptr PointerStackChunk + len: int + data: array[128, pointer] -include chains + template head(s: PointerStackChunk): untyped = s.prev + template tail(s: PointerStackChunk): untyped = s.next -proc push(r: var MemRegion; s: var PointerStackChunk; x: pointer) = - if s.len < high(s.data): - s.data[s.len] = x - inc s.len - else: - let fresh = cast[ptr PointerStackChunk](alloc(r, sizeof(PointerStackChunk))) - fresh.len = 1 - fresh.data[0] = x - fresh.next = nil - fresh.prev = nil - append(s, fresh) - - -proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; - dest, src: pointer, mt: PNimType) {.benign.} -proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; - dest, src: pointer, n: ptr TNimNode) {.benign.} = - var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) - case n.kind - of nkSlot: - genericDeepCopyAux(cast[pointer](d +% n.offset), - cast[pointer](s +% n.offset), n.typ) - of nkList: - for i in 0..n.len-1: - genericDeepCopyAux(dest, src, n.sons[i]) - of nkCase: - var dd = selectBranch(dest, n) - var m = selectBranch(src, n) - # reset if different branches are in use; note different branches also - # imply that's not self-assignment (``x = x``)! - if m != dd and dd != nil: - genericResetAux(dest, dd) - copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), - n.typ.size) - if m != nil: - genericDeepCopyAux(dest, src, m) - of nkNone: sysAssert(false, "genericDeepCopyAux") - -proc copyDeepString(dr: var MemRegion; stack: var PointerStackChunk; src: NimString): NimString {.inline.} = - result = rawNewStringNoInit(dr, src.len) - result.len = src.len - c_memcpy(result.data, src.data, src.len + 1) - -proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; - dest, src: pointer, mt: PNimType) = - var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) - sysAssert(mt != nil, "genericDeepCopyAux 2") - case mt.kind - of tyString: - var x = cast[PPointer](dest) - var s2 = cast[PPointer](s)[] - if s2 == nil: - x[] = nil - else: - x[] = copyDeepString(cast[NimString](s2)) - of tySequence: - var s2 = cast[PPointer](src)[] - var seq = cast[PGenericSeq](s2) - var x = cast[PPointer](dest) - if s2 == nil: - x[] = nil - return - sysAssert(dest != nil, "genericDeepCopyAux 3") - x[] = newSeq(mt, seq.len) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) - for i in 0..seq.len-1: - genericDeepCopyAux(dr, stack, - cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), - cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% - GenericSeqSize), - mt.base) - of tyObject: - # we need to copy m_type field for tyObject, as it could be empty for - # sequence reallocations: - var pint = cast[ptr PNimType](dest) - pint[] = cast[ptr PNimType](src)[] - if mt.base != nil: - genericDeepCopyAux(dr, stack, dest, src, mt.base) - genericDeepCopyAux(dr, stack, dest, src, mt.node) - of tyTuple: - genericDeepCopyAux(dr, stack, dest, src, mt.node) - of tyArray, tyArrayConstr: - for i in 0..(mt.size div mt.base.size)-1: - genericDeepCopyAux(dr, stack, - cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base) - of tyRef: - let s2 = cast[PPointer](src)[] - if s2 == nil: - cast[PPointer](dest)[] = nil + include chains + + proc push(r: var MemRegion; s: var PointerStackChunk; x: pointer) = + if s.len < high(s.data): + s.data[s.len] = x + inc s.len else: - # we modify the header of the cell temporarily; instead of the type - # field we store a forwarding pointer. XXX This is bad when the cloning - # fails due to OOM etc. - let x = usrToCell(s2) - let forw = cast[int](x.typ) - if (forw and 1) == 1: - # we stored a forwarding pointer, so let's use that: - let z = cast[pointer](forw and not 1) - unsureAsgnRef(cast[PPointer](dest), z) + let fresh = cast[ptr PointerStackChunk](alloc(r, sizeof(PointerStackChunk))) + fresh.len = 1 + fresh.data[0] = x + fresh.next = nil + fresh.prev = nil + append(s, fresh) + + + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, mt: PNimType) {.benign.} + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, n: ptr TNimNode) {.benign.} = + var + d = cast[ByteAddress](dest) + s = cast[ByteAddress](src) + case n.kind + of nkSlot: + genericDeepCopyAux(cast[pointer](d +% n.offset), + cast[pointer](s +% n.offset), n.typ) + of nkList: + for i in 0..n.len-1: + genericDeepCopyAux(dest, src, n.sons[i]) + of nkCase: + var dd = selectBranch(dest, n) + var m = selectBranch(src, n) + # reset if different branches are in use; note different branches also + # imply that's not self-assignment (``x = x``)! + if m != dd and dd != nil: + genericResetAux(dest, dd) + copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), + n.typ.size) + if m != nil: + genericDeepCopyAux(dest, src, m) + of nkNone: sysAssert(false, "genericDeepCopyAux") + + proc copyDeepString(dr: var MemRegion; stack: var PointerStackChunk; src: NimString): NimString {.inline.} = + result = rawNewStringNoInit(dr, src.len) + result.len = src.len + copyMem(result.data, src.data, src.len + 1) + + proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, mt: PNimType) = + var + d = cast[ByteAddress](dest) + s = cast[ByteAddress](src) + sysAssert(mt != nil, "genericDeepCopyAux 2") + case mt.kind + of tyString: + var x = cast[PPointer](dest) + var s2 = cast[PPointer](s)[] + if s2 == nil: + x[] = nil else: - let realType = x.typ - let z = newObj(realType, realType.base.size) - - unsureAsgnRef(cast[PPointer](dest), z) - x.typ = cast[PNimType](cast[int](z) or 1) - genericDeepCopyAux(dr, stack, z, s2, realType.base) - x.typ = realType - else: - copyMem(dest, src, mt.size) - -proc joinAliveDataFromRegion*(dest: var MemRegion; src: var MemRegion; - root: pointer): pointer = - # we mark the alive data and copy only alive data over to 'dest'. - # This is O(liveset) but it nicely compacts memory, so it's fine. - # We use the 'typ' field as a forwarding pointer. The forwarding - # pointers have bit 0 set, so we can disambiguate them. - # We allocate a temporary stack in 'src' that we later free: - var s: PointerStackChunk - s.len = 1 - s.data[0] = root - while s.len > 0: - var p: pointer - if s.tail == nil: - p = s.data[s.len-1] - dec s.len + x[] = copyDeepString(cast[NimString](s2)) + of tySequence: + var s2 = cast[PPointer](src)[] + var seq = cast[PGenericSeq](s2) + var x = cast[PPointer](dest) + if s2 == nil: + x[] = nil + return + sysAssert(dest != nil, "genericDeepCopyAux 3") + x[] = newSeq(mt, seq.len) + var dst = cast[ByteAddress](cast[PPointer](dest)[]) + for i in 0..seq.len-1: + genericDeepCopyAux(dr, stack, + cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), + cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% + GenericSeqSize), + mt.base) + of tyObject: + # we need to copy m_type field for tyObject, as it could be empty for + # sequence reallocations: + var pint = cast[ptr PNimType](dest) + pint[] = cast[ptr PNimType](src)[] + if mt.base != nil: + genericDeepCopyAux(dr, stack, dest, src, mt.base) + genericDeepCopyAux(dr, stack, dest, src, mt.node) + of tyTuple: + genericDeepCopyAux(dr, stack, dest, src, mt.node) + of tyArray, tyArrayConstr: + for i in 0..(mt.size div mt.base.size)-1: + genericDeepCopyAux(dr, stack, + cast[pointer](d +% i*% mt.base.size), + cast[pointer](s +% i*% mt.base.size), mt.base) + of tyRef: + let s2 = cast[PPointer](src)[] + if s2 == nil: + cast[PPointer](dest)[] = nil + else: + # we modify the header of the cell temporarily; instead of the type + # field we store a forwarding pointer. XXX This is bad when the cloning + # fails due to OOM etc. + let x = usrToCell(s2) + let forw = cast[int](x.typ) + if (forw and 1) == 1: + # we stored a forwarding pointer, so let's use that: + let z = cast[pointer](forw and not 1) + unsureAsgnRef(cast[PPointer](dest), z) + else: + let realType = x.typ + let z = newObj(realType, realType.base.size) + + unsureAsgnRef(cast[PPointer](dest), z) + x.typ = cast[PNimType](cast[int](z) or 1) + genericDeepCopyAux(dr, stack, z, s2, realType.base) + x.typ = realType else: - p = s.tail.data[s.tail.len-1] - dec s.tail.len - if s.tail.len == 0: - unlink(s, s.tail) + copyMem(dest, src, mt.size) + + proc joinAliveDataFromRegion*(dest: var MemRegion; src: var MemRegion; + root: pointer): pointer = + # we mark the alive data and copy only alive data over to 'dest'. + # This is O(liveset) but it nicely compacts memory, so it's fine. + # We use the 'typ' field as a forwarding pointer. The forwarding + # pointers have bit 0 set, so we can disambiguate them. + # We allocate a temporary stack in 'src' that we later free: + var s: PointerStackChunk + s.len = 1 + s.data[0] = root + while s.len > 0: + var p: pointer + if s.tail == nil: + p = s.data[s.len-1] + dec s.len + else: + p = s.tail.data[s.tail.len-1] + dec s.tail.len + if s.tail.len == 0: + unlink(s, s.tail) proc rawNewObj(r: var MemRegion, typ: PNimType, size: int): pointer = var res = cast[ptr ObjHeader](alloc(r, size + sizeof(ObjHeader))) res.typ = typ if typ.finalizer != nil: - res.nextFinal = r.chunk.head - r.chunk.head = res + res.nextFinal = r.head.head + r.head.head = res result = res +! sizeof(ObjHeader) proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, region) + result = rawNewObj(tlRegion, typ, size) zeroMem(result, size) when defined(memProfiler): nimProfile(size) proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, region) + result = rawNewObj(tlRegion, typ, size) when defined(memProfiler): nimProfile(size) proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = @@ -351,7 +389,7 @@ proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = cast[PGenericSeq](result).reserved = len proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, gch) + result = rawNewObj(tlRegion, typ, size) zeroMem(result, size) proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = @@ -360,23 +398,70 @@ proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len -proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = - collectCT(gch) - var ol = usrToCell(old) - sysAssert(ol.typ != nil, "growObj: 1") - gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") - - var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) - var elemSize = 1 - if ol.typ.kind != tyString: elemSize = ol.typ.base.size - - var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize - copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), - newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") - result = cellToUsr(res) +proc growObj(region: var MemRegion; old: pointer, newsize: int): pointer = + let typ = cast[ptr ObjHeader](old -! sizeof(ObjHeader)).typ + result = rawNewObj(region, typ, newsize) + let elemSize = if typ.kind == tyString: 1 else: typ.base.size + let oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize + copyMem(result, old, oldsize) + zeroMem(result +! oldsize, newsize-oldsize) proc growObj(old: pointer, newsize: int): pointer {.rtl.} = - result = growObj(old, newsize, region) - + result = growObj(tlRegion, old, newsize) + +proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src +proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src + +proc alloc(size: Natural): pointer = + result = c_malloc(size) + if result == nil: raiseOutOfMem() +proc alloc0(size: Natural): pointer = + result = alloc(size) + zeroMem(result, size) +proc realloc(p: pointer, newsize: Natural): pointer = + result = c_realloc(p, newsize) + if result == nil: raiseOutOfMem() +proc dealloc(p: pointer) = c_free(p) + +proc alloc0(r: var MemRegion; size: Natural): pointer = + # ignore the region. That is correct for the channels module + # but incorrect in general. XXX + result = alloc0(size) + +proc dealloc(r: var MemRegion; p: pointer) = dealloc(p) + +proc allocShared(size: Natural): pointer = + result = c_malloc(size) + if result == nil: raiseOutOfMem() +proc allocShared0(size: Natural): pointer = + result = alloc(size) + zeroMem(result, size) +proc reallocShared(p: pointer, newsize: Natural): pointer = + result = c_realloc(p, newsize) + if result == nil: raiseOutOfMem() +proc deallocShared(p: pointer) = c_free(p) + +when hasThreadSupport: + proc getFreeSharedMem(): int = 0 + proc getTotalSharedMem(): int = 0 + proc getOccupiedSharedMem(): int = 0 + +proc GC_disable() = discard +proc GC_enable() = discard +proc GC_fullCollect() = discard +proc GC_setStrategy(strategy: GC_Strategy) = discard +proc GC_enableMarkAndSweep() = discard +proc GC_disableMarkAndSweep() = discard +proc GC_getStatistics(): string = return "" + +proc getOccupiedMem(): int = + result = tlRegion.totalSize - tlRegion.remaining +proc getFreeMem(): int = tlRegion.remaining +proc getTotalMem(): int = + result = tlRegion.totalSize + +proc setStackBottom(theStackBottom: pointer) = discard diff --git a/lib/system/hti.nim b/lib/system/hti.nim index bfb13059e..984f888cb 100644 --- a/lib/system/hti.nim +++ b/lib/system/hti.nim @@ -71,7 +71,7 @@ type typ: ptr TNimType name: cstring len: int - sons: ptr array [0..0x7fff, ptr TNimNode] + sons: ptr array[0..0x7fff, ptr TNimNode] TNimTypeFlag = enum ntfNoRefs = 0, # type contains no tyRef, tySequence, tyString diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim index 3caeefcbc..f9e6754ef 100644 --- a/lib/system/inclrtl.nim +++ b/lib/system/inclrtl.nim @@ -19,6 +19,11 @@ when not defined(nimNewShared): {.pragma: gcsafe.} +when not defined(nimImmediateDeprecated): + {.pragma: oldimmediate, immediate.} +else: + {.pragma: oldimmediate.} + when defined(createNimRtl): when defined(useNimRtl): {.error: "Cannot create and use nimrtl at the same time!".} diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 1b98883b9..9c8a18bfe 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -65,7 +65,7 @@ proc auxWriteStackTrace(f: PCallFrame): string = it = f i = 0 total = 0 - tempFrames: array [0..63, TempFrame] + tempFrames: array[0..63, TempFrame] while it != nil and i <= high(tempFrames): tempFrames[i].procname = it.procname tempFrames[i].line = it.line @@ -97,6 +97,8 @@ proc rawWriteStackTrace(): string = else: result = "No stack traceback available\n" +proc getStackTrace*(): string = rawWriteStackTrace() + proc unhandledException(e: ref Exception) {. compilerproc, asmNoStackFrame.} = when NimStackTrace: @@ -119,7 +121,10 @@ proc raiseException(e: ref Exception, ename: cstring) {. when not defined(noUnhandledHandler): if excHandler == 0: unhandledException(e) - asm "throw `e`;" + when defined(nimphp): + asm """throw new Exception($`e`["message"]);""" + else: + asm "throw `e`;" proc reraiseException() {.compilerproc, asmNoStackFrame.} = if lastJSError == nil: @@ -243,8 +248,12 @@ proc toJSStr(s: string): cstring {.asmNoStackFrame, compilerproc.} = for (var i = 0; i < len; ++i) { if (nonAsciiPart !== null) { var offset = (i - nonAsciiOffset) * 2; + var code = `s`[i].toString(16); + if (code.length == 1) { + code = "0"+code; + } nonAsciiPart[offset] = "%"; - nonAsciiPart[offset + 1] = `s`[i].toString(16); + nonAsciiPart[offset + 1] = code; } else if (`s`[i] < 128) asciiPart[i] = fcc(`s`[i]); @@ -729,16 +738,19 @@ proc genericReset(x: JSRef, ti: PNimType): JSRef {.compilerproc.} = else: discard -proc arrayConstr(len: int, value: JSRef, typ: PNimType): JSRef {. - asmNoStackFrame, compilerproc.} = - # types are fake - when defined(nimphp): +when defined(nimphp): + proc arrayConstr(len: int, value: string, typ: string): JSRef {. + asmNoStackFrame, compilerproc.} = + # types are fake asm """ $result = array(); for ($i = 0; $i < `len`; $i++) $result[] = `value`; return $result; """ - else: +else: + proc arrayConstr(len: int, value: JSRef, typ: PNimType): JSRef {. + asmNoStackFrame, compilerproc.} = + # types are fake asm """ var result = new Array(`len`); for (var i = 0; i < `len`; ++i) result[i] = nimCopy(null, `value`, `typ`); diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 5e576f0a3..186349152 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -300,7 +300,7 @@ elif defined(gogc): proc setStackBottom(theStackBottom: pointer) = discard proc alloc(size: Natural): pointer = - result = cmalloc(size) + result = c_malloc(size) if result == nil: raiseOutOfMem() proc alloc0(size: Natural): pointer = @@ -308,13 +308,13 @@ elif defined(gogc): zeroMem(result, size) proc realloc(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) + result = c_realloc(p, newsize) if result == nil: raiseOutOfMem() - proc dealloc(p: pointer) = cfree(p) + proc dealloc(p: pointer) = c_free(p) proc allocShared(size: Natural): pointer = - result = cmalloc(size) + result = c_malloc(size) if result == nil: raiseOutOfMem() proc allocShared0(size: Natural): pointer = @@ -322,10 +322,10 @@ elif defined(gogc): zeroMem(result, size) proc reallocShared(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) + result = c_realloc(p, newsize) if result == nil: raiseOutOfMem() - proc deallocShared(p: pointer) = cfree(p) + proc deallocShared(p: pointer) = c_free(p) when hasThreadSupport: proc getFreeSharedMem(): int = discard @@ -354,6 +354,12 @@ elif defined(gogc): cast[PGenericSeq](result).reserved = len cast[PGenericSeq](result).elemSize = typ.base.size + proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = + result = newObj(typ, cap * typ.base.size + GenericSeqSize) + cast[PGenericSeq](result).len = 0 + cast[PGenericSeq](result).reserved = cap + cast[PGenericSeq](result).elemSize = typ.base.size + proc growObj(old: pointer, newsize: int): pointer = # the Go GC doesn't have a realloc var @@ -389,26 +395,41 @@ elif defined(nogc) and defined(useMalloc): when not defined(useNimRtl): proc alloc(size: Natural): pointer = - result = cmalloc(size) - if result == nil: raiseOutOfMem() + var x = c_malloc(size + sizeof(size)) + if x == nil: raiseOutOfMem() + + cast[ptr int](x)[] = size + result = cast[pointer](cast[int](x) + sizeof(size)) + proc alloc0(size: Natural): pointer = result = alloc(size) zeroMem(result, size) proc realloc(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) - if result == nil: raiseOutOfMem() - proc dealloc(p: pointer) = cfree(p) + var x = cast[pointer](cast[int](p) - sizeof(newsize)) + let oldsize = cast[ptr int](x)[] + + x = c_realloc(x, newsize + sizeof(newsize)) + + if x == nil: raiseOutOfMem() + + cast[ptr int](x)[] = newsize + result = cast[pointer](cast[int](x) + sizeof(newsize)) + + if newsize > oldsize: + zeroMem(cast[pointer](cast[int](result) + oldsize), newsize - oldsize) + + proc dealloc(p: pointer) = c_free(cast[pointer](cast[int](p) - sizeof(int))) proc allocShared(size: Natural): pointer = - result = cmalloc(size) + result = c_malloc(size) if result == nil: raiseOutOfMem() proc allocShared0(size: Natural): pointer = result = alloc(size) zeroMem(result, size) proc reallocShared(p: pointer, newsize: Natural): pointer = - result = crealloc(p, newsize) + result = c_realloc(p, newsize) if result == nil: raiseOutOfMem() - proc deallocShared(p: pointer) = cfree(p) + proc deallocShared(p: pointer) = c_free(p) proc GC_disable() = discard proc GC_enable() = discard @@ -432,6 +453,7 @@ elif defined(nogc) and defined(useMalloc): result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len + proc newObjNoInit(typ: PNimType, size: int): pointer = result = alloc(size) @@ -491,6 +513,7 @@ elif defined(nogc): result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len + proc growObj(old: pointer, newsize: int): pointer = result = realloc(old, newsize) @@ -511,11 +534,12 @@ elif defined(nogc): include "system/cellsets" else: - include "system/alloc" + when not defined(gcStack): + include "system/alloc" - include "system/cellsets" - when not leakDetector and not useCellIds: - sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") + include "system/cellsets" + when not leakDetector and not useCellIds: + sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") when compileOption("gc", "v2"): include "system/gc2" elif defined(gcStack): @@ -529,4 +553,10 @@ else: else: include "system/gc" +when not declared(nimNewSeqOfCap): + proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = + result = newObj(typ, addInt(mulInt(cap, typ.base.size), GenericSeqSize)) + cast[PGenericSeq](result).len = 0 + cast[PGenericSeq](result).reserved = cap + {.pop.} diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index d587d772f..fc6b8c99d 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -39,6 +39,9 @@ proc getCurrentDir(): string = builtin proc rawExec(cmd: string): int {.tags: [ExecIOEffect], raises: [OSError].} = builtin +proc warningImpl(arg, orig: string) = discard +proc hintImpl(arg, orig: string) = discard + proc paramStr*(i: int): string = ## Retrieves the ``i``'th command line parameter. builtin @@ -52,6 +55,31 @@ proc switch*(key: string, val="") = ## example ``switch("checks", "on")``. builtin +proc warning*(name: string; val: bool) = + ## Disables or enables a specific warning. + let v = if val: "on" else: "off" + warningImpl(name & "]:" & v, "warning[" & name & "]:" & v) + +proc hint*(name: string; val: bool) = + ## Disables or enables a specific hint. + let v = if val: "on" else: "off" + hintImpl(name & "]:" & v, "hint[" & name & "]:" & v) + +proc patchFile*(package, filename, replacement: string) = + ## Overrides the location of a given file belonging to the + ## passed package. + ## If the ``replacement`` is not an absolute path, the path + ## is interpreted to be local to the Nimscript file that contains + ## the call to ``patchFile``, Nim's ``--path`` is not used at all + ## to resolve the filename! + ## + ## Example: + ## + ## .. code-block:: nim + ## + ## patchFile("stdlib", "asyncdispatch", "patches/replacement") + discard + proc getCommand*(): string = ## Gets the Nim command that the compiler has been invoked with, for example ## "c", "js", "build", "help". @@ -94,6 +122,10 @@ proc existsDir*(dir: string): bool = ## An alias for ``dirExists``. dirExists(dir) +proc selfExe*(): string = + ## Returns the currently running nim or nimble executable. + builtin + proc toExe*(filename: string): string = ## On Windows adds ".exe" to `filename`, else returns `filename` unmodified. (when defined(windows): filename & ".exe" else: filename) @@ -180,6 +212,15 @@ proc exec*(command: string, input: string, cache = "") {. log "exec: " & command: echo staticExec(command, input, cache) +proc selfExec*(command: string) = + ## Executes an external command with the current nim/nimble executable. + ## ``Command`` must not contain the "nim " part. + let c = selfExe() & " " & command + log "exec: " & c: + if rawExec(c) != 0: + raise newException(OSError, "FAILED: " & c) + checkOsError() + proc put*(key, value: string) = ## Sets a configuration 'key' like 'gcc.options.always' to its value. builtin @@ -206,7 +247,7 @@ proc cd*(dir: string) {.raises: [OSError].} = ## ## The change is permanent for the rest of the execution, since this is just ## a shortcut for `os.setCurrentDir() - ## <http://nim-lang.org/os.html#setCurrentDir,string>`_ . Use the `withDir() + ## <http://nim-lang.org/docs/os.html#setCurrentDir,string>`_ . Use the `withDir() ## <#withDir>`_ template if you want to perform a temporary change only. setCurrentDir(dir) checkOsError() diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 78410d716..b07a362a0 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -68,11 +68,11 @@ when defined(emscripten): mmapDescr.realSize = realSize mmapDescr.realPointer = realPointer - c_fprintf(c_stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) + #c_fprintf(stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) proc osTryAllocPages(size: int): pointer = osAllocPages(size) - proc osDeallocPages(p: pointer, size: int) {.inline} = + proc osDeallocPages(p: pointer, size: int) {.inline.} = var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) munmap(mmapDescr.realPointer, mmapDescr.realSize) @@ -87,6 +87,8 @@ elif defined(posix): const MAP_ANONYMOUS = 0x1000 elif defined(solaris): const MAP_ANONYMOUS = 0x100 + elif defined(linux): + const MAP_ANONYMOUS = 0x20 else: var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint @@ -107,7 +109,7 @@ elif defined(posix): MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) if result == cast[pointer](-1): result = nil - proc osDeallocPages(p: pointer, size: int) {.inline} = + proc osDeallocPages(p: pointer, size: int) {.inline.} = when reallyOsDealloc: discard munmap(p, size) elif defined(windows): @@ -148,8 +150,9 @@ elif defined(windows): #VirtualFree(p, size, MEM_DECOMMIT) elif hostOS == "standalone": + const StandaloneHeapSize {.intdefine.}: int = 1024 * PageSize var - theHeap: array[1024*PageSize, float64] # 'float64' for alignment + theHeap: array[StandaloneHeapSize, float64] # 'float64' for alignment bumpPointer = cast[int](addr theHeap) proc osAllocPages(size: int): pointer {.inline.} = diff --git a/lib/system/profiler.nim b/lib/system/profiler.nim index ae8ff4e19..7146500d9 100644 --- a/lib/system/profiler.nim +++ b/lib/system/profiler.nim @@ -19,10 +19,14 @@ const MaxTraceLen = 20 # tracking the last 20 calls is enough type - StackTrace* = array [0..MaxTraceLen-1, cstring] + StackTrace* = object + lines*: array[0..MaxTraceLen-1, cstring] + files*: array[0..MaxTraceLen-1, cstring] ProfilerHook* = proc (st: StackTrace) {.nimcall.} {.deprecated: [TStackTrace: StackTrace, TProfilerHook: ProfilerHook].} +proc `[]`*(st: StackTrace, i: int): cstring = st.lines[i] + proc captureStackTrace(f: PFrame, st: var StackTrace) = const firstCalls = 5 @@ -30,9 +34,10 @@ proc captureStackTrace(f: PFrame, st: var StackTrace) = it = f i = 0 total = 0 - while it != nil and i <= high(st)-(firstCalls-1): + while it != nil and i <= high(st.lines)-(firstCalls-1): # the (-1) is for the "..." entry - st[i] = it.procname + st.lines[i] = it.procname + st.files[i] = it.filename inc(i) inc(total) it = it.prev @@ -43,10 +48,12 @@ proc captureStackTrace(f: PFrame, st: var StackTrace) = for j in 1..total-i-(firstCalls-1): if b != nil: b = b.prev if total != i: - st[i] = "..." + st.lines[i] = "..." + st.files[i] = "..." inc(i) - while b != nil and i <= high(st): - st[i] = b.procname + while b != nil and i <= high(st.lines): + st.lines[i] = b.procname + st.files[i] = b.filename inc(i) b = b.prev diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 4da4781ef..cf7d6d7a9 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -16,7 +16,7 @@ proc reprInt(x: int64): string {.compilerproc.} = return $x proc reprFloat(x: float): string {.compilerproc.} = return $x proc reprPointer(x: pointer): string {.compilerproc.} = - var buf: array [0..59, char] + var buf: array[0..59, char] discard c_sprintf(buf, "%p", x) return $buf @@ -24,7 +24,7 @@ proc `$`(x: uint64): string = if x == 0: result = "0" else: - var buf: array [60, char] + var buf: array[60, char] var i = 0 var n = x while n != 0: @@ -73,23 +73,20 @@ proc reprChar(x: char): string {.compilerRtl.} = add result, "\'" proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = - # we read an 'int' but this may have been too large, so mask the other bits: - let b = (sizeof(int)-typ.size)*8 # bits - let m = 1 shl (b-1) # mask - var o = e and ((1 shl b)-1) # clear upper bits - o = (o xor m) - m # sign extend - # XXX we need a proper narrowing based on signedness here - #e and ((1 shl (typ.size*8)) - 1) + ## Return string representation for enumeration values + var n = typ.node if ntfEnumHole notin typ.flags: - if o <% typ.node.len: - return $typ.node.sons[o].name + let o = e - n.sons[0].offset + if o >= 0 and o <% typ.node.len: + return $n.sons[o].name else: # ugh we need a slow linear search: - var n = typ.node var s = n.sons for i in 0 .. n.len-1: - if s[i].offset == o: return $s[i].name - result = $o & " (invalid data!)" + if s[i].offset == e: + return $s[i].name + + result = $e & " (invalid data!)" type PByteArray = ptr array[0.. 0xffff, int8] diff --git a/lib/system/sets.nim b/lib/system/sets.nim index 66877de30..53d222468 100644 --- a/lib/system/sets.nim +++ b/lib/system/sets.nim @@ -10,7 +10,7 @@ # set handling type - NimSet = array [0..4*2048-1, uint8] + NimSet = array[0..4*2048-1, uint8] {.deprecated: [TNimSet: NimSet].} proc countBits32(n: int32): int {.compilerproc.} = diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index d0bba6775..3e9657ce0 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -16,54 +16,56 @@ # of the standard library! -proc fputs(c: cstring, f: File) {.importc: "fputs", header: "<stdio.h>", - tags: [WriteIOEffect].} -proc fgets(c: cstring, n: int, f: File): cstring {. +proc c_fdopen(filehandle: cint, mode: cstring): File {. + importc: "fdopen", header: "<stdio.h>".} +proc c_fputs(c: cstring, f: File): cint {. + importc: "fputs", header: "<stdio.h>", tags: [WriteIOEffect].} +proc c_fgets(c: cstring, n: cint, f: File): cstring {. importc: "fgets", header: "<stdio.h>", tags: [ReadIOEffect].} -proc fgetc(stream: File): cint {.importc: "fgetc", header: "<stdio.h>", - tags: [ReadIOEffect].} -proc ungetc(c: cint, f: File) {.importc: "ungetc", header: "<stdio.h>", - tags: [].} -proc putc(c: char, stream: File) {.importc: "putc", header: "<stdio.h>", - tags: [WriteIOEffect].} -proc fprintf(f: File, frmt: cstring) {.importc: "fprintf", - header: "<stdio.h>", varargs, tags: [WriteIOEffect].} -proc strlen(c: cstring): int {. - importc: "strlen", header: "<string.h>", tags: [].} +proc c_fgetc(stream: File): cint {. + importc: "fgetc", header: "<stdio.h>", tags: [ReadIOEffect].} +proc c_ungetc(c: cint, f: File): cint {. + importc: "ungetc", header: "<stdio.h>", tags: [].} +proc c_putc(c: cint, stream: File): cint {. + importc: "putc", header: "<stdio.h>", tags: [WriteIOEffect].} +proc c_fflush(f: File): cint {. + importc: "fflush", header: "<stdio.h>".} +proc c_fclose(f: File): cint {. + importc: "fclose", header: "<stdio.h>".} # C routine that is used here: -proc fread(buf: pointer, size, n: int, f: File): int {. +proc c_fread(buf: pointer, size, n: csize, f: File): csize {. importc: "fread", header: "<stdio.h>", tags: [ReadIOEffect].} -proc fseek(f: File, offset: clong, whence: int): int {. +proc c_fseek(f: File, offset: clong, whence: cint): cint {. importc: "fseek", header: "<stdio.h>", tags: [].} -proc ftell(f: File): int {.importc: "ftell", header: "<stdio.h>", tags: [].} -proc ferror(f: File): int {.importc: "ferror", header: "<stdio.h>", tags: [].} -proc setvbuf(stream: File, buf: pointer, typ, size: cint): cint {. - importc, header: "<stdio.h>", tags: [].} -proc memchr(s: pointer, c: cint, n: csize): pointer {. - importc: "memchr", header: "<string.h>", tags: [].} -proc memset(s: pointer, c: cint, n: csize) {. - header: "<string.h>", importc: "memset", tags: [].} -proc fwrite(buf: pointer, size, n: int, f: File): int {. - importc: "fwrite", noDecl.} +proc c_ftell(f: File): clong {. + importc: "ftell", header: "<stdio.h>", tags: [].} +proc c_ferror(f: File): cint {. + importc: "ferror", header: "<stdio.h>", tags: [].} +proc c_setvbuf(f: File, buf: pointer, mode: cint, size: csize): cint {. + importc: "setvbuf", header: "<stdio.h>", tags: [].} +proc c_fwrite(buf: pointer, size, n: csize, f: File): cint {. + importc: "fwrite", header: "<stdio.h>".} proc raiseEIO(msg: string) {.noinline, noreturn.} = sysFatal(IOError, msg) {.push stackTrace:off, profiler:off.} proc readBuffer(f: File, buffer: pointer, len: Natural): int = - result = fread(buffer, 1, len, f) + result = c_fread(buffer, 1, len, f) proc readBytes(f: File, a: var openArray[int8|uint8], start, len: Natural): int = result = readBuffer(f, addr(a[start]), len) proc readChars(f: File, a: var openArray[char], start, len: Natural): int = + if (start + len) > len(a): + raiseEIO("buffer overflow: (start+len) > length of openarray buffer") result = readBuffer(f, addr(a[start]), len) -proc write(f: File, c: cstring) = fputs(c, f) +proc write(f: File, c: cstring) = discard c_fputs(c, f) proc writeBuffer(f: File, buffer: pointer, len: Natural): int = - result = fwrite(buffer, 1, len, f) + result = c_fwrite(buffer, 1, len, f) proc writeBytes(f: File, a: openArray[int8|uint8], start, len: Natural): int = var x = cast[ptr array[0..1000_000_000, int8]](a) @@ -95,23 +97,28 @@ else: const BufSize = 4000 +proc close*(f: File) = discard c_fclose(f) +proc readChar*(f: File): char = result = char(c_fgetc(f)) +proc flushFile*(f: File) = discard c_fflush(f) +proc getFileHandle*(f: File): FileHandle = c_fileno(f) + proc readLine(f: File, line: var TaintedString): bool = var pos = 0 # Use the currently reserved space for a first try when defined(nimscript): - var space = 80 + var space: cint = 80 else: - var space = cast[PGenericSeq](line.string).space + var space: cint = cint(cast[PGenericSeq](line.string).space) line.string.setLen(space) while true: # memset to \l so that we can tell how far fgets wrote, even on EOF, where # fgets doesn't append an \l - memset(addr line.string[pos], '\l'.ord, space) - if fgets(addr line.string[pos], space, f) == nil: + c_memset(addr line.string[pos], '\l'.ord, space) + if c_fgets(addr line.string[pos], space, f) == nil: line.string.setLen(0) return false - let m = memchr(addr line.string[pos], '\l'.ord, space) + let m = c_memchr(addr line.string[pos], '\l'.ord, space) if m != nil: # \l found: Could be our own or the one by fgets, in any case, we're done var last = cast[ByteAddress](m) - cast[ByteAddress](addr line.string[0]) @@ -140,23 +147,23 @@ proc readLine(f: File): TaintedString = proc write(f: File, i: int) = when sizeof(int) == 8: - fprintf(f, "%lld", i) + c_fprintf(f, "%lld", i) else: - fprintf(f, "%ld", i) + c_fprintf(f, "%ld", i) proc write(f: File, i: BiggestInt) = when sizeof(BiggestInt) == 8: - fprintf(f, "%lld", i) + c_fprintf(f, "%lld", i) else: - fprintf(f, "%ld", i) + c_fprintf(f, "%ld", i) proc write(f: File, b: bool) = if b: write(f, "true") else: write(f, "false") -proc write(f: File, r: float32) = fprintf(f, "%g", r) -proc write(f: File, r: BiggestFloat) = fprintf(f, "%g", r) +proc write(f: File, r: float32) = c_fprintf(f, "%g", r) +proc write(f: File, r: BiggestFloat) = c_fprintf(f, "%g", r) -proc write(f: File, c: char) = putc(c, f) +proc write(f: File, c: char) = discard c_putc(ord(c), f) proc write(f: File, a: varargs[string, `$`]) = for x in items(a): write(f, x) @@ -176,15 +183,15 @@ proc readAllBuffer(file: File): string = proc rawFileSize(file: File): int = # this does not raise an error opposed to `getFileSize` - var oldPos = ftell(file) - discard fseek(file, 0, 2) # seek the end of the file - result = ftell(file) - discard fseek(file, clong(oldPos), 0) + var oldPos = c_ftell(file) + discard c_fseek(file, 0, 2) # seek the end of the file + result = c_ftell(file) + discard c_fseek(file, clong(oldPos), 0) proc endOfFile(f: File): bool = # do not blame me; blame the ANSI C standard this is so brain-damaged - var c = fgetc(f) - ungetc(c, f) + var c = c_fgetc(f) + discard c_ungetc(c, f) return c < 0'i32 proc readAllFile(file: File, len: int): string = @@ -195,7 +202,7 @@ proc readAllFile(file: File, len: int): string = if endOfFile(file): if bytes < len: result.setLen(bytes) - elif ferror(file) != 0: + elif c_ferror(file) != 0: raiseEIO("error while reading from file") else: # We read all the bytes but did not reach the EOF @@ -234,8 +241,7 @@ when declared(stdout): # interface to the C procs: -when (defined(windows) and not defined(useWinAnsi)) or defined(nimdoc): - include "system/widestrs" +include "system/widestrs" when defined(windows) and not defined(useWinAnsi): when defined(cpp): @@ -265,23 +271,40 @@ else: importc: "freopen", nodecl.} const - FormatOpen: array [FileMode, string] = ["rb", "wb", "w+b", "r+b", "ab"] + FormatOpen: array[FileMode, string] = ["rb", "wb", "w+b", "r+b", "ab"] #"rt", "wt", "w+t", "r+t", "at" # we always use binary here as for Nim the OS line ending # should not be translated. when defined(posix) and not defined(nimscript): - type - Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint + when defined(linux) and defined(amd64): + type + Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint + + # fillers ensure correct size & offsets + Stat {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + filler_1: array[24, char] + st_mode: Mode ## Mode of file + filler_2: array[144 - 24 - 4, char] + + proc S_ISDIR(m: Mode): bool = + ## Test for a directory. + (m and 0o170000) == 0o40000 + + else: + type + Mode {.importc: "mode_t", header: "<sys/types.h>".} = cint - Stat {.importc: "struct stat", - header: "<sys/stat.h>", final, pure.} = object ## struct stat - st_mode: Mode ## Mode of file + Stat {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + st_mode: Mode ## Mode of file - proc S_ISDIR(m: Mode): bool {.importc, header: "<sys/stat.h>".} - ## Test for a directory. + proc S_ISDIR(m: Mode): bool {.importc, header: "<sys/stat.h>".} + ## Test for a directory. - proc fstat(a1: cint, a2: var Stat): cint {.importc, header: "<sys/stat.h>".} + proc c_fstat(a1: cint, a2: var Stat): cint {. + importc: "fstat", header: "<sys/stat.h>".} proc open(f: var File, filename: string, mode: FileMode = fmRead, @@ -294,45 +317,38 @@ proc open(f: var File, filename: string, # be opened. var f2 = cast[File](p) var res: Stat - if fstat(getFileHandle(f2), res) >= 0'i32 and S_ISDIR(res.st_mode): + if c_fstat(getFileHandle(f2), res) >= 0'i32 and S_ISDIR(res.st_mode): close(f2) return false result = true f = cast[File](p) if bufSize > 0 and bufSize <= high(cint).int: - discard setvbuf(f, nil, IOFBF, bufSize.cint) + discard c_setvbuf(f, nil, IOFBF, bufSize.cint) elif bufSize == 0: - discard setvbuf(f, nil, IONBF, 0) + discard c_setvbuf(f, nil, IONBF, 0) proc reopen(f: File, filename: string, mode: FileMode = fmRead): bool = var p: pointer = freopen(filename, FormatOpen[mode], f) result = p != nil -proc fdopen(filehandle: FileHandle, mode: cstring): File {. - importc: "fdopen", header: "<stdio.h>".} - proc open(f: var File, filehandle: FileHandle, mode: FileMode): bool = - f = fdopen(filehandle, FormatOpen[mode]) + f = c_fdopen(filehandle, FormatOpen[mode]) result = f != nil proc setFilePos(f: File, pos: int64) = - if fseek(f, clong(pos), 0) != 0: + if c_fseek(f, clong(pos), 0) != 0: raiseEIO("cannot set file position") proc getFilePos(f: File): int64 = - result = ftell(f) + result = c_ftell(f) if result < 0: raiseEIO("cannot retrieve file position") proc getFileSize(f: File): int64 = var oldPos = getFilePos(f) - discard fseek(f, 0, 2) # seek the end of the file + discard c_fseek(f, 0, 2) # seek the end of the file result = getFilePos(f) setFilePos(f, oldPos) -when not declared(close): - proc close(f: File) {. - importc: "fclose", header: "<stdio.h>", tags: [].} - proc readFile(filename: string): TaintedString = var f: File if open(f, filename): @@ -353,4 +369,12 @@ proc writeFile(filename, content: string) = else: sysFatal(IOError, "cannot open: ", filename) +proc setStdIoUnbuffered() = + when declared(stdout): + discard c_setvbuf(stdout, nil, IONBF, 0) + when declared(stderr): + discard c_setvbuf(stderr, nil, IONBF, 0) + when declared(stdin): + discard c_setvbuf(stdin, nil, IONBF, 0) + {.pop.} diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index 6dcdfff0d..c3e23052b 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -14,39 +14,41 @@ when defined(Windows): type Handle = int - SysLock {.final, pure.} = object # CRITICAL_SECTION in WinApi + + SysLock {.importc: "CRITICAL_SECTION", + header: "<windows.h>", final, pure.} = object # CRITICAL_SECTION in WinApi DebugInfo: pointer LockCount: int32 RecursionCount: int32 OwningThread: int LockSemaphore: int - Reserved: int32 + SpinCount: int SysCond = Handle {.deprecated: [THandle: Handle, TSysLock: SysLock, TSysCond: SysCond].} - proc initSysLock(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "InitializeCriticalSection".} + proc initSysLock(L: var SysLock) {.importc: "InitializeCriticalSection", + header: "<windows.h>".} ## Initializes the lock `L`. - proc tryAcquireSysAux(L: var SysLock): int32 {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "TryEnterCriticalSection".} + proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", + header: "<windows.h>".} ## Tries to acquire the lock `L`. proc tryAcquireSys(L: var SysLock): bool {.inline.} = result = tryAcquireSysAux(L) != 0'i32 - proc acquireSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "EnterCriticalSection".} + proc acquireSys(L: var SysLock) {.importc: "EnterCriticalSection", + header: "<windows.h>".} ## Acquires the lock `L`. - proc releaseSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "LeaveCriticalSection".} + proc releaseSys(L: var SysLock) {.importc: "LeaveCriticalSection", + header: "<windows.h>".} ## Releases the lock `L`. - proc deinitSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "DeleteCriticalSection".} + proc deinitSys(L: var SysLock) {.importc: "DeleteCriticalSection", + header: "<windows.h>".} proc createEvent(lpEventAttributes: pointer, bManualReset, bInitialState: int32, @@ -86,17 +88,16 @@ else: #include <pthread.h>""".} = object SysLockType = distinct cint - proc SysLockType_Reentrant: SysLockType = - {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} - proc initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) {. importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} - proc initSysLockAttr(a: var SysLockAttr) {. - importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} - - proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. - importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} + when insideRLocksModule: + proc SysLockType_Reentrant: SysLockType = + {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} + proc initSysLockAttr(a: var SysLockAttr) {. + importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} + proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. + importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} proc acquireSys(L: var SysLock) {.noSideEffect, importc: "pthread_mutex_lock", header: "<pthread.h>".} @@ -111,14 +112,14 @@ else: proc deinitSys(L: var SysLock) {.noSideEffect, importc: "pthread_mutex_destroy", header: "<pthread.h>".} - proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. - importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} - proc waitSysCond(cond: var SysCond, lock: var SysLock) {. - importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} - proc signalSysCond(cond: var SysCond) {. - importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} - - proc deinitSysCond(cond: var SysCond) {.noSideEffect, - importc: "pthread_cond_destroy", header: "<pthread.h>".} + when not insideRLocksModule: + proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. + importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} + proc waitSysCond(cond: var SysCond, lock: var SysLock) {. + importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} + proc signalSysCond(cond: var SysCond) {. + importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} + proc deinitSysCond(cond: var SysCond) {.noSideEffect, + importc: "pthread_cond_destroy", header: "<pthread.h>".} {.pop.} diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index e2137e8f4..eb3d276e0 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -30,7 +30,7 @@ proc eqStrings(a, b: NimString): bool {.inline, compilerProc.} = if a == b: return true if a == nil or b == nil: return false return a.len == b.len and - c_memcmp(a.data, b.data, a.len) == 0'i32 + equalMem(addr(a.data), addr(b.data), a.len) when declared(allocAtomic): template allocStr(size: expr): expr = @@ -71,7 +71,7 @@ proc copyStrLast(s: NimString, start, last: int): NimString {.compilerProc.} = if len > 0: result = rawNewStringNoInit(len) result.len = len - c_memcpy(result.data, addr(s.data[start]), len) + copyMem(addr(result.data), addr(s.data[start]), len) result.data[len] = '\0' else: result = rawNewString(len) @@ -82,10 +82,11 @@ proc copyStr(s: NimString, start: int): NimString {.compilerProc.} = proc toNimStr(str: cstring, len: int): NimString {.compilerProc.} = result = rawNewStringNoInit(len) result.len = len - c_memcpy(result.data, str, len + 1) + copyMem(addr(result.data), str, len + 1) proc cstrToNimstr(str: cstring): NimString {.compilerRtl.} = - result = toNimStr(str, c_strlen(str)) + if str == nil: NimString(nil) + else: toNimStr(str, str.len) proc copyString(src: NimString): NimString {.compilerRtl.} = if src != nil: @@ -94,7 +95,7 @@ proc copyString(src: NimString): NimString {.compilerRtl.} = else: result = rawNewStringNoInit(src.len) result.len = src.len - c_memcpy(result.data, src.data, src.len + 1) + copyMem(addr(result.data), addr(src.data), src.len + 1) proc copyStringRC1(src: NimString): NimString {.compilerRtl.} = if src != nil: @@ -107,7 +108,7 @@ proc copyStringRC1(src: NimString): NimString {.compilerRtl.} = else: result = rawNewStringNoInit(src.len) result.len = src.len - c_memcpy(result.data, src.data, src.len + 1) + copyMem(addr(result.data), addr(src.data), src.len + 1) proc hashString(s: string): int {.compilerproc.} = @@ -177,7 +178,7 @@ proc resizeString(dest: NimString, addlen: int): NimString {.compilerRtl.} = # DO NOT UPDATE LEN YET: dest.len = newLen proc appendString(dest, src: NimString) {.compilerproc, inline.} = - c_memcpy(addr(dest.data[dest.len]), src.data, src.len + 1) + copyMem(addr(dest.data[dest.len]), addr(src.data), src.len + 1) inc(dest.len, src.len) proc appendChar(dest: NimString, c: char) {.compilerproc, inline.} = @@ -228,7 +229,8 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. elif newLen < result.len: # we need to decref here, otherwise the GC leaks! when not defined(boehmGC) and not defined(nogc) and - not defined(gcMarkAndSweep) and not defined(gogc): + not defined(gcMarkAndSweep) and not defined(gogc) and + not defined(gcStack): when false: # compileOption("gc", "v2"): for i in newLen..result.len-1: let len0 = gch.tempStack.len @@ -300,46 +302,42 @@ proc nimFloatToStr(f: float): string {.compilerproc.} = else: result = $buf -proc strtod(buf: cstring, endptr: ptr cstring): float64 {.importc, - header: "<stdlib.h>", noSideEffect.} - -var decimalPoint: char - -proc getDecimalPoint(): char = - result = decimalPoint - if result == '\0': - if strtod("0,5", nil) == 0.5: result = ',' - else: result = '.' - # yes this is threadsafe in practice, spare me: - decimalPoint = result +proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {. + importc: "strtod", header: "<stdlib.h>", noSideEffect.} const IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} + powtens = [ 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, + 1e20, 1e21, 1e22] proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start = 0): int {.compilerProc.} = - # This routine leverages `strtod()` for the non-trivial task of - # parsing floating point numbers correctly. Because `strtod()` is - # locale-dependent with respect to the radix character, we create - # a copy where the decimal point is replaced with the locale's - # radix character. + # This routine attempt to parse float that can parsed quickly. + # ie whose integer part can fit inside a 53bits integer. + # their real exponent must also be <= 22. If the float doesn't follow + # these restrictions, transform the float into this form: + # INTEGER * 10 ^ exponent and leave the work to standard `strtod()`. + # This avoid the problems of decimal character portability. + # see: http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ var i = start sign = 1.0 - t: array[500, char] # flaviu says: 325 is the longest reasonable literal - ti = 0 - hasdigits = false - - template addToBuf(c) = - if ti < t.high: - t[ti] = c; inc(ti) + kdigits, fdigits = 0 + exponent: int + integer: uint64 + fraction: uint64 + frac_exponent= 0 + exp_sign = 1 + first_digit = -1 + has_sign = false # Sign? if s[i] == '+' or s[i] == '-': + has_sign = true if s[i] == '-': sign = -1.0 - t[ti] = s[i] - inc(i); inc(ti) + inc(i) # NaN? if s[i] == 'N' or s[i] == 'n': @@ -359,40 +357,111 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, return i+3 - start return 0 + if s[i] in {'0'..'9'}: + first_digit = (s[i].ord - '0'.ord) # Integer part? while s[i] in {'0'..'9'}: - hasdigits = true - addToBuf(s[i]) - inc(i); + inc(kdigits) + integer = integer * 10'u64 + (s[i].ord - '0'.ord).uint64 + inc(i) while s[i] == '_': inc(i) # Fractional part? if s[i] == '.': - addToBuf(getDecimalPoint()) inc(i) + # if no integer part, Skip leading zeros + if kdigits <= 0: + while s[i] == '0': + inc(frac_exponent) + inc(i) + while s[i] == '_': inc(i) + + if first_digit == -1 and s[i] in {'0'..'9'}: + first_digit = (s[i].ord - '0'.ord) + # get fractional part while s[i] in {'0'..'9'}: - hasdigits = true - addToBuf(s[i]) + inc(fdigits) + inc(frac_exponent) + integer = integer * 10'u64 + (s[i].ord - '0'.ord).uint64 inc(i) while s[i] == '_': inc(i) - if not hasdigits: + + # if has no digits: return error + if kdigits + fdigits <= 0 and + (i == start or # no char consumed (empty string). + (i == start + 1 and has_sign)): # or only '+' or '- return 0 - # Exponent? if s[i] in {'e', 'E'}: - addToBuf(s[i]) inc(i) - if s[i] in {'+', '-'}: - addToBuf(s[i]) + if s[i] == '+' or s[i] == '-': + if s[i] == '-': + exp_sign = -1 + inc(i) if s[i] notin {'0'..'9'}: return 0 while s[i] in {'0'..'9'}: - addToBuf(s[i]) + exponent = exponent * 10 + (ord(s[i]) - ord('0')) inc(i) - while s[i] == '_': inc(i) - number = strtod(t, nil) + while s[i] == '_': inc(i) # underscores are allowed and ignored + + var real_exponent = exp_sign*exponent - frac_exponent + let exp_negative = real_exponent < 0 + var abs_exponent = abs(real_exponent) + + # if exponent greater than can be represented: +/- zero or infinity + if abs_exponent > 999: + if exp_negative: + number = 0.0*sign + else: + number = Inf*sign + return i - start + + # if integer is representable in 53 bits: fast path + # max fast path integer is 1<<53 - 1 or 8999999999999999 (16 digits) + if kdigits + fdigits <= 16 and first_digit <= 8: + # max float power of ten with set bits above the 53th bit is 10^22 + if abs_exponent <= 22: + if exp_negative: + number = sign * integer.float / powtens[abs_exponent] + else: + number = sign * integer.float * powtens[abs_exponent] + return i - start + + # if exponent is greater try to fit extra exponent above 22 by multiplying + # integer part is there is space left. + let slop = 15 - kdigits - fdigits + if abs_exponent <= 22 + slop and not exp_negative: + number = sign * integer.float * powtens[slop] * powtens[abs_exponent-slop] + return i - start + + # if failed: slow path with strtod. + var t: array[500, char] # flaviu says: 325 is the longest reasonable literal + var ti = 0 + let maxlen = t.high - "e+000".len # reserve enough space for exponent + result = i - start + i = start + # re-parse without error checking, any error should be handled by the code above. + while s[i] in {'0'..'9','+','-'}: + if ti < maxlen: + t[ti] = s[i]; inc(ti) + inc(i) + while s[i] in {'.', '_'}: # skip underscore and decimal point + inc(i) + + # insert exponent + t[ti] = 'E'; inc(ti) + t[ti] = if exp_negative: '-' else: '+'; inc(ti) + inc(ti, 3) + + # insert adjusted exponent + t[ti-1] = ('0'.ord + abs_exponent mod 10).char; abs_exponent = abs_exponent div 10 + t[ti-2] = ('0'.ord + abs_exponent mod 10).char; abs_exponent = abs_exponent div 10 + t[ti-3] = ('0'.ord + abs_exponent mod 10).char + + number = c_strtod(t, nil) proc nimInt64ToStr(x: int64): string {.compilerRtl.} = result = newString(sizeof(x)*4) diff --git a/lib/system/threads.nim b/lib/system/threads.nim index bdb737e35..583e3ae86 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -24,7 +24,7 @@ ## import locks ## ## var -## thr: array [0..4, Thread[tuple[a,b: int]]] +## thr: array[0..4, Thread[tuple[a,b: int]]] ## L: Lock ## ## proc threadFunc(interval: tuple[a,b: int]) {.thread.} = @@ -51,7 +51,7 @@ const when defined(windows): type - SysThread = Handle + SysThread* = Handle WinThreadProc = proc (x: pointer): int32 {.stdcall.} {.deprecated: [TSysThread: SysThread, TWinThreadProc: WinThreadProc].} @@ -117,16 +117,21 @@ else: schedh = "#define _GNU_SOURCE\n#include <sched.h>" pthreadh = "#define _GNU_SOURCE\n#include <pthread.h>" + when defined(linux): + type Time = clong + else: + type Time = int + type - SysThread {.importc: "pthread_t", header: "<sys/types.h>", + SysThread* {.importc: "pthread_t", header: "<sys/types.h>", final, pure.} = object Pthread_attr {.importc: "pthread_attr_t", header: "<sys/types.h>", final, pure.} = object Timespec {.importc: "struct timespec", header: "<time.h>", final, pure.} = object - tv_sec: int - tv_nsec: int + tv_sec: Time + tv_nsec: clong {.deprecated: [TSysThread: SysThread, Tpthread_attr: PThreadAttr, Ttimespec: Timespec].} @@ -193,7 +198,7 @@ when emulatedThreadVars: # allocations are needed. Currently less than 7K are used on a 64bit machine. # We use ``float`` for proper alignment: type - ThreadLocalStorage = array [0..1_000, float] + ThreadLocalStorage = array[0..1_000, float] PGcThread = ptr GcThread GcThread {.pure, inheritable.} = object @@ -301,7 +306,7 @@ type ## a pointer as a thread ID. {.deprecated: [TThread: Thread, TThreadId: ThreadId].} -when not defined(boehmgc) and not hasSharedHeap and not defined(gogc): +when not defined(boehmgc) and not hasSharedHeap and not defined(gogc) and not defined(gcstack): proc deallocOsPages() when defined(boehmgc): @@ -331,7 +336,7 @@ else: proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) = when defined(boehmgc): boehmGC_call_with_stack_base(threadProcWrapDispatch[TArg], thrd) - elif not defined(nogc) and not defined(gogc): + elif not defined(nogc) and not defined(gogc) and not defined(gcstack): var p {.volatile.}: proc(a: ptr Thread[TArg]) {.nimcall.} = threadProcWrapDispatch[TArg] when not hasSharedHeap: @@ -374,6 +379,10 @@ proc running*[TArg](t: Thread[TArg]): bool {.inline.} = ## returns true if `t` is running. result = t.dataFn != nil +proc handle*[TArg](t: Thread[TArg]): SysThread {.inline.} = + ## returns the thread handle of `t`. + result = t.sys + when hostOS == "windows": proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = ## waits for the thread `t` to finish. diff --git a/lib/system/timers.nim b/lib/system/timers.nim index ac8418824..129a7d092 100644 --- a/lib/system/timers.nim +++ b/lib/system/timers.nim @@ -61,7 +61,7 @@ elif defined(posixRealtime): final, pure.} = object ## struct timespec tv_sec: int ## Seconds. tv_nsec: int ## Nanoseconds. - {.deprecated: [TClockid: Clickid, TTimeSpec: TimeSpec].} + {.deprecated: [TClockid: Clockid, TTimeSpec: TimeSpec].} var CLOCK_REALTIME {.importc: "CLOCK_REALTIME", header: "<time.h>".}: Clockid @@ -78,11 +78,16 @@ elif defined(posixRealtime): else: # fallback Posix implementation: + when defined(linux): + type Time = clong + else: + type Time = int + type Timeval {.importc: "struct timeval", header: "<sys/select.h>", final, pure.} = object ## struct timeval - tv_sec: int ## Seconds. - tv_usec: int ## Microseconds. + tv_sec: Time ## Seconds. + tv_usec: clong ## Microseconds. {.deprecated: [Ttimeval: Timeval].} proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {. importc: "gettimeofday", header: "<sys/time.h>".} diff --git a/lib/system/widestrs.nim b/lib/system/widestrs.nim index 5a30a7c0f..578bebe80 100644 --- a/lib/system/widestrs.nim +++ b/lib/system/widestrs.nim @@ -73,11 +73,11 @@ template fastRuneAt(s: cstring, i: int, result: expr, doInc = true) = result = 0xFFFD when doInc: inc(i) -iterator runes(s: cstring): int = +iterator runes(s: cstring, L: int): int = var i = 0 result: int - while s[i] != '\0': + while i < L: fastRuneAt(s, i, result, true) yield result @@ -85,7 +85,7 @@ proc newWideCString*(source: cstring, L: int): WideCString = unsafeNew(result, L * 4 + 2) #result = cast[wideCString](alloc(L * 4 + 2)) var d = 0 - for ch in runes(source): + for ch in runes(source, L): if ch <=% UNI_MAX_BMP: if ch >=% UNI_SUR_HIGH_START and ch <=% UNI_SUR_LOW_END: result[d] = UNI_REPLACEMENT_CHAR @@ -104,12 +104,7 @@ proc newWideCString*(source: cstring, L: int): WideCString = proc newWideCString*(s: cstring): WideCString = if s.isNil: return nil - when not declared(c_strlen): - proc c_strlen(a: cstring): int {. - header: "<string.h>", noSideEffect, importc: "strlen".} - - let L = c_strlen(s) - result = newWideCString(s, L) + result = newWideCString(s, s.len) proc newWideCString*(s: string): WideCString = result = newWideCString(s, s.len) diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim new file mode 100644 index 000000000..19c9815d2 --- /dev/null +++ b/lib/upcoming/asyncdispatch.nim @@ -0,0 +1,2154 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +include "system/inclrtl" + +import os, oids, tables, strutils, macros, times, heapqueue + +import nativesockets, net, queues + +export Port, SocketFlag + +#{.injectStmt: newGcInvariant().} + +## AsyncDispatch +## ************* +## +## This module implements asynchronous IO. This includes a dispatcher, +## a ``Future`` type implementation, and an ``async`` macro which allows +## asynchronous code to be written in a synchronous style with the ``await`` +## keyword. +## +## The dispatcher acts as a kind of event loop. You must call ``poll`` on it +## (or a function which does so for you such as ``waitFor`` or ``runForever``) +## in order to poll for any outstanding events. The underlying implementation +## is based on epoll on Linux, IO Completion Ports on Windows and select on +## other operating systems. +## +## The ``poll`` function will not, on its own, return any events. Instead +## an appropriate ``Future`` object will be completed. A ``Future`` is a +## type which holds a value which is not yet available, but which *may* be +## available in the future. You can check whether a future is finished +## by using the ``finished`` function. When a future is finished it means that +## either the value that it holds is now available or it holds an error instead. +## The latter situation occurs when the operation to complete a future fails +## with an exception. You can distinguish between the two situations with the +## ``failed`` function. +## +## Future objects can also store a callback procedure which will be called +## automatically once the future completes. +## +## Futures therefore can be thought of as an implementation of the proactor +## pattern. In this +## pattern you make a request for an action, and once that action is fulfilled +## a future is completed with the result of that action. Requests can be +## made by calling the appropriate functions. For example: calling the ``recv`` +## function will create a request for some data to be read from a socket. The +## future which the ``recv`` function returns will then complete once the +## requested amount of data is read **or** an exception occurs. +## +## Code to read some data from a socket may look something like this: +## +## .. code-block::nim +## var future = socket.recv(100) +## future.callback = +## proc () = +## echo(future.read) +## +## All asynchronous functions returning a ``Future`` will not block. They +## will not however return immediately. An asynchronous function will have +## code which will be executed before an asynchronous request is made, in most +## cases this code sets up the request. +## +## In the above example, the ``recv`` function will return a brand new +## ``Future`` instance once the request for data to be read from the socket +## is made. This ``Future`` instance will complete once the requested amount +## of data is read, in this case it is 100 bytes. The second line sets a +## callback on this future which will be called once the future completes. +## All the callback does is write the data stored in the future to ``stdout``. +## The ``read`` function is used for this and it checks whether the future +## completes with an error for you (if it did it will simply raise the +## error), if there is no error however it returns the value of the future. +## +## Asynchronous procedures +## ----------------------- +## +## Asynchronous procedures remove the pain of working with callbacks. They do +## this by allowing you to write asynchronous code the same way as you would +## write synchronous code. +## +## An asynchronous procedure is marked using the ``{.async.}`` pragma. +## When marking a procedure with the ``{.async.}`` pragma it must have a +## ``Future[T]`` return type or no return type at all. If you do not specify +## a return type then ``Future[void]`` is assumed. +## +## Inside asynchronous procedures ``await`` can be used to call any +## procedures which return a +## ``Future``; this includes asynchronous procedures. When a procedure is +## "awaited", the asynchronous procedure it is awaited in will +## suspend its execution +## until the awaited procedure's Future completes. At which point the +## asynchronous procedure will resume its execution. During the period +## when an asynchronous procedure is suspended other asynchronous procedures +## will be run by the dispatcher. +## +## The ``await`` call may be used in many contexts. It can be used on the right +## hand side of a variable declaration: ``var data = await socket.recv(100)``, +## in which case the variable will be set to the value of the future +## automatically. It can be used to await a ``Future`` object, and it can +## be used to await a procedure returning a ``Future[void]``: +## ``await socket.send("foobar")``. +## +## Discarding futures +## ------------------ +## +## Futures should **never** be discarded. This is because they may contain +## errors. If you do not care for the result of a Future then you should +## use the ``asyncCheck`` procedure instead of the ``discard`` keyword. +## +## Examples +## -------- +## +## For examples take a look at the documentation for the modules implementing +## asynchronous IO. A good place to start is the +## `asyncnet module <asyncnet.html>`_. +## +## Limitations/Bugs +## ---------------- +## +## * The effect system (``raises: []``) does not work with async procedures. +## * Can't await in a ``except`` body +## * Forward declarations for async procs are broken, +## link includes workaround: https://github.com/nim-lang/Nim/issues/3182. +## * FutureVar[T] needs to be completed manually. + +# TODO: Check if yielded future is nil and throw a more meaningful exception + +# -- Futures + +type + FutureBase* = ref object of RootObj ## Untyped future. + cb: proc () {.closure,gcsafe.} + finished: bool + error*: ref Exception ## Stored exception + errorStackTrace*: string + when not defined(release): + stackTrace: string ## For debugging purposes only. + id: int + fromProc: string + + Future*[T] = ref object of FutureBase ## Typed future. + value: T ## Stored value + + FutureVar*[T] = distinct Future[T] + + FutureError* = object of Exception + cause*: FutureBase + +{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} + +when not defined(release): + var currentID = 0 + +proc callSoon*(cbproc: proc ()) {.gcsafe.} + +proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = + ## Creates a new future. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + new(result) + result.finished = false + when not defined(release): + result.stackTrace = getStackTrace() + result.id = currentID + result.fromProc = fromProc + currentID.inc() + +proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] = + ## Create a new ``FutureVar``. This Future type is ideally suited for + ## situations where you want to avoid unnecessary allocations of Futures. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + result = FutureVar[T](newFuture[T](fromProc)) + +proc clean*[T](future: FutureVar[T]) = + ## Resets the ``finished`` status of ``future``. + Future[T](future).finished = false + Future[T](future).error = nil + +proc checkFinished[T](future: Future[T]) = + ## Checks whether `future` is finished. If it is then raises a + ## ``FutureError``. + when not defined(release): + if future.finished: + var msg = "" + msg.add("An attempt was made to complete a Future more than once. ") + msg.add("Details:") + msg.add("\n Future ID: " & $future.id) + msg.add("\n Created in proc: " & future.fromProc) + msg.add("\n Stack trace to moment of creation:") + msg.add("\n" & indent(future.stackTrace.strip(), 4)) + when T is string: + msg.add("\n Contents (string): ") + msg.add("\n" & indent(future.value.repr, 4)) + msg.add("\n Stack trace to moment of secondary completion:") + msg.add("\n" & indent(getStackTrace().strip(), 4)) + var err = newException(FutureError, msg) + err.cause = future + raise err + +proc complete*[T](future: Future[T], val: T) = + ## Completes ``future`` with value ``val``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.value = val + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*(future: Future[void]) = + ## Completes a void ``future``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*[T](future: FutureVar[T]) = + ## Completes a ``FutureVar``. + template fut: expr = Future[T](future) + checkFinished(fut) + assert(fut.error == nil) + fut.finished = true + if fut.cb != nil: + fut.cb() + +proc fail*[T](future: Future[T], error: ref Exception) = + ## Completes ``future`` with ``error``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + future.finished = true + future.error = error + future.errorStackTrace = + if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) + if future.cb != nil: + future.cb() + else: + # This is to prevent exceptions from being silently ignored when a future + # is discarded. + # TODO: This may turn out to be a bad idea. + # Turns out this is a bad idea. + #raise error + discard + +proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + ## + ## **Note**: You most likely want the other ``callback`` setter which + ## passes ``future`` as a param to the callback. + future.cb = cb + if future.finished: + callSoon(future.cb) + +proc `callback=`*[T](future: Future[T], + cb: proc (future: Future[T]) {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + future.callback = proc () = cb(future) + +proc injectStacktrace[T](future: Future[T]) = + # TODO: Come up with something better. + when not defined(release): + var msg = "" + msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") + + if not future.errorStackTrace.isNil and future.errorStackTrace != "": + msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) + else: + msg.add("\n Empty or nil stack trace.") + future.error.msg.add(msg) + +proc read*[T](future: Future[T]): T = + ## Retrieves the value of ``future``. Future must be finished otherwise + ## this function will fail with a ``ValueError`` exception. + ## + ## If the result of the future is an error then that error will be raised. + if future.finished: + if future.error != nil: + injectStacktrace(future) + raise future.error + when T isnot void: + return future.value + else: + # TODO: Make a custom exception type for this? + raise newException(ValueError, "Future still in progress.") + +proc readError*[T](future: Future[T]): ref Exception = + ## Retrieves the exception stored in ``future``. + ## + ## An ``ValueError`` exception will be thrown if no exception exists + ## in the specified Future. + if future.error != nil: return future.error + else: + raise newException(ValueError, "No error in future.") + +proc mget*[T](future: FutureVar[T]): var T = + ## Returns a mutable value stored in ``future``. + ## + ## Unlike ``read``, this function will not raise an exception if the + ## Future has not been finished. + result = Future[T](future).value + +proc finished*[T](future: Future[T]): bool = + ## Determines whether ``future`` has completed. + ## + ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. + future.finished + +proc failed*(future: FutureBase): bool = + ## Determines whether ``future`` completed with an error. + return future.error != nil + +proc asyncCheck*[T](future: Future[T]) = + ## Sets a callback on ``future`` which raises an exception if the future + ## finished with an error. + ## + ## This should be used instead of ``discard`` to discard void futures. + future.callback = + proc () = + if future.failed: + injectStacktrace(future) + raise future.error + +proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once both ``fut1`` and ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`and`") + fut1.callback = + proc () = + if fut2.finished: retFuture.complete() + fut2.callback = + proc () = + if fut1.finished: retFuture.complete() + return retFuture + +proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once either ``fut1`` or ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`or`") + proc cb() = + if not retFuture.finished: retFuture.complete() + fut1.callback = cb + fut2.callback = cb + return retFuture + +proc all*[T](futs: varargs[Future[T]]): auto = + ## Returns a future which will complete once + ## all futures in ``futs`` complete. + ## + ## If the awaited futures are not ``Future[void]``, the returned future + ## will hold the values of all awaited futures in a sequence. + ## + ## If the awaited futures *are* ``Future[void]``, + ## this proc returns ``Future[void]``. + + when T is void: + var + retFuture = newFuture[void]("asyncdispatch.all") + completedFutures = 0 + + let totalFutures = len(futs) + + for fut in futs: + fut.callback = proc(f: Future[T]) = + inc(completedFutures) + + if completedFutures == totalFutures: + retFuture.complete() + + return retFuture + + else: + var + retFuture = newFuture[seq[T]]("asyncdispatch.all") + retValues = newSeq[T](len(futs)) + completedFutures = 0 + + for i, fut in futs: + proc setCallback(i: int) = + fut.callback = proc(f: Future[T]) = + retValues[i] = f.read() + inc(completedFutures) + + if completedFutures == len(retValues): + retFuture.complete(retValues) + + setCallback(i) + + return retFuture + +type + PDispatcherBase = ref object of RootRef + timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]] + callbacks: Queue[proc ()] + +proc processTimers(p: PDispatcherBase) {.inline.} = + while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt: + p.timers.pop().fut.complete() + +proc processPendingCallbacks(p: PDispatcherBase) = + while p.callbacks.len > 0: + var cb = p.callbacks.dequeue() + cb() + +proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = + # If dispatcher has active timers this proc returns the timeout + # of the nearest timer. Returns `timeout` otherwise. + result = timeout + if p.timers.len > 0: + let timerTimeout = p.timers[0].finishAt + let curTime = epochTime() + if timeout == -1 or (curTime + (timeout / 1000)) > timerTimeout: + result = int((timerTimeout - curTime) * 1000) + if result < 0: result = 0 + +when defined(windows) or defined(nimdoc): + import winlean, sets, hashes + type + CompletionKey = ULONG_PTR + + CompletionData* = object + fd*: AsyncFD # TODO: Rename this. + cb*: proc (fd: AsyncFD, bytesTransferred: Dword, + errcode: OSErrorCode) {.closure,gcsafe.} + cell*: ForeignCell # we need this `cell` to protect our `cb` environment, + # when using RegisterWaitForSingleObject, because + # waiting is done in different thread. + + PDispatcher* = ref object of PDispatcherBase + ioPort: Handle + handles: HashSet[AsyncFD] + + CustomOverlapped = object of OVERLAPPED + data*: CompletionData + + PCustomOverlapped* = ref CustomOverlapped + + AsyncFD* = distinct int + + PostCallbackData = object + ioPort: Handle + handleFd: AsyncFD + waitFd: Handle + ovl: PCustomOverlapped + PostCallbackDataPtr = ptr PostCallbackData + + AsyncEventImpl = object + hEvent: Handle + hWaiter: Handle + pcd: PostCallbackDataPtr + AsyncEvent* = ptr AsyncEventImpl + + Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} + {.deprecated: [TCompletionKey: CompletionKey, TAsyncFD: AsyncFD, + TCustomOverlapped: CustomOverlapped, TCompletionData: CompletionData].} + + proc hash(x: AsyncFD): Hash {.borrow.} + proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow.} + + proc newDispatcher*(): PDispatcher = + ## Creates a new Dispatcher instance. + new result + result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) + result.handles = initSet[AsyncFD]() + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + proc getGlobalDispatcher*(): PDispatcher = + ## Retrieves the global thread-local dispatcher. + if gDisp.isNil: gDisp = newDispatcher() + result = gDisp + + proc register*(fd: AsyncFD) = + ## Registers ``fd`` with the dispatcher. + let p = getGlobalDispatcher() + if createIoCompletionPort(fd.Handle, p.ioPort, + cast[CompletionKey](fd), 1) == 0: + raiseOSError(osLastError()) + p.handles.incl(fd) + + proc verifyPresence(fd: AsyncFD) = + ## Ensures that file descriptor has been registered with the dispatcher. + let p = getGlobalDispatcher() + if fd notin p.handles: + raise newException(ValueError, + "Operation performed on a socket which has not been registered with" & + " the dispatcher yet.") + + proc poll*(timeout = 500) = + ## Waits for completion events and processes them. + let p = getGlobalDispatcher() + if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: + raise newException(ValueError, + "No handles or timers registered in dispatcher.") + + let at = p.adjustedTimeout(timeout) + var llTimeout = + if at == -1: winlean.INFINITE + else: at.int32 + + var lpNumberOfBytesTransferred: Dword + var lpCompletionKey: ULONG_PTR + var customOverlapped: PCustomOverlapped + let res = getQueuedCompletionStatus(p.ioPort, + addr lpNumberOfBytesTransferred, addr lpCompletionKey, + cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool + + # http://stackoverflow.com/a/12277264/492186 + # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html + if res: + # This is useful for ensuring the reliability of the overlapped struct. + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, OSErrorCode(-1)) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + + GC_unref(customOverlapped) + else: + let errCode = osLastError() + if customOverlapped != nil: + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, errCode) + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) + else: + if errCode.int32 == WAIT_TIMEOUT: + # Timed out + discard + else: raiseOSError(errCode) + + # Timer processing. + processTimers(p) + # Callback queue processing + processPendingCallbacks(p) + + var acceptEx*: WSAPROC_ACCEPTEX + var connectEx*: WSAPROC_CONNECTEX + var getAcceptExSockAddrs*: WSAPROC_GETACCEPTEXSOCKADDRS + + proc initPointer(s: SocketHandle, fun: var pointer, guid: var GUID): bool = + # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c + var bytesRet: Dword + fun = nil + result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid, + sizeof(GUID).Dword, addr fun, sizeof(pointer).Dword, + addr bytesRet, nil, nil) == 0 + + proc initAll() = + let dummySock = newNativeSocket() + if dummySock == INVALID_SOCKET: + raiseOSError(osLastError()) + var fun: pointer = nil + if not initPointer(dummySock, fun, WSAID_CONNECTEX): + raiseOSError(osLastError()) + connectEx = cast[WSAPROC_CONNECTEX](fun) + if not initPointer(dummySock, fun, WSAID_ACCEPTEX): + raiseOSError(osLastError()) + acceptEx = cast[WSAPROC_ACCEPTEX](fun) + if not initPointer(dummySock, fun, WSAID_GETACCEPTEXSOCKADDRS): + raiseOSError(osLastError()) + getAcceptExSockAddrs = cast[WSAPROC_GETACCEPTEXSOCKADDRS](fun) + close(dummySock) + + proc connect*(socket: AsyncFD, address: string, port: Port, + domain = nativesockets.AF_INET): Future[void] = + ## Connects ``socket`` to server at ``address:port``. + ## + ## Returns a ``Future`` which will complete when the connection succeeds + ## or an error occurs. + verifyPresence(socket) + var retFuture = newFuture[void]("connect") + # Apparently ``ConnectEx`` expects the socket to be initially bound: + var saddr: Sockaddr_in + saddr.sin_family = int16(toInt(domain)) + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(socket.SocketHandle, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) < 0'i32: + raiseOSError(osLastError()) + + var aiList = getAddrInfo(address, port, domain) + var success = false + var lastError: OSErrorCode + var it = aiList + while it != nil: + # "the OVERLAPPED structure must remain valid until the I/O completes" + # http://blogs.msdn.com/b/oldnewthing/archive/2011/02/02/10123392.aspx + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + var ret = connectEx(socket.SocketHandle, it.ai_addr, + sizeof(Sockaddr_in).cint, nil, 0, nil, + cast[POVERLAPPED](ol)) + if ret: + # Request to connect completed immediately. + success = true + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + break + else: + lastError = osLastError() + if lastError.int32 == ERROR_IO_PENDING: + # In this case ``ol`` will be deallocated in ``poll``. + success = true + break + else: + GC_unref(ol) + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + return retFuture + + proc recv*(socket: AsyncFD, size: int, + flags = {SocketFlag.SafeDisconn}): Future[string] = + ## Reads **up to** ``size`` bytes from ``socket``. Returned future will + ## complete once all the data requested is read, a part of the data has been + ## read, or the socket has disconnected in which case the future will + ## complete with a value of ``""``. + ## + ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + + + # Things to note: + # * When WSARecv completes immediately then ``bytesReceived`` is very + # unreliable. + # * Still need to implement message-oriented socket disconnection, + # '\0' in the message currently signifies a socket disconnect. Who + # knows what will happen when someone sends that to our socket. + verifyPresence(socket) + assert SocketFlag.Peek notin flags, "Peek not supported on Windows." + + var retFuture = newFuture[string]("recv") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](alloc0(size)) + dataBuf.len = size.ULONG + + var bytesReceived: Dword + var flagsio = flags.toOSFlags().Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + if bytesCount == 0 and dataBuf.buf[0] == '\0': + retFuture.complete("") + else: + var data = newString(bytesCount) + assert bytesCount <= size + copyMem(addr data[0], addr dataBuf.buf[0], bytesCount) + retFuture.complete($data) + else: + if flags.isDisconnectionError(errcode): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + if dataBuf.buf != nil: + dealloc dataBuf.buf + dataBuf.buf = nil + ) + + let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + addr flagsio, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + if dataBuf.buf != nil: + dealloc dataBuf.buf + dataBuf.buf = nil + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + var data = newString(bytesReceived) + assert bytesReceived <= size + copyMem(addr data[0], addr dataBuf.buf[0], bytesReceived) + retFuture.complete($data) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete("") + return retFuture + + proc recvInto*(socket: AsyncFD, buf: cstring, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must + ## at least be of that size. Returned future will complete once all the + ## data requested is read, a part of the data has been read, or the socket + ## has disconnected in which case the future will complete with a value of + ## ``0``. + ## + ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + + + # Things to note: + # * When WSARecv completes immediately then ``bytesReceived`` is very + # unreliable. + # * Still need to implement message-oriented socket disconnection, + # '\0' in the message currently signifies a socket disconnect. Who + # knows what will happen when someone sends that to our socket. + verifyPresence(socket) + assert SocketFlag.Peek notin flags, "Peek not supported on Windows." + + var retFuture = newFuture[int]("recvInto") + + #buf[] = '\0' + var dataBuf: TWSABuf + dataBuf.buf = buf + dataBuf.len = size.ULONG + + var bytesReceived: Dword + var flagsio = flags.toOSFlags().Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + if bytesCount == 0 and dataBuf.buf[0] == '\0': + retFuture.complete(0) + else: + retFuture.complete(bytesCount) + else: + if flags.isDisconnectionError(errcode): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + if dataBuf.buf != nil: + dataBuf.buf = nil + ) + + let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + addr flagsio, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + if dataBuf.buf != nil: + dataBuf.buf = nil + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + elif ret == 0: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) + return retFuture + + proc send*(socket: AsyncFD, data: string, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to ``socket``. The returned future will complete once all + ## data has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("send") + + var dataBuf: TWSABuf + dataBuf.buf = data # since this is not used in a callback, this is fine + dataBuf.len = data.len.ULONG + + var bytesReceived, lowFlags: Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + if flags.isDisconnectionError(errcode): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + lowFlags, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: Socklen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to specified destination ``saddr``, using + ## socket ``socket``. The returned future will complete once all data + ## has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]("sendTo") + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](data) + dataBuf.len = size.ULONG + var bytesSent = 0.Dword + var lowFlags = 0.Dword + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen: cint = cint(saddrLen) + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, + lowFlags, cast[ptr SockAddr](addr(staddr[0])), + stalen, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``buf``, which must + ## be at least of size ``size``, address of datagram's sender will be + ## stored into ``saddr`` and ``saddrLen``. Returned future will complete + ## once one datagram has been received, and will return size of packet + ## received. + verifyPresence(socket) + var retFuture = newFuture[int]("recvFromInto") + + var dataBuf = TWSABuf(buf: cast[cstring](data), len: size.ULONG) + + var bytesReceived = 0.Dword + var lowFlags = 0.Dword + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount <= size + retFuture.complete(bytesCount) + else: + # datagram sockets don't have disconnection, + # so we can just raise an exception + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, + addr bytesReceived, addr lowFlags, + saddr, cast[ptr cint](saddrLen), + cast[POVERLAPPED](ol), nil) + if res == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + # Request completed immediately. + if bytesReceived != 0: + assert bytesReceived <= size + retFuture.complete(bytesReceived) + else: + if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): + retFuture.complete(bytesReceived) + return retFuture + + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): + Future[tuple[address: string, client: AsyncFD]] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection and the remote address of the client. + ## The future will complete when the connection is successfully accepted. + ## + ## The resulting client socket is automatically registered to the + ## dispatcher. + ## + ## The ``accept`` call may result in an error if the connecting socket + ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## flag is specified then this error will not be raised and instead + ## accept will be called again. + verifyPresence(socket) + var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") + + var clientSock = newNativeSocket() + if clientSock == osInvalidSocket: raiseOSError(osLastError()) + + const lpOutputLen = 1024 + var lpOutputBuf = newString(lpOutputLen) + var dwBytesReceived: Dword + let dwReceiveDataLength = 0.Dword # We don't want any data to be read. + let dwLocalAddressLength = Dword(sizeof (Sockaddr_in) + 16) + let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16) + + template completeAccept(): stmt {.immediate, dirty.} = + var listenSock = socket + let setoptRet = setsockopt(clientSock, SOL_SOCKET, + SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, + sizeof(listenSock).SockLen) + if setoptRet != 0: raiseOSError(osLastError()) + + var localSockaddr, remoteSockaddr: ptr SockAddr + var localLen, remoteLen: int32 + getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, + dwLocalAddressLength, dwRemoteAddressLength, + addr localSockaddr, addr localLen, + addr remoteSockaddr, addr remoteLen) + register(clientSock.AsyncFD) + # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 + retFuture.complete( + (address: $inet_ntoa(cast[ptr Sockaddr_in](remoteSockAddr).sin_addr), + client: clientSock.AsyncFD) + ) + + template failAccept(errcode): stmt = + if flags.isDisconnectionError(errcode): + var newAcceptFut = acceptAddr(socket, flags) + newAcceptFut.callback = + proc () = + if newAcceptFut.failed: + retFuture.fail(newAcceptFut.readError) + else: + retFuture.complete(newAcceptFut.read) + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + completeAccept() + else: + failAccept(errcode) + ) + + # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx + let ret = acceptEx(socket.SocketHandle, clientSock, addr lpOutputBuf[0], + dwReceiveDataLength, + dwLocalAddressLength, + dwRemoteAddressLength, + addr dwBytesReceived, cast[POVERLAPPED](ol)) + + if not ret: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + failAccept(err) + GC_unref(ol) + else: + completeAccept() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + + return retFuture + + proc newAsyncNativeSocket*(domain, sockType, protocol: cint): AsyncFD = + ## Creates a new socket and registers it with the dispatcher implicitly. + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + register(result) + + proc newAsyncNativeSocket*(domain: Domain = nativesockets.AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP): AsyncFD = + ## Creates a new socket and registers it with the dispatcher implicitly. + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + register(result) + + proc closeSocket*(socket: AsyncFD) = + ## Closes a socket and ensures that it is unregistered. + socket.SocketHandle.close() + getGlobalDispatcher().handles.excl(socket) + + proc unregister*(fd: AsyncFD) = + ## Unregisters ``fd``. + getGlobalDispatcher().handles.excl(fd) + + {.push stackTrace:off.} + proc waitableCallback(param: pointer, + timerOrWaitFired: WINBOOL): void {.stdcall.} = + var p = cast[PostCallbackDataPtr](param) + discard postQueuedCompletionStatus(p.ioPort, timerOrWaitFired.Dword, + ULONG_PTR(p.handleFd), + cast[pointer](p.ovl)) + {.pop.} + + template registerWaitableEvent(mask) = + let p = getGlobalDispatcher() + var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword + var hEvent = wsaCreateEvent() + if hEvent == 0: + raiseOSError(osLastError()) + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + pcd.ioPort = p.ioPort + pcd.handleFd = fd + var ol = PCustomOverlapped() + GC_ref(ol) + + ol.data = CompletionData(fd: fd, cb: + proc(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + # we excluding our `fd` because cb(fd) can register own handler + # for this `fd` + p.handles.excl(fd) + # unregisterWait() is called before callback, because appropriate + # winsockets function can re-enable event. + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms741576(v=vs.85).aspx + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + if cb(fd): + # callback returned `true`, so we free all allocated resources + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + # pcd.ovl will be unrefed in poll(). + else: + # callback returned `false` we need to continue + if p.handles.contains(fd): + # new callback was already registered with `fd`, so we free all + # allocated resources. This happens because in callback `cb` + # addRead/addWrite was called with same `fd`. + deallocShared(cast[pointer](pcd)) + if not wsaCloseEvent(hEvent): + raiseOSError(osLastError()) + else: + # we need to include `fd` again + p.handles.incl(fd) + # and register WaitForSingleObject again + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + # pcd.ovl will be unrefed in poll() + discard wsaCloseEvent(hEvent) + deallocShared(cast[pointer](pcd)) + raiseOSError(osLastError()) + else: + # we ref pcd.ovl one more time, because it will be unrefed in + # poll() + GC_ref(pcd.ovl) + ) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + # This is main part of `hacky way` is using WSAEventSelect, so `hEvent` + # will be signaled when appropriate `mask` events will be triggered. + if wsaEventSelect(fd.SocketHandle, hEvent, mask) != 0: + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + p.handles.incl(fd) + + proc addRead*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for read availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addRead` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.recv() + ## or asyncdispatch.accept(), because they are using IOCP, please use + ## nativesockets.recv() and nativesockets.accept() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `read` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) + + proc addWrite*(fd: AsyncFD, cb: Callback) = + ## Start watching the file descriptor for write availability and then call + ## the callback ``cb``. + ## + ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## so if you can avoid it, please do it. Use `addWrite` only if really + ## need it (main usecase is adaptation of `unix like` libraries to be + ## asynchronous on Windows). + ## If you use this function, you dont need to use asyncdispatch.send() + ## or asyncdispatch.connect(), because they are using IOCP, please use + ## nativesockets.send() and nativesockets.connect() instead. + ## + ## Be sure your callback ``cb`` returns ``true``, if you want to remove + ## watch of `write` notifications, and ``false``, if you want to continue + ## receiving notifies. + registerWaitableEvent(FD_WRITE or FD_CONNECT or FD_CLOSE) + + template registerWaitableHandle(p, hEvent, flags, pcd, handleCallback) = + let handleFD = AsyncFD(hEvent) + pcd.ioPort = p.ioPort + pcd.handleFd = handleFD + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: handleFD, cb: handleCallback) + # We need to protect our callback environment value, so GC will not free it + # accidentally. + ol.data.cell = system.protect(rawEnv(ol.data.cb)) + + pcd.ovl = ol + if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](pcd), INFINITE, flags): + GC_unref(ol) + deallocShared(cast[pointer](pcd)) + discard wsaCloseEvent(hEvent) + raiseOSError(osLastError()) + p.handles.incl(handleFD) + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Registers callback ``cb`` to be called when timer expired. + ## ``timeout`` - timeout value in milliseconds. + ## ``oneshot`` - `true`, to generate only one timeout event, `false`, to + ## generate timeout events periodically. + + doAssert(timeout > 0) + let p = getGlobalDispatcher() + + var hEvent = createEvent(nil, 1, 0, nil) + if hEvent == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + if oneshot: flags = flags or WT_EXECUTEONLYONCE + + proc timercb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + let res = cb(fd) + if res or oneshot: + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + discard closeHandle(hEvent) + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + + registerWaitableHandle(p, hEvent, flags, pcd, timercb) + + proc addProcess*(pid: int, cb: Callback) = + ## Registers callback ``cb`` to be called when process with pid ``pid`` + ## exited. + let p = getGlobalDispatcher() + let procFlags = SYNCHRONIZE + var hProcess = openProcess(procFlags, 0, pid.Dword) + if hProcess == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + + proc proccb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + discard closeHandle(hProcess) + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + discard cb(fd) + + registerWaitableHandle(p, hProcess, flags, pcd, proccb) + + proc newAsyncEvent*(): AsyncEvent = + ## Creates new ``AsyncEvent`` object. + var sa = SECURITY_ATTRIBUTES( + nLength: sizeof(SECURITY_ATTRIBUTES).cint, + bInheritHandle: 1 + ) + var event = createEvent(addr(sa), 0'i32, 0'i32, nil) + if event == INVALID_HANDLE_VALUE: + raiseOSError(osLastError()) + result = cast[AsyncEvent](allocShared0(sizeof(AsyncEventImpl))) + + proc setEvent*(ev: AsyncEvent) = + ## Set event ``ev`` to signaled state. + if setEvent(ev.hEvent) == 0: + raiseOSError(osLastError()) + + proc close*(ev: AsyncEvent) = + ## Closes event ``ev``. + if ev.hWaiter != 0: + let p = getGlobalDispatcher() + if unregisterWait(ev.hWaiter) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + p.handles.excl(AsyncFD(ev.hEvent)) + + if closeHandle(ev.hEvent) == 0: + raiseOSError(osLastError()) + deallocShared(cast[pointer](ev)) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Registers callback ``cb`` to be called when ``ev`` will be signaled + if ev.hWaiter != 0: + raise newException(ValueError, "Event is already registered!") + + let p = getGlobalDispatcher() + let hEvent = ev.hEvent + + var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) + var flags = WT_EXECUTEINWAITTHREAD.Dword + + proc eventcb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if cb(fd): + if unregisterWait(pcd.waitFd) == 0: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + raiseOSError(osLastError()) + ev.hWaiter = 0 + deallocShared(cast[pointer](pcd)) + p.handles.excl(fd) + + registerWaitableHandle(p, hEvent, flags, pcd, eventcb) + ev.hWaiter = pcd.waitFd + + initAll() +else: + import ioselectors + when defined(windows): + import winlean + const + EINTR = WSAEINPROGRESS + EINPROGRESS = WSAEINPROGRESS + EWOULDBLOCK = WSAEWOULDBLOCK + EAGAIN = EINPROGRESS + MSG_NOSIGNAL = 0 + else: + from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, + MSG_NOSIGNAL + + const supportedPlatform = defined(linux) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(macosx) + + type + AsyncFD* = distinct cint + Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} + + AsyncData = object + readCB: Callback + writeCB: Callback + + AsyncEvent* = SelectEvent + + PDispatcher* = ref object of PDispatcherBase + selector: Selector[AsyncData] + {.deprecated: [TAsyncFD: AsyncFD, TCallback: Callback].} + + proc `==`*(x, y: AsyncFD): bool {.borrow.} + + proc newDispatcher*(): PDispatcher = + new result + result.selector = newSelector[AsyncData]() + result.timers.newHeapQueue() + result.callbacks = initQueue[proc ()](64) + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + proc getGlobalDispatcher*(): PDispatcher = + if gDisp.isNil: gDisp = newDispatcher() + result = gDisp + + proc register*(fd: AsyncFD) = + let p = getGlobalDispatcher() + var data = AsyncData() + p.selector.registerHandle(fd.SocketHandle, {}, data) + + proc newAsyncNativeSocket*(domain: cint, sockType: cint, + protocol: cint): AsyncFD = + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + when defined(macosx): + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) + register(result) + + proc newAsyncNativeSocket*(domain: Domain = AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP): AsyncFD = + result = newNativeSocket(domain, sockType, protocol).AsyncFD + result.SocketHandle.setBlocking(false) + when defined(macosx): + result.SocketHandle.setSockOptInt(SOL_SOCKET, SO_NOSIGPIPE, 1) + register(result) + + proc closeSocket*(sock: AsyncFD) = + let disp = getGlobalDispatcher() + disp.selector.unregister(sock.SocketHandle) + sock.SocketHandle.close() + + proc unregister*(fd: AsyncFD) = + getGlobalDispatcher().selector.unregister(fd.SocketHandle) + + # proc unregister*(ev: AsyncEvent) = + # getGlobalDispatcher().selector.unregister(SelectEvent(ev)) + + proc addRead*(fd: AsyncFD, cb: Callback) = + let p = getGlobalDispatcher() + withData(p.selector, fd.SocketHandle, adata) do: + adata.readCB = cb + do: + raise newException(ValueError, "File descriptor not registered.") + p.selector.updateHandle(fd.SocketHandle, {Event.Read}) + + proc addWrite*(fd: AsyncFD, cb: Callback) = + let p = getGlobalDispatcher() + withData(p.selector, fd.SocketHandle, adata) do: + adata.writeCB = cb + do: + raise newException(ValueError, "File descriptor not registered.") + p.selector.updateHandle(fd.SocketHandle, {Event.Write}) + + proc poll*(timeout = 500) = + var keys: array[64, ReadyKey[AsyncData]] + + let p = getGlobalDispatcher() + when supportedPlatform: + let customSet = {Event.Timer, Event.Signal, Event.Process, + Event.Vnode, Event.User} + + if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: + raise newException(ValueError, + "No handles or timers registered in dispatcher.") + + if not p.selector.isEmpty(): + var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys) + var i = 0 + while i < count: + var update = false + var fd = keys[i].fd.SocketHandle + let events = keys[i].events + + if Event.Read in events: + let cb = keys[i].data.readCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.readCB == cb: + adata.readCB = nil + update = true + + if Event.Write in events: + let cb = keys[i].data.writeCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.writeCB == cb: + adata.writeCB = nil + update = true + + when supportedPlatform: + if (customSet * events) != {}: + let cb = keys[i].data.readCB + doAssert(cb != nil) + if cb(fd.AsyncFD): + p.selector.withData(fd, adata) do: + if adata.readCB == cb: + adata.readCB = nil + p.selector.unregister(fd) + + if update: + var newEvents: set[Event] = {} + p.selector.withData(fd, adata) do: + if adata.readCB != nil: incl(newEvents, Event.Read) + if adata.writeCB != nil: incl(newEvents, Event.Write) + p.selector.updateHandle(fd, newEvents) + inc(i) + + # Timer processing. + processTimers(p) + # Callback queue processing + processPendingCallbacks(p) + + proc connect*(socket: AsyncFD, address: string, port: Port, + domain = AF_INET): Future[void] = + var retFuture = newFuture[void]("connect") + + proc cb(fd: AsyncFD): bool = + var ret = SocketHandle(fd).getSockOptInt(cint(SOL_SOCKET), cint(SO_ERROR)) + if ret == 0: + # We have connected. + retFuture.complete() + return true + elif ret == EINTR: + # interrupted, keep waiting + return false + else: + retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + return true + + assert getSockDomain(socket.SocketHandle) == domain + var aiList = getAddrInfo(address, port, domain) + var success = false + var lastError: OSErrorCode + var it = aiList + while it != nil: + var ret = connect(socket.SocketHandle, it.ai_addr, it.ai_addrlen.Socklen) + if ret == 0: + # Request to connect completed immediately. + success = true + retFuture.complete() + break + else: + lastError = osLastError() + if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: + success = true + addWrite(socket, cb) + break + else: + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + return retFuture + + proc recv*(socket: AsyncFD, size: int, + flags = {SocketFlag.SafeDisconn}): Future[string] = + var retFuture = newFuture[string]("recv") + + var readBuffer = newString(size) + + proc cb(sock: AsyncFD): bool = + result = true + let res = recv(sock.SocketHandle, addr readBuffer[0], size.cint, + flags.toOSFlags()) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + elif res == 0: + # Disconnected + retFuture.complete("") + else: + readBuffer.setLen(res) + retFuture.complete(readBuffer) + # TODO: The following causes a massive slowdown. + #if not cb(socket): + addRead(socket, cb) + return retFuture + + proc recvInto*(socket: AsyncFD, buf: cstring, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = + var retFuture = newFuture[int]("recvInto") + + proc cb(sock: AsyncFD): bool = + result = true + let res = recv(sock.SocketHandle, buf, size.cint, + flags.toOSFlags()) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete(res) + # TODO: The following causes a massive slowdown. + #if not cb(socket): + addRead(socket, cb) + return retFuture + + proc send*(socket: AsyncFD, data: string, + flags = {SocketFlag.SafeDisconn}): Future[void] = + var retFuture = newFuture[void]("send") + + var written = 0 + + proc cb(sock: AsyncFD): bool = + result = true + let netSize = data.len-written + var d = data.cstring + let res = send(sock.SocketHandle, addr d[written], netSize.cint, + MSG_NOSIGNAL) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + written.inc(res) + if res != netSize: + result = false # We still have data to send. + else: + retFuture.complete() + # TODO: The following causes crashes. + #if not cb(socket): + addWrite(socket, cb) + return retFuture + + proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, + saddrLen: SockLen, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` of size ``size`` in bytes to specified destination + ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``. + ## The returned future will complete once all data has been sent. + var retFuture = newFuture[void]("sendTo") + + # we will preserve address in our stack + var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes + var stalen = saddrLen + zeroMem(addr(staddr[0]), 128) + copyMem(addr(staddr[0]), saddr, saddrLen) + + proc cb(sock: AsyncFD): bool = + result = true + let res = sendto(sock.SocketHandle, data, size, MSG_NOSIGNAL, + cast[ptr SockAddr](addr(staddr[0])), stalen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + retFuture.complete() + + addWrite(socket, cb) + return retFuture + + proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, + saddr: ptr SockAddr, saddrLen: ptr SockLen, + flags = {SocketFlag.SafeDisconn}): Future[int] = + ## Receives a datagram data from ``socket`` into ``data``, which must + ## be at least of size ``size`` in bytes, address of datagram's sender + ## will be stored into ``saddr`` and ``saddrLen``. Returned future will + ## complete once one datagram has been received, and will return size + ## of packet received. + var retFuture = newFuture[int]("recvFromInto") + proc cb(sock: AsyncFD): bool = + result = true + let res = recvfrom(sock.SocketHandle, data, size.cint, flags.toOSFlags(), + saddr, saddrLen) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false + else: + retFuture.complete(res) + addRead(socket, cb) + return retFuture + + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): + Future[tuple[address: string, client: AsyncFD]] = + var retFuture = newFuture[tuple[address: string, + client: AsyncFD]]("acceptAddr") + proc cb(sock: AsyncFD): bool = + result = true + var sockAddress: Sockaddr_storage + var addrLen = sizeof(sockAddress).Socklen + var client = accept(sock.SocketHandle, + cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + if client == osInvalidSocket: + let lastError = osLastError() + assert lastError.int32 notin {EWOULDBLOCK, EAGAIN} + if lastError.int32 == EINTR: + return false + else: + if flags.isDisconnectionError(lastError): + return false + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + register(client.AsyncFD) + retFuture.complete((getAddrString(cast[ptr SockAddr](addr sockAddress)), + client.AsyncFD)) + addRead(socket, cb) + return retFuture + + when supportedPlatform: + + proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = + ## Start watching for timeout expiration, and then call the + ## callback ``cb``. + ## ``timeout`` - time in milliseconds, + ## ``oneshot`` - if ``true`` only one event will be dispatched, + ## if ``false`` continuous events every ``timeout`` milliseconds. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerTimer(timeout, oneshot, data) + + proc addSignal*(signal: int, cb: Callback) = + ## Start watching signal ``signal``, and when signal appears, call the + ## callback ``cb``. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerSignal(signal, data) + + proc addProcess*(pid: int, cb: Callback) = + ## Start watching for process exit with pid ``pid``, and then call + ## the callback ``cb``. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerProcess(pid, data) + + proc newAsyncEvent*(): AsyncEvent = + ## Creates new ``AsyncEvent``. + result = AsyncEvent(ioselectors.newSelectEvent()) + + proc setEvent*(ev: AsyncEvent) = + ## Sets new ``AsyncEvent`` to signaled state. + ioselectors.setEvent(SelectEvent(ev)) + + proc close*(ev: AsyncEvent) = + ## Closes ``AsyncEvent`` + ioselectors.close(SelectEvent(ev)) + + proc addEvent*(ev: AsyncEvent, cb: Callback) = + ## Start watching for event ``ev``, and call callback ``cb``, when + ## ev will be set to signaled state. + let p = getGlobalDispatcher() + var data = AsyncData(readCB: cb) + p.selector.registerEvent(SelectEvent(ev), data) + +proc sleepAsync*(ms: int): Future[void] = + ## Suspends the execution of the current async procedure for the next + ## ``ms`` milliseconds. + var retFuture = newFuture[void]("sleepAsync") + let p = getGlobalDispatcher() + p.timers.push((epochTime() + (ms / 1000), retFuture)) + return retFuture + +proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] = + ## Returns a future which will complete once ``fut`` completes or after + ## ``timeout`` milliseconds has elapsed. + ## + ## If ``fut`` completes first the returned future will hold true, + ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned + ## future will hold false. + + var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") + var timeoutFuture = sleepAsync(timeout) + fut.callback = + proc () = + if not retFuture.finished: retFuture.complete(true) + timeoutFuture.callback = + proc () = + if not retFuture.finished: retFuture.complete(false) + return retFuture + +proc accept*(socket: AsyncFD, + flags = {SocketFlag.SafeDisconn}): Future[AsyncFD] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection. + ## The future will complete when the connection is successfully accepted. + var retFut = newFuture[AsyncFD]("accept") + var fut = acceptAddr(socket, flags) + fut.callback = + proc (future: Future[tuple[address: string, client: AsyncFD]]) = + assert future.finished + if future.failed: + retFut.fail(future.error) + else: + retFut.complete(future.read.client) + return retFut + +# -- Await Macro + +proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = + # Skips a nest of StmtList's. + result = node + if node[0].kind == nnkStmtList: + result = skipUntilStmtList(node[0]) + +proc skipStmtList(node: NimNode): NimNode {.compileTime.} = + result = node + if node[0].kind == nnkStmtList: + result = node[0] + +template createCb(retFutureSym, iteratorNameSym, + name: expr): stmt {.immediate.} = + var nameIterVar = iteratorNameSym + #{.push stackTrace: off.} + proc cb {.closure,gcsafe.} = + try: + if not nameIterVar.finished: + var next = nameIterVar() + if next == nil: + assert retFutureSym.finished, "Async procedure's (" & + name & ") return Future was not finished." + else: + next.callback = cb + except: + if retFutureSym.finished: + # Take a look at tasyncexceptions for the bug which this fixes. + # That test explains it better than I can here. + raise + else: + retFutureSym.fail(getCurrentException()) + cb() + #{.pop.} +proc generateExceptionCheck(futSym, + tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = + if tryStmt.kind == nnkNilLit: + result = rootReceiver + else: + var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[] + let errorNode = newDotExpr(futSym, newIdentNode("error")) + for i in 1 .. <tryStmt.len: + let exceptBranch = tryStmt[i] + if exceptBranch[0].kind == nnkStmtList: + exceptionChecks.add((newIdentNode("true"), exceptBranch[0])) + else: + var exceptIdentCount = 0 + var ifCond: NimNode + for i in 0 .. <exceptBranch.len: + let child = exceptBranch[i] + if child.kind == nnkIdent: + let cond = infix(errorNode, "of", child) + if exceptIdentCount == 0: + ifCond = cond + else: + ifCond = infix(ifCond, "or", cond) + else: + break + exceptIdentCount.inc + + expectKind(exceptBranch[exceptIdentCount], nnkStmtList) + exceptionChecks.add((ifCond, exceptBranch[exceptIdentCount])) + # -> -> else: raise futSym.error + exceptionChecks.add((newIdentNode("true"), + newNimNode(nnkRaiseStmt).add(errorNode))) + # Read the future if there is no error. + # -> else: futSym.read + let elseNode = newNimNode(nnkElse, fromNode) + elseNode.add newNimNode(nnkStmtList, fromNode) + elseNode[0].add rootReceiver + + let ifBody = newStmtList() + ifBody.add newCall(newIdentNode("setCurrentException"), errorNode) + ifBody.add newIfStmt(exceptionChecks) + ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit()) + + result = newIfStmt( + (newDotExpr(futSym, newIdentNode("failed")), ifBody) + ) + result.add elseNode + +template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, + rootReceiver: expr, fromNode: NimNode) = + ## Params: + ## futureVarNode: The NimNode which is a symbol identifying the Future[T] + ## variable to yield. + ## fromNode: Used for better debug information (to give context). + ## valueReceiver: The node which defines an expression that retrieves the + ## future's value. + ## + ## rootReceiver: ??? TODO + # -> yield future<x> + result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) + # -> future<x>.read + valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) + result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, + fromNode) + +template createVar(result: var NimNode, futSymName: string, + asyncProc: NimNode, + valueReceiver, rootReceiver: expr, + fromNode: NimNode) = + result = newNimNode(nnkStmtList, fromNode) + var futSym = genSym(nskVar, "future") + result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y + useVar(result, futSym, valueReceiver, rootReceiver, fromNode) + +proc processBody(node, retFutureSym: NimNode, + subTypeIsVoid: bool, + tryStmt: NimNode): NimNode {.compileTime.} = + #echo(node.treeRepr) + result = node + case node.kind + of nnkReturnStmt: + result = newNimNode(nnkStmtList, node) + if node[0].kind == nnkEmpty: + if not subTypeIsVoid: + result.add newCall(newIdentNode("complete"), retFutureSym, + newIdentNode("result")) + else: + result.add newCall(newIdentNode("complete"), retFutureSym) + else: + result.add newCall(newIdentNode("complete"), retFutureSym, + node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt)) + + result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) + return # Don't process the children of this return stmt + of nnkCommand, nnkCall: + if node[0].kind == nnkIdent and node[0].ident == !"await": + case node[1].kind + of nnkIdent, nnkInfix, nnkDotExpr: + # await x + # await x or y + result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x + of nnkCall, nnkCommand: + # await foo(p, x) + # await foo p, x + var futureValue: NimNode + result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, + futureValue, node) + else: + error("Invalid node kind in 'await', got: " & $node[1].kind) + elif node.len > 1 and node[1].kind == nnkCommand and + node[1][0].kind == nnkIdent and node[1][0].ident == !"await": + # foo await x + var newCommand = node + result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], + newCommand, node) + + of nnkVarSection, nnkLetSection: + case node[0][2].kind + of nnkCommand: + if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": + # var x = await y + var newVarSection = node # TODO: Should this use copyNimNode? + result.createVar("future" & $node[0][0].ident, node[0][2][1], + newVarSection[0][2], newVarSection, node) + else: discard + of nnkAsgn: + case node[1].kind + of nnkCommand: + if node[1][0].ident == !"await": + # x = await y + var newAsgn = node + result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node) + else: discard + of nnkDiscardStmt: + # discard await x + if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and + node[0][0].ident == !"await": + var newDiscard = node + result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], + newDiscard[0], newDiscard, node) + of nnkTryStmt: + # try: await x; except: ... + result = newNimNode(nnkStmtList, node) + template wrapInTry(n, tryBody: expr) = + var temp = n + n[0] = tryBody + tryBody = temp + + # Transform ``except`` body. + # TODO: Could we perform some ``await`` transformation here to get it + # working in ``except``? + tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, nil) + + proc processForTry(n: NimNode, i: var int, + res: NimNode): bool {.compileTime.} = + ## Transforms the body of the tryStmt. Does not transform the + ## body in ``except``. + ## Returns true if the tryStmt node was transformed into an ifStmt. + result = false + var skipped = n.skipStmtList() + while i < skipped.len: + var processed = processBody(skipped[i], retFutureSym, + subTypeIsVoid, n) + + # Check if we transformed the node into an exception check. + # This suggests skipped[i] contains ``await``. + if processed.kind != skipped[i].kind or processed.len != skipped[i].len: + processed = processed.skipUntilStmtList() + expectKind(processed, nnkStmtList) + expectKind(processed[2][1], nnkElse) + i.inc + + if not processForTry(n, i, processed[2][1][0]): + # We need to wrap the nnkElse nodes back into a tryStmt. + # As they are executed if an exception does not happen + # inside the awaited future. + # The following code will wrap the nodes inside the + # original tryStmt. + wrapInTry(n, processed[2][1][0]) + + res.add processed + result = true + else: + res.add skipped[i] + i.inc + var i = 0 + if not processForTry(node, i, result): + # If the tryStmt hasn't been transformed we can just put the body + # back into it. + wrapInTry(node, result) + return + else: discard + + for i in 0 .. <result.len: + result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, nil) + +proc getName(node: NimNode): string {.compileTime.} = + case node.kind + of nnkPostfix: + return $node[1].ident + of nnkIdent: + return $node.ident + of nnkEmpty: + return "anonymous" + else: + error("Unknown name.") + +proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = + ## This macro transforms a single procedure into a closure iterator. + ## The ``async`` macro supports a stmtList holding multiple async procedures. + if prc.kind notin {nnkProcDef, nnkLambda}: + error("Cannot transform this node kind into an async proc." & + " Proc definition or lambda node expected.") + + hint("Processing " & prc[0].getName & " as an async proc.") + + let returnType = prc[3][0] + var baseType: NimNode + # Verify that the return type is a Future[T] + if returnType.kind == nnkBracketExpr: + let fut = repr(returnType[0]) + if fut != "Future": + error("Expected return type of 'Future' got '" & fut & "'") + baseType = returnType[1] + elif returnType.kind in nnkCallKinds and $returnType[0] == "[]": + let fut = repr(returnType[1]) + if fut != "Future": + error("Expected return type of 'Future' got '" & fut & "'") + baseType = returnType[2] + elif returnType.kind == nnkEmpty: + baseType = returnType + else: + error("Expected return type of 'Future' got '" & repr(returnType) & "'") + + let subtypeIsVoid = returnType.kind == nnkEmpty or + (baseType.kind == nnkIdent and returnType[1].ident == !"void") + + var outerProcBody = newNimNode(nnkStmtList, prc[6]) + + # -> var retFuture = newFuture[T]() + var retFutureSym = genSym(nskVar, "retFuture") + var subRetType = + if returnType.kind == nnkEmpty: newIdentNode("void") + else: baseType + outerProcBody.add( + newVarStmt(retFutureSym, + newCall( + newNimNode(nnkBracketExpr, prc[6]).add( + newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. + subRetType), + newLit(prc[0].getName)))) # Get type from return type of this proc + + # -> iterator nameIter(): FutureBase {.closure.} = + # -> {.push warning[resultshadowed]: off.} + # -> var result: T + # -> {.pop.} + # -> <proc_body> + # -> complete(retFuture, result) + var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") + var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) + if not subtypeIsVoid: + procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), + newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( + newIdentNode("warning"), newIdentNode("resultshadowed")), + newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} + + procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( + newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T + + procBody.insert(2, newNimNode(nnkPragma).add( + newIdentNode("pop"))) # -> {.pop.}) + + procBody.add( + newCall(newIdentNode("complete"), + retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) + else: + # -> complete(retFuture) + procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + + var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], + procBody, nnkIteratorDef) + closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) + outerProcBody.add(closureIterator) + + # -> createCb(retFuture) + #var cbName = newIdentNode("cb") + var procCb = newCall(bindSym"createCb", retFutureSym, iteratorNameSym, + newStrLitNode(prc[0].getName)) + outerProcBody.add procCb + + # -> return retFuture + outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) + + result = prc + + # Remove the 'async' pragma. + for i in 0 .. <result[4].len: + if result[4][i].kind == nnkIdent and result[4][i].ident == !"async": + result[4].del(i) + result[4] = newEmptyNode() + if subtypeIsVoid: + # Add discardable pragma. + if returnType.kind == nnkEmpty: + # Add Future[void] + result[3][0] = parseExpr("Future[void]") + + result[6] = outerProcBody + + #echo(treeRepr(result)) + #if prc[0].getName == "testInfix": + # echo(toStrLit(result)) + +macro async*(prc: stmt): stmt {.immediate.} = + ## Macro which processes async procedures into the appropriate + ## iterators and yield statements. + if prc.kind == nnkStmtList: + for oneProc in prc: + result = newStmtList() + result.add asyncSingleProc(oneProc) + else: + result = asyncSingleProc(prc) + +proc recvLine*(socket: AsyncFD): Future[string] {.async.} = + ## Reads a line of data from ``socket``. Returned future will complete once + ## a full line is read or an error occurs. + ## + ## If a full line is read ``\r\L`` is not + ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## will be set to it. + ## + ## If the socket is disconnected, ``line`` will be set to ``""``. + ## + ## If the socket is disconnected in the middle of a line (before ``\r\L`` + ## is read) then line will be set to ``""``. + ## The partial line **will be lost**. + ## + ## **Warning**: This assumes that lines are delimited by ``\r\L``. + ## + ## **Note**: This procedure is mostly used for testing. You likely want to + ## use ``asyncnet.recvLine`` instead. + + template addNLIfEmpty(): stmt = + if result.len == 0: + result.add("\c\L") + + result = "" + var c = "" + while true: + c = await recv(socket, 1) + if c.len == 0: + return "" + if c == "\r": + c = await recv(socket, 1) + assert c == "\l" + addNLIfEmpty() + return + elif c == "\L": + addNLIfEmpty() + return + add(result, c) + +proc callSoon*(cbproc: proc ()) = + ## Schedule `cbproc` to be called as soon as possible. + ## The callback is called when control returns to the event loop. + getGlobalDispatcher().callbacks.enqueue(cbproc) + +proc runForever*() = + ## Begins a never ending global dispatcher poll loop. + while true: + poll() + +proc waitFor*[T](fut: Future[T]): T = + ## **Blocks** the current thread until the specified future completes. + while not fut.finished: + poll() + + fut.read diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 5e899dd1a..b766ead9c 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -36,6 +36,8 @@ type DWORD* = int32 PDWORD* = ptr DWORD LPINT* = ptr int32 + ULONG_PTR* = uint + PULONG_PTR* = ptr uint HDC* = Handle HGLRC* = Handle @@ -92,7 +94,7 @@ type dwMinorVersion*: DWORD dwBuildNumber*: DWORD dwPlatformId*: DWORD - szCSDVersion*: array[0..127, WinChar]; + szCSDVersion*: array[0..127, WinChar] {.deprecated: [THandle: Handle, TSECURITY_ATTRIBUTES: SECURITY_ATTRIBUTES, TSTARTUPINFO: STARTUPINFO, TPROCESS_INFORMATION: PROCESS_INFORMATION, @@ -440,6 +442,8 @@ type sa_family*: int16 # unsigned sa_data: array[0..13, char] + PSockAddr = ptr SockAddr + InAddr* {.importc: "IN_ADDR", header: "winsock2.h".} = object s_addr*: uint32 # IP address @@ -451,7 +455,7 @@ type sin_zero*: array[0..7, char] In6_addr* {.importc: "IN6_ADDR", header: "winsock2.h".} = object - bytes*: array[0..15, char] + bytes* {.importc: "u.Byte".}: array[0..15, char] Sockaddr_in6* {.importc: "SOCKADDR_IN6", header: "ws2tcpip.h".} = object @@ -516,6 +520,8 @@ var SO_DEBUG* {.importc, header: "winsock2.h".}: cint ## turn on debugging info recording SO_ACCEPTCONN* {.importc, header: "winsock2.h".}: cint # socket has had listen() SO_REUSEADDR* {.importc, header: "winsock2.h".}: cint # allow local address reuse + SO_REUSEPORT* {.importc: "SO_REUSEADDR", header: "winsock2.h".}: cint # allow port reuse. Since Windows does not really support it, mapped to SO_REUSEADDR. This shouldn't cause problems. + SO_KEEPALIVE* {.importc, header: "winsock2.h".}: cint # keep connections alive SO_DONTROUTE* {.importc, header: "winsock2.h".}: cint # just use interface addresses SO_BROADCAST* {.importc, header: "winsock2.h".}: cint # permit sending of broadcast msgs @@ -744,7 +750,7 @@ type D1*: int32 D2*: int16 D3*: int16 - D4*: array [0..7, int8] + D4*: array[0..7, int8] {.deprecated: [TOVERLAPPED: OVERLAPPED, TGUID: GUID].} const @@ -757,22 +763,29 @@ const WSAENETRESET* = 10052 WSAETIMEDOUT* = 10060 ERROR_NETNAME_DELETED* = 64 + STATUS_PENDING* = 0x103 proc createIoCompletionPort*(FileHandle: Handle, ExistingCompletionPort: Handle, - CompletionKey: DWORD, + CompletionKey: ULONG_PTR, NumberOfConcurrentThreads: DWORD): Handle{.stdcall, dynlib: "kernel32", importc: "CreateIoCompletionPort".} proc getQueuedCompletionStatus*(CompletionPort: Handle, - lpNumberOfBytesTransferred: PDWORD, lpCompletionKey: PULONG, + lpNumberOfBytesTransferred: PDWORD, lpCompletionKey: PULONG_PTR, lpOverlapped: ptr POVERLAPPED, dwMilliseconds: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", importc: "GetQueuedCompletionStatus".} -proc getOverlappedResult*(hFile: Handle, lpOverlapped: OVERLAPPED, +proc getOverlappedResult*(hFile: Handle, lpOverlapped: POVERLAPPED, lpNumberOfBytesTransferred: var DWORD, bWait: WINBOOL): WINBOOL{. stdcall, dynlib: "kernel32", importc: "GetOverlappedResult".} +# this is copy of HasOverlappedIoCompleted() macro from <winbase.h> +# because we have declared own OVERLAPPED structure with member names not +# compatible with original names. +template hasOverlappedIoCompleted*(lpOverlapped): bool = + (cast[uint](lpOverlapped.internal) != STATUS_PENDING) + const IOC_OUT* = 0x40000000 IOC_IN* = 0x80000000 @@ -812,11 +825,23 @@ proc WSARecv*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. stdcall, importc: "WSARecv", dynlib: "Ws2_32.dll".} +proc WSARecvFrom*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, + bytesReceived: PDWORD, flags: PDWORD, name: ptr SockAddr, + namelen: ptr cint, lpOverlapped: POVERLAPPED, + completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. + stdcall, importc: "WSARecvFrom", dynlib: "Ws2_32.dll".} + proc WSASend*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, bytesSent: PDWORD, flags: DWORD, lpOverlapped: POVERLAPPED, completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. stdcall, importc: "WSASend", dynlib: "Ws2_32.dll".} +proc WSASendTo*(s: SocketHandle, buf: ptr TWSABuf, bufCount: DWORD, + bytesSent: PDWORD, flags: DWORD, name: ptr SockAddr, + namelen: cint, lpOverlapped: POVERLAPPED, + completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {. + stdcall, importc: "WSASendTo", dynlib: "Ws2_32.dll".} + proc get_osfhandle*(fd:FileHandle): Handle {. importc: "_get_osfhandle", header:"<io.h>".} @@ -878,3 +903,136 @@ proc inet_ntop*(family: cint, paddr: pointer, pStringBuffer: cstring, result = inet_ntop_real(family, paddr, pStringBuffer, stringBufSize) else: result = inet_ntop_emulated(family, paddr, pStringBuffer, stringBufSize) + +type + WSAPROC_ACCEPTEX* = proc (sListenSocket: SocketHandle, + sAcceptSocket: SocketHandle, + lpOutputBuffer: pointer, dwReceiveDataLength: DWORD, + dwLocalAddressLength: DWORD, + dwRemoteAddressLength: DWORD, + lpdwBytesReceived: ptr DWORD, + lpOverlapped: POVERLAPPED): bool {. + stdcall,gcsafe.} + + WSAPROC_CONNECTEX* = proc (s: SocketHandle, name: ptr SockAddr, namelen: cint, + lpSendBuffer: pointer, dwSendDataLength: DWORD, + lpdwBytesSent: ptr DWORD, + lpOverlapped: POVERLAPPED): bool {. + stdcall,gcsafe.} + + WSAPROC_GETACCEPTEXSOCKADDRS* = proc(lpOutputBuffer: pointer, + dwReceiveDataLength: DWORD, + dwLocalAddressLength: DWORD, + dwRemoteAddressLength: DWORD, + LocalSockaddr: ptr PSockAddr, + LocalSockaddrLength: ptr cint, + RemoteSockaddr: ptr PSockAddr, + RemoteSockaddrLength: ptr cint) {. + stdcall,gcsafe.} + +const + WT_EXECUTEDEFAULT* = 0x00000000'i32 + WT_EXECUTEINIOTHREAD* = 0x00000001'i32 + WT_EXECUTEINUITHREAD* = 0x00000002'i32 + WT_EXECUTEINWAITTHREAD* = 0x00000004'i32 + WT_EXECUTEONLYONCE* = 0x00000008'i32 + WT_EXECUTELONGFUNCTION* = 0x00000010'i32 + WT_EXECUTEINTIMERTHREAD* = 0x00000020'i32 + WT_EXECUTEINPERSISTENTIOTHREAD* = 0x00000040'i32 + WT_EXECUTEINPERSISTENTTHREAD* = 0x00000080'i32 + WT_TRANSFER_IMPERSONATION* = 0x00000100'i32 + PROCESS_TERMINATE* = 0x00000001'i32 + PROCESS_CREATE_THREAD* = 0x00000002'i32 + PROCESS_SET_SESSIONID* = 0x00000004'i32 + PROCESS_VM_OPERATION* = 0x00000008'i32 + PROCESS_VM_READ* = 0x00000010'i32 + PROCESS_VM_WRITE* = 0x00000020'i32 + PROCESS_DUP_HANDLE* = 0x00000040'i32 + PROCESS_CREATE_PROCESS* = 0x00000080'i32 + PROCESS_SET_QUOTA* = 0x00000100'i32 + PROCESS_SET_INFORMATION* = 0x00000200'i32 + PROCESS_QUERY_INFORMATION* = 0x00000400'i32 + PROCESS_SUSPEND_RESUME* = 0x00000800'i32 + PROCESS_QUERY_LIMITED_INFORMATION* = 0x00001000'i32 + PROCESS_SET_LIMITED_INFORMATION* = 0x00002000'i32 +type + WAITORTIMERCALLBACK* = proc(para1: pointer, para2: int32): void {.stdcall.} + +proc postQueuedCompletionStatus*(CompletionPort: HANDLE, + dwNumberOfBytesTransferred: DWORD, + dwCompletionKey: ULONG_PTR, + lpOverlapped: pointer): bool + {.stdcall, dynlib: "kernel32", importc: "PostQueuedCompletionStatus".} + +proc registerWaitForSingleObject*(phNewWaitObject: ptr Handle, hObject: Handle, + Callback: WAITORTIMERCALLBACK, + Context: pointer, + dwMilliseconds: ULONG, + dwFlags: ULONG): bool + {.stdcall, dynlib: "kernel32", importc: "RegisterWaitForSingleObject".} + +proc unregisterWait*(WaitHandle: HANDLE): DWORD + {.stdcall, dynlib: "kernel32", importc: "UnregisterWait".} + +proc openProcess*(dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, + dwProcessId: DWORD): Handle + {.stdcall, dynlib: "kernel32", importc: "OpenProcess".} + +when defined(useWinAnsi): + proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, + bManualReset: DWORD, bInitialState: DWORD, + lpName: cstring): Handle + {.stdcall, dynlib: "kernel32", importc: "CreateEventA".} +else: + proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, + bManualReset: DWORD, bInitialState: DWORD, + lpName: ptr Utf16Char): Handle + {.stdcall, dynlib: "kernel32", importc: "CreateEventW".} + +proc setEvent*(hEvent: Handle): cint + {.stdcall, dynlib: "kernel32", importc: "SetEvent".} + +const + FD_READ* = 0x00000001'i32 + FD_WRITE* = 0x00000002'i32 + FD_OOB* = 0x00000004'i32 + FD_ACCEPT* = 0x00000008'i32 + FD_CONNECT* = 0x00000010'i32 + FD_CLOSE* = 0x00000020'i32 + FD_QQS* = 0x00000040'i32 + FD_GROUP_QQS* = 0x00000080'i32 + FD_ROUTING_INTERFACE_CHANGE* = 0x00000100'i32 + FD_ADDRESS_LIST_CHANGE* = 0x00000200'i32 + FD_ALL_EVENTS* = 0x000003FF'i32 + +proc wsaEventSelect*(s: SocketHandle, hEventObject: Handle, + lNetworkEvents: clong): cint + {.stdcall, importc: "WSAEventSelect", dynlib: "ws2_32.dll".} + +proc wsaCreateEvent*(): Handle + {.stdcall, importc: "WSACreateEvent", dynlib: "ws2_32.dll".} + +proc wsaCloseEvent*(hEvent: Handle): bool + {.stdcall, importc: "WSACloseEvent", dynlib: "ws2_32.dll".} + +proc wsaResetEvent*(hEvent: Handle): bool + {.stdcall, importc: "WSAResetEvent", dynlib: "ws2_32.dll".} + +type + KEY_EVENT_RECORD* {.final, pure.} = object + eventType*: int16 + bKeyDown*: WINBOOL + wRepeatCount*: int16 + wVirtualKeyCode*: int16 + wVirtualScanCode*: int16 + uChar*: int16 + dwControlKeyState*: DWORD + +when defined(useWinAnsi): + proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, + lpNumberOfEventsRead: ptr cint): cint + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputA".} +else: + proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, + lpNumberOfEventsRead: ptr cint): cint + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} \ No newline at end of file diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 05843e2d3..9dad7e489 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -197,6 +197,7 @@ proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_free*(ssl: SslPtr){.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get_SSL_CTX*(ssl: SslPtr): SslCtx {.cdecl, dynlib: DLLSSLName, importc.} proc SSL_CTX_new*(meth: PSSL_METHOD): SslCtx{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_CTX_load_verify_locations*(ctx: SslCtx, CAfile: cstring, @@ -216,6 +217,10 @@ proc SSL_CTX_use_PrivateKey_file*(ctx: SslCtx, proc SSL_CTX_check_private_key*(ctx: SslCtx): cInt{.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_get_ex_new_index*(argl: clong, argp: pointer, new_func: pointer, dup_func: pointer, free_func: pointer): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_ex_data*(ssl: SslCtx, idx: cint, arg: pointer): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_get_ex_data*(ssl: SslCtx, idx: cint): pointer {.cdecl, dynlib: DLLSSLName, importc.} + proc SSL_set_fd*(ssl: SslPtr, fd: SocketHandle): cint{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_shutdown*(ssl: SslPtr): cInt{.cdecl, dynlib: DLLSSLName, importc.} @@ -260,7 +265,7 @@ proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSS proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} -when not useWinVersion: +when not useWinVersion and not defined(macosx): proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, dynlib: DLLUtilName, importc.} @@ -314,6 +319,25 @@ proc SSL_CTX_set_tlsext_servername_arg*(ctx: SslCtx, arg: pointer): int = ## Set the pointer to be used in the callback registered to ``SSL_CTX_set_tlsext_servername_callback``. result = SSL_CTX_ctrl(ctx, SSL_CTRL_SET_TLSEXT_SERVERNAME_ARG, 0, arg) +type + PskClientCallback* = proc (ssl: SslPtr; + hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar; + max_psk_len: cuint): cuint {.cdecl.} + + PskServerCallback* = proc (ssl: SslPtr; + identity: cstring; psk: ptr cuchar; max_psk_len: cint): cuint {.cdecl.} + +proc SSL_CTX_set_psk_client_callback*(ctx: SslCtx; callback: PskClientCallback) {.cdecl, dynlib: DLLSSLName, importc.} + ## Set callback called when OpenSSL needs PSK (for client). + +proc SSL_CTX_set_psk_server_callback*(ctx: SslCtx; callback: PskServerCallback) {.cdecl, dynlib: DLLSSLName, importc.} + ## Set callback called when OpenSSL needs PSK (for server). + +proc SSL_CTX_use_psk_identity_hint*(ctx: SslCtx; hint: cstring): cint {.cdecl, dynlib: DLLSSLName, importc.} + ## Set PSK identity hint to use. + +proc SSL_get_psk_identity*(ssl: SslPtr): cstring {.cdecl, dynlib: DLLSSLName, importc.} + ## Get PSK identity. proc bioNew*(b: PBIO_METHOD): BIO{.cdecl, dynlib: DLLUtilName, importc: "BIO_new".} proc bioFreeAll*(b: BIO){.cdecl, dynlib: DLLUtilName, importc: "BIO_free_all".} @@ -532,7 +556,7 @@ proc md5_File* (file: string): string {.raises: [IOError,Exception].} = result = hexStr(buf) -proc md5_Str* (str:string): string {.raises:[IOError].} = +proc md5_Str*(str:string): string = ##Generate MD5 hash for a string. Result is a 32 character #hex string with lowercase characters var diff --git a/readme.md b/readme.md index 27b2273f0..f56b054d6 100644 --- a/readme.md +++ b/readme.md @@ -1,18 +1,9 @@ -# Nim Compiler - -[](https://webchat.freenode.net/?channels=nim) -[](http://forum.nim-lang.org) -[](http://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15) -[](https://twitter.com/nim_lang) - -[](https://travis-ci.org/nim-lang/Nim) - -[](https://gratipay.com/nim/) -[](https://www.bountysource.com/teams/nim) - +# <img src="https://raw.githubusercontent.com/nim-lang/assets/master/Art/logo-crown.png" width="36"> Nim [](https://travis-ci.org/nim-lang/Nim) This repo contains the Nim compiler, Nim's stdlib, tools and -documentation. +documentation. For more information about Nim, including downloads +and documentation for the latest release, check out +[Nim's website](http://nim-lang.org). ## Compiling Compiling the Nim compiler is quite straightforward. Because @@ -36,6 +27,8 @@ To build from source you will need: are: clang, Visual C++, Intel's C++ compiler * git or wget +**Note:** When installing ``gcc`` on Ubuntu (and likely other distros) ensure that the ``build-essentials`` package is installed also. + If you are on a fairly modern *nix system, the following steps should work: ``` @@ -48,14 +41,15 @@ $ bin/nim c koch $ ./koch boot -d:release ``` -``koch install [dir]`` may then be used to install Nim, but lots of things -don't work then so don't do that. Add it to your PATH instead. More ``koch`` -related options are documented in [doc/koch.txt](doc/koch.txt). +You should then add the ``bin`` directory to your PATH, to make it easily +executable on your system. The above steps can be performed on Windows in a similar fashion, the ``build.bat`` and ``build64.bat`` (for x86_64 systems) are provided to be used instead of ``build.sh``. +The ``koch`` tool is the Nim build tool, more ``koch`` related options are +documented in [doc/koch.rst](doc/koch.rst). ## Nimble [Nimble](https://github.com/nim-lang/nimble) is Nim's package manager. For the @@ -66,12 +60,80 @@ the easiest way of installing Nimble is via: $ nim e install_nimble.nims ``` -## Getting help -A [forum](http://forum.nim-lang.org/) is available if you have any -questions, and you can also get help in the IRC channel on -[Freenode](irc://irc.freenode.net/nim) in #nim. If you ask questions on -[StackOverflow use the nim -tag](http://stackoverflow.com/questions/tagged/nim). +**Warning:** If you install Nimble this way, you will not be able to use binary +Nimble packages or update Nimble easily. +The [Nimble readme](https://github.com/nim-lang/nimble#installation) +provides thorough instructions on how to install Nimble, so that this isn't a +problem. + +## Community +[](https://webchat.freenode.net/?channels=nim) +[](http://forum.nim-lang.org) +[](http://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15) +[](https://twitter.com/nim_lang) + +* The [forum](http://forum.nim-lang.org/) - the best place to ask questions and to discuss Nim. +* [IRC (Freenode#nim)](https://webchat.freenode.net/?channels=nim) - the best place to discuss + Nim in real-time, this is also where most development decision get made! +* [Stackoverflow](http://stackoverflow.com/questions/tagged/nim) + +## Contributing + +[](https://gratipay.com/nim/) +[](https://www.bountysource.com/teams/nim) + +We welcome everyone's contributions to Nim. No matter how small or large +the contribution is, anything from small spelling fixes to large modules +intended to be included in the standard library are accepted. Before +you get started, you should know the following about this repositories +structure: + +* ``bin/``, ``build/`` - these directories are empty, but are used when Nim is built. +* ``compiler/`` - the compiler source code, all the Nim source code files in this + directory implement the compiler. This also includes nimfix, and plugins + which live in ``compiler/nimfix`` and ``compiler/plugins`` + respectively. Nimsuggest used to live in the ``compiler`` directory also, + but was moved to https://github.com/nim-lang/nimsuggest. +* ``config/`` - the configuration for the compiler and documentation generator. +* ``doc/`` - the documentation files in reStructuredText format. +* ``lib/`` - where the standard library lives. + * ``pure/`` - modules in the standard library written in pure Nim. + * ``impure/`` - modules in the standard library written in pure Nim which + depend on libraries written in other languages. + * ``wrappers/`` - modules which wrap libraries written in other languages. +* ``tests/`` - contains tests for the compiler and standard library, organised by + category. +* ``tools/`` - the tools including ``niminst`` and ``nimweb``, most of these are invoked + via ``koch``. +* ``web/`` - the Nim website (http://nim-lang.org). +* ``koch.nim`` - tool used to bootstrap Nim, generate C sources, build the website, documentation + and more. + +Most importantly, the ``koch`` tool can be used to run the test suite. To do so compile it first +by executing ``nim c koch``, then execute ``./koch tests``. The test suite takes a while to run, +but you can run specific tests by specifying a category to run, for example ``./koch tests cat async``. + +Make sure that the tests all pass before +[submitting your pull request](https://help.github.com/articles/using-pull-requests/). +If you're short on time, you can +just run the tests specific to your change. Just run the category which corresponds to the change +you've made. When you create your pull request, Travis CI will verify that all the tests pass +anyway. + +If you're looking for things to do, take a look at our +[issue tracker](https://github.com/nim-lang/Nim/issues). There is always plenty of issues +labelled [``Easy``](https://github.com/nim-lang/Nim/labels/Easy), these should be a good +starting point if this is your first contribution to Nim. + +You can also help with the development of Nim by making donations. You can do so +in many ways: + +* [Gratipay](https://gratipay.com/nim/) +* [Bountysource](https://www.bountysource.com/teams/nim) +* Bitcoin - 1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ + +Finally, if you have any questions feel free to submit a question on the issue tracker, +on the [Nim forum](http://forum.nim-lang.org), or on IRC. ## License The compiler and the standard library are licensed under the MIT license, diff --git a/tests/assert/tunittests.nim b/tests/assert/tunittests.nim index cbbfe63c6..de917511c 100644 --- a/tests/assert/tunittests.nim +++ b/tests/assert/tunittests.nim @@ -1 +1,4 @@ +discard """ +output: "" +""" import "../template/utemplates", "../closure/uclosures" diff --git a/tests/async/tasync_forward.nim b/tests/async/tasync_forward.nim new file mode 100644 index 000000000..ffb7acafd --- /dev/null +++ b/tests/async/tasync_forward.nim @@ -0,0 +1,9 @@ + +import asyncdispatch + +# bug #1970 + +proc foo {.async.} + +proc foo {.async.} = + discard diff --git a/tests/async/tasyncall.nim b/tests/async/tasyncall.nim new file mode 100644 index 000000000..60ba557cc --- /dev/null +++ b/tests/async/tasyncall.nim @@ -0,0 +1,68 @@ +discard """ + file: "tasyncall.nim" + exitcode: 0 +""" +import times, sequtils +import asyncdispatch + +const + taskCount = 10 + sleepDuration = 500 + +proc futureWithValue(x: int): Future[int] {.async.} = + await sleepAsync(sleepDuration) + return x + +proc futureWithoutValue() {.async.} = + await sleepAsync(1000) + +proc testFuturesWithValue(x: int): seq[int] = + var tasks = newSeq[Future[int]](taskCount) + + for i in 0..<taskCount: + tasks[i] = futureWithValue(x) + + result = waitFor all(tasks) + +proc testFuturesWithoutValues() = + var tasks = newSeq[Future[void]](taskCount) + + for i in 0..<taskCount: + tasks[i] = futureWithoutValue() + + waitFor all(tasks) + +proc testVarargs(x, y, z: int): seq[int] = + let + a = futureWithValue(x) + b = futureWithValue(y) + c = futureWithValue(z) + + result = waitFor all(a, b, c) + +block: + let + startTime = cpuTime() + results = testFuturesWithValue(42) + expected = repeat(42, taskCount) + execTime = cpuTime() - startTime + + doAssert execTime * 1000 < taskCount * sleepDuration + doAssert results == expected + +block: + let startTime = cpuTime() + testFuturesWithoutValues() + let execTime = cpuTime() - startTime + + doAssert execTime * 1000 < taskCount * sleepDuration + +block: + let + startTime = cpuTime() + results = testVarargs(1, 2, 3) + expected = @[1, 2, 3] + execTime = cpuTime() - startTime + + doAssert execTime * 100 < taskCount * sleepDuration + doAssert results == expected diff --git a/tests/async/tasyncdiscard.nim b/tests/async/tasyncdiscard.nim index 71aba29e2..e7c87ad42 100644 --- a/tests/async/tasyncdiscard.nim +++ b/tests/async/tasyncdiscard.nim @@ -36,4 +36,4 @@ proc main {.async.} = discard await g() echo 6 -asyncCheck main() +waitFor(main()) diff --git a/tests/async/tasynceverror.nim b/tests/async/tasynceverror.nim index 22b4fe9a7..dd05c831b 100644 --- a/tests/async/tasynceverror.nim +++ b/tests/async/tasynceverror.nim @@ -1,9 +1,9 @@ discard """ file: "tasynceverror.nim" exitcode: 1 - outputsub: "Error: unhandled exception: Connection reset by peer" + outputsub: "Error: unhandled exception: " """ - +# error message is actually different on OSX import asyncdispatch, asyncnet, @@ -43,7 +43,7 @@ else: await s.connect(testHost, testPort) var ps = await ls.accept() - SocketHandle(ls).close() + closeSocket(ls) await ps.send("test 1", flags={}) s.close() diff --git a/tests/async/tasyncexceptions.nim b/tests/async/tasyncexceptions.nim index aab08e30f..efe31ef27 100644 --- a/tests/async/tasyncexceptions.nim +++ b/tests/async/tasyncexceptions.nim @@ -5,6 +5,8 @@ discard """ """ import asyncdispatch +# Note: This is a test case for a bug. + proc accept(): Future[int] {.async.} = await sleepAsync(100) result = 4 diff --git a/tests/async/tasyncfile.nim b/tests/async/tasyncfile.nim index 05cda5e5f..26a9bb391 100644 --- a/tests/async/tasyncfile.nim +++ b/tests/async/tasyncfile.nim @@ -24,7 +24,7 @@ proc main() {.async.} = var file = openAsync(fn, fmAppend) await file.write("\ntest2") let errorTest = file.readAll() - await errorTest + echo await errorTest doAssert errorTest.failed file.close() file = openAsync(fn, fmRead) diff --git a/tests/async/tasynctry.nim b/tests/async/tasynctry.nim index f77198e2e..5930f296f 100644 --- a/tests/async/tasynctry.nim +++ b/tests/async/tasynctry.nim @@ -48,7 +48,7 @@ proc catch() {.async.} = except OSError, EInvalidField: assert false -asyncCheck catch() +waitFor catch() proc test(): Future[bool] {.async.} = result = false @@ -92,13 +92,13 @@ proc test4(): Future[int] {.async.} = result = 2 var x = test() -assert x.read +assert x.waitFor() x = test2() -assert x.read +assert x.waitFor() var y = test3() -assert y.read == 2 +assert y.waitFor() == 2 y = test4() -assert y.read == 2 +assert y.waitFor() == 2 diff --git a/tests/async/tawaitsemantics.nim b/tests/async/tawaitsemantics.nim new file mode 100644 index 000000000..3e0c3903e --- /dev/null +++ b/tests/async/tawaitsemantics.nim @@ -0,0 +1,59 @@ +discard """ + file: "tawaitsemantics.nim" + exitcode: 0 + output: ''' +Error caught +Test infix +Test call +''' +""" + +import asyncdispatch + +# This tests the behaviour of 'await' under different circumstances. +# For example, when awaiting Future variable and this future has failed the +# exception shouldn't be raised as described here +# https://github.com/nim-lang/Nim/issues/4170 + +proc thrower(): Future[void] = + result = newFuture[void]() + result.fail(newException(Exception, "Test")) + +proc dummy: Future[void] = + result = newFuture[void]() + result.complete() + +proc testInfix() {.async.} = + # Test the infix operator semantics. + var fut = thrower() + var fut2 = dummy() + await fut or fut2 # Shouldn't raise. + # TODO: what about: await thrower() or fut2? + +proc testCall() {.async.} = + await thrower() + +proc tester() {.async.} = + # Test that we can handle exceptions without 'try' + var fut = thrower() + doAssert fut.finished + doAssert fut.failed + doAssert fut.error.msg == "Test" + await fut # We are awaiting a 'Future', so no `read` occurs. + doAssert fut.finished + doAssert fut.failed + doAssert fut.error.msg == "Test" + echo("Error caught") + + fut = testInfix() + await fut + doAssert fut.finished + doAssert(not fut.failed) + echo("Test infix") + + fut = testCall() + await fut + doAssert fut.failed + echo("Test call") + +waitFor(tester()) diff --git a/tests/async/tgeneric_async.nim b/tests/async/tgeneric_async.nim new file mode 100644 index 000000000..af6370181 --- /dev/null +++ b/tests/async/tgeneric_async.nim @@ -0,0 +1,9 @@ + +import asyncdispatch + +when true: + # bug #2377 + proc test[T](v: T) {.async.} = + echo $v + + asyncCheck test[int](1) diff --git a/tests/async/tioselectors.nim b/tests/async/tioselectors.nim new file mode 100644 index 000000000..2237de01a --- /dev/null +++ b/tests/async/tioselectors.nim @@ -0,0 +1,408 @@ +discard """ + file: "tioselectors.nim" + output: "All tests passed!" +""" +import ioselectors + +const hasThreadSupport = compileOption("threads") + +template processTest(t, x: untyped) = + #stdout.write(t) + #stdout.flushFile() + if not x: echo(t & " FAILED\r\n") + +when not defined(windows): + import os, posix, osproc, nativesockets, times + + const supportedPlatform = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(linux) + + proc socket_notification_test(): bool = + proc create_test_socket(): SocketHandle = + var sock = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP) + var x: int = fcntl(sock, F_GETFL, 0) + if x == -1: raiseOSError(osLastError()) + else: + var mode = x or O_NONBLOCK + if fcntl(sock, F_SETFL, mode) == -1: + raiseOSError(osLastError()) + result = sock + + var client_message = "SERVER HELLO =>" + var server_message = "CLIENT HELLO" + var buffer : array[128, char] + + var selector = newSelector[int]() + var client_socket = create_test_socket() + var server_socket = create_test_socket() + + registerHandle(selector, server_socket, {Event.Read}, 0) + registerHandle(selector, client_socket, {Event.Write}, 0) + + var option : int32 = 1 + if setsockopt(server_socket, cint(SOL_SOCKET), cint(SO_REUSEADDR), + addr(option), sizeof(option).SockLen) < 0: + raiseOSError(osLastError()) + + var aiList = getAddrInfo("0.0.0.0", Port(13337)) + if bindAddr(server_socket, aiList.ai_addr, + aiList.ai_addrlen.Socklen) < 0'i32: + dealloc(aiList) + raiseOSError(osLastError()) + discard server_socket.listen() + dealloc(aiList) + + aiList = getAddrInfo("127.0.0.1", Port(13337)) + discard posix.connect(client_socket, aiList.ai_addr, + aiList.ai_addrlen.Socklen) + dealloc(aiList) + discard selector.select(100) + var rc1 = selector.select(100) + assert(len(rc1) == 2) + + var sockAddress: SockAddr + var addrLen = sizeof(sockAddress).Socklen + var server2_socket = accept(server_socket, + cast[ptr SockAddr](addr(sockAddress)), + addr(addrLen)) + assert(server2_socket != osInvalidSocket) + selector.registerHandle(server2_socket, {Event.Read}, 0) + + if posix.send(client_socket, addr(client_message[0]), + len(client_message), 0) == -1: + raiseOSError(osLastError()) + + selector.updateHandle(client_socket, {Event.Read}) + + var rc2 = selector.select(100) + assert(len(rc2) == 1) + + var read_count = posix.recv(server2_socket, addr buffer[0], 128, 0) + if read_count == -1: + raiseOSError(osLastError()) + + assert(read_count == len(client_message)) + var test1 = true + for i in 0..<read_count: + if client_message[i] != buffer[i]: + test1 = false + break + assert(test1) + + selector.updateHandle(server2_socket, {Event.Write}) + var rc3 = selector.select(0) + assert(len(rc3) == 1) + if posix.send(server2_socket, addr(server_message[0]), + len(server_message), 0) == -1: + raiseOSError(osLastError()) + selector.updateHandle(server2_socket, {Event.Read}) + + var rc4 = selector.select(100) + assert(len(rc4) == 1) + read_count = posix.recv(client_socket, addr(buffer[0]), 128, 0) + if read_count == -1: + raiseOSError(osLastError()) + + assert(read_count == len(server_message)) + var test2 = true + for i in 0..<read_count: + if server_message[i] != buffer[i]: + test2 = false + break + assert(test2) + + selector.unregister(server_socket) + selector.unregister(server2_socket) + selector.unregister(client_socket) + discard posix.close(server_socket) + discard posix.close(server2_socket) + discard posix.close(client_socket) + assert(selector.isEmpty()) + close(selector) + result = true + + proc event_notification_test(): bool = + var selector = newSelector[int]() + var event = newSelectEvent() + selector.registerEvent(event, 1) + selector.flush() + event.setEvent() + var rc1 = selector.select(0) + event.setEvent() + var rc2 = selector.select(0) + var rc3 = selector.select(0) + assert(len(rc1) == 1 and len(rc2) == 1 and len(rc3) == 0) + var ev1 = rc1[0].data + var ev2 = rc2[0].data + assert(ev1 == 1 and ev2 == 1) + selector.unregister(event) + event.close() + assert(selector.isEmpty()) + selector.close() + result = true + + when supportedPlatform: + proc timer_notification_test(): bool = + var selector = newSelector[int]() + var timer = selector.registerTimer(100, false, 0) + var rc1 = selector.select(140) + var rc2 = selector.select(140) + assert(len(rc1) == 1 and len(rc2) == 1) + selector.unregister(timer) + selector.flush() + selector.registerTimer(100, true, 0) + var rc3 = selector.select(120) + var rc4 = selector.select(120) + assert(len(rc3) == 1 and len(rc4) == 0) + assert(selector.isEmpty()) + selector.close() + result = true + + proc process_notification_test(): bool = + var selector = newSelector[int]() + var process2 = startProcess("/bin/sleep", "", ["2"], nil, + {poStdErrToStdOut, poUsePath}) + discard startProcess("/bin/sleep", "", ["1"], nil, + {poStdErrToStdOut, poUsePath}) + + selector.registerProcess(process2.processID, 0) + var rc1 = selector.select(1500) + var rc2 = selector.select(1500) + var r = len(rc1) + len(rc2) + assert(r == 1) + result = true + + proc signal_notification_test(): bool = + var sigset1n, sigset1o, sigset2n, sigset2o: Sigset + var pid = posix.getpid() + + discard sigemptyset(sigset1n) + discard sigemptyset(sigset1o) + discard sigemptyset(sigset2n) + discard sigemptyset(sigset2o) + + when hasThreadSupport: + if pthread_sigmask(SIG_BLOCK, sigset1n, sigset1o) == -1: + raiseOSError(osLastError()) + else: + if sigprocmask(SIG_BLOCK, sigset1n, sigset1o) == -1: + raiseOSError(osLastError()) + + var selector = newSelector[int]() + var s1 = selector.registerSignal(SIGUSR1, 1) + var s2 = selector.registerSignal(SIGUSR2, 2) + var s3 = selector.registerSignal(SIGTERM, 3) + selector.flush() + + discard posix.kill(pid, SIGUSR1) + discard posix.kill(pid, SIGUSR2) + discard posix.kill(pid, SIGTERM) + var rc = selector.select(0) + selector.unregister(s1) + selector.unregister(s2) + selector.unregister(s3) + + when hasThreadSupport: + if pthread_sigmask(SIG_BLOCK, sigset2n, sigset2o) == -1: + raiseOSError(osLastError()) + else: + if sigprocmask(SIG_BLOCK, sigset2n, sigset2o) == -1: + raiseOSError(osLastError()) + + assert(len(rc) == 3) + assert(rc[0].data + rc[1].data + rc[2].data == 6) # 1 + 2 + 3 + assert(equalMem(addr sigset1o, addr sigset2o, sizeof(Sigset))) + assert(selector.isEmpty()) + result = true + + when hasThreadSupport: + + var counter = 0 + + proc event_wait_thread(event: SelectEvent) {.thread.} = + var selector = newSelector[int]() + selector.registerEvent(event, 1) + selector.flush() + var rc = selector.select(1000) + if len(rc) == 1: + inc(counter) + selector.unregister(event) + assert(selector.isEmpty()) + + proc mt_event_test(): bool = + var + thr: array[0..7, Thread[SelectEvent]] + var selector = newSelector[int]() + var sock = newNativeSocket() + var event = newSelectEvent() + for i in 0..high(thr): + createThread(thr[i], event_wait_thread, event) + selector.registerHandle(sock, {Event.Read}, 1) + discard selector.select(500) + selector.unregister(sock) + event.setEvent() + joinThreads(thr) + assert(counter == 1) + result = true + + processTest("Socket notification test...", socket_notification_test()) + processTest("User event notification test...", event_notification_test()) + when hasThreadSupport: + processTest("Multithreaded user event notification test...", + mt_event_test()) + when supportedPlatform: + processTest("Timer notification test...", timer_notification_test()) + processTest("Process notification test...", process_notification_test()) + processTest("Signal notification test...", signal_notification_test()) + echo("All tests passed!") +else: + import nativesockets, winlean, os, osproc + + proc socket_notification_test(): bool = + proc create_test_socket(): SocketHandle = + var sock = newNativeSocket() + setBlocking(sock, false) + result = sock + + var client_message = "SERVER HELLO =>" + var server_message = "CLIENT HELLO" + var buffer : array[128, char] + + var selector = newSelector[int]() + var client_socket = create_test_socket() + var server_socket = create_test_socket() + + selector.registerHandle(server_socket, {Event.Read}, 0) + selector.registerHandle(client_socket, {Event.Write}, 0) + + var option : int32 = 1 + if setsockopt(server_socket, cint(SOL_SOCKET), cint(SO_REUSEADDR), + addr(option), sizeof(option).SockLen) < 0: + raiseOSError(osLastError()) + + var aiList = getAddrInfo("0.0.0.0", Port(13337)) + if bindAddr(server_socket, aiList.ai_addr, + aiList.ai_addrlen.Socklen) < 0'i32: + dealloc(aiList) + raiseOSError(osLastError()) + discard server_socket.listen() + dealloc(aiList) + + aiList = getAddrInfo("127.0.0.1", Port(13337)) + discard connect(client_socket, aiList.ai_addr, + aiList.ai_addrlen.Socklen) + dealloc(aiList) + # for some reason Windows select doesn't return both + # descriptors from first call, so we need to make 2 calls + discard selector.select(100) + var rcm = selector.select(100) + assert(len(rcm) == 2) + + var sockAddress = SockAddr() + var addrLen = sizeof(sockAddress).Socklen + var server2_socket = accept(server_socket, + cast[ptr SockAddr](addr(sockAddress)), + addr(addrLen)) + assert(server2_socket != osInvalidSocket) + selector.registerHandle(server2_socket, {Event.Read}, 0) + + if send(client_socket, cast[pointer](addr(client_message[0])), + cint(len(client_message)), 0) == -1: + raiseOSError(osLastError()) + + selector.updateHandle(client_socket, {Event.Read}) + + var rc2 = selector.select(100) + assert(len(rc2) == 1) + + var read_count = recv(server2_socket, addr buffer[0], 128, 0) + if read_count == -1: + raiseOSError(osLastError()) + + assert(read_count == len(client_message)) + var test1 = true + for i in 0..<read_count: + if client_message[i] != buffer[i]: + test1 = false + break + assert(test1) + + if send(server2_socket, cast[pointer](addr(server_message[0])), + cint(len(server_message)), 0) == -1: + raiseOSError(osLastError()) + + var rc3 = selector.select(0) + assert(len(rc3) == 1) + read_count = recv(client_socket, addr(buffer[0]), 128, 0) + if read_count == -1: + raiseOSError(osLastError()) + + assert(read_count == len(server_message)) + var test2 = true + for i in 0..<read_count: + if server_message[i] != buffer[i]: + test2 = false + break + assert(test2) + + selector.unregister(server_socket) + selector.unregister(server2_socket) + selector.unregister(client_socket) + close(server_socket) + close(server2_socket) + close(client_socket) + assert(selector.isEmpty()) + close(selector) + result = true + + proc event_notification_test(): bool = + var selector = newSelector[int]() + var event = newSelectEvent() + selector.registerEvent(event, 1) + selector.flush() + event.setEvent() + var rc1 = selector.select(0) + event.setEvent() + var rc2 = selector.select(0) + var rc3 = selector.select(0) + assert(len(rc1) == 1 and len(rc2) == 1 and len(rc3) == 0) + var ev1 = rc1[0].data + var ev2 = rc2[0].data + assert(ev1 == 1 and ev2 == 1) + selector.unregister(event) + event.close() + assert(selector.isEmpty()) + selector.close() + result = true + + when hasThreadSupport: + var counter = 0 + + proc event_wait_thread(event: SelectEvent) {.thread.} = + var selector = newSelector[int]() + selector.registerEvent(event, 1) + selector.flush() + var rc = selector.select(500) + if len(rc) == 1: + inc(counter) + selector.unregister(event) + assert(selector.isEmpty()) + + proc mt_event_test(): bool = + var thr: array[0..7, Thread[SelectEvent]] + var event = newSelectEvent() + for i in 0..high(thr): + createThread(thr[i], event_wait_thread, event) + event.setEvent() + joinThreads(thr) + assert(counter == 1) + result = true + + processTest("Socket notification test...", socket_notification_test()) + processTest("User event notification test...", event_notification_test()) + when hasThreadSupport: + processTest("Multithreaded user event notification test...", + mt_event_test()) + echo("All tests passed!") diff --git a/tests/async/tioselectors.nim.cfg b/tests/async/tioselectors.nim.cfg new file mode 100644 index 000000000..b1b896858 --- /dev/null +++ b/tests/async/tioselectors.nim.cfg @@ -0,0 +1 @@ +threads:on -d:threadsafe diff --git a/tests/async/tnewasyncudp.nim b/tests/async/tnewasyncudp.nim new file mode 100644 index 000000000..7025fa20d --- /dev/null +++ b/tests/async/tnewasyncudp.nim @@ -0,0 +1,102 @@ +discard """ + file: "tnewasyncudp.nim" + output: "5000" +""" +import asyncdispatch, nativesockets, net, strutils, os + +when defined(windows): + import winlean +else: + import posix + +var msgCount = 0 +var recvCount = 0 + +const + messagesToSend = 100 + swarmSize = 50 + serverPort = 10333 + +var + sendports = 0 + recvports = 0 + +proc saveSendingPort(port: int) = + sendports = sendports + port + +proc saveReceivedPort(port: int) = + recvports = recvports + port + +proc prepareAddress(intaddr: uint32, intport: uint16): ptr Sockaddr_in = + result = cast[ptr Sockaddr_in](alloc0(sizeof(Sockaddr_in))) + when defined(windows): + result.sin_family = toInt(nativesockets.AF_INET).int16 + else: + result.sin_family = toInt(nativesockets.AF_INET) + result.sin_port = htons(intport) + result.sin_addr.s_addr = htonl(intaddr) + +proc launchSwarm(name: ptr SockAddr) {.async.} = + var i = 0 + var k = 0 + while i < swarmSize: + var peeraddr = prepareAddress(INADDR_ANY, 0) + var sock = newAsyncNativeSocket(nativesockets.AF_INET, + nativesockets.SOCK_DGRAM, + Protocol.IPPROTO_UDP) + if bindAddr(sock.SocketHandle, cast[ptr SockAddr](peeraddr), + sizeof(Sockaddr_in).Socklen) < 0'i32: + raiseOSError(osLastError()) + let sockport = getSockName(sock.SocketHandle).int + k = 0 + while k < messagesToSend: + var message = "Message " & $(i * messagesToSend + k) + await sendTo(sock, addr message[0], len(message), + name, sizeof(Sockaddr_in).SockLen) + saveSendingPort(sockport) + inc(k) + closeSocket(sock) + inc(i) + +proc readMessages(server: AsyncFD) {.async.} = + var buffer: array[16384, char] + var slen = sizeof(Sockaddr_in).SockLen + var saddr = Sockaddr_in() + var maxResponses = (swarmSize * messagesToSend) + + var i = 0 + while i < maxResponses: + zeroMem(addr(buffer[0]), 16384) + zeroMem(cast[pointer](addr(saddr)), sizeof(Sockaddr_in)) + var size = await recvFromInto(server, cast[cstring](addr buffer[0]), + 16384, cast[ptr SockAddr](addr(saddr)), + addr(slen)) + size = 0 + var grammString = $buffer + if grammString.startswith("Message ") and + saddr.sin_addr.s_addr == 0x100007F: + inc(msgCount) + saveReceivedPort(ntohs(saddr.sin_port).int) + inc(recvCount) + inc(i) + +proc createServer() {.async.} = + var name = prepareAddress(INADDR_ANY, serverPort) + var server = newAsyncNativeSocket(nativesockets.AF_INET, + nativesockets.SOCK_DGRAM, + Protocol.IPPROTO_UDP) + if bindAddr(server.SocketHandle, cast[ptr SockAddr](name), + sizeof(Sockaddr_in).Socklen) < 0'i32: + raiseOSError(osLastError()) + asyncCheck readMessages(server) + +var name = prepareAddress(0x7F000001, serverPort) # 127.0.0.1 +asyncCheck createServer() +asyncCheck launchSwarm(cast[ptr SockAddr](name)) +while true: + poll() + if recvCount == swarmSize * messagesToSend: + break +assert msgCount == swarmSize * messagesToSend +assert sendports == recvports +echo msgCount diff --git a/tests/async/treturn_await.nim b/tests/async/treturn_await.nim new file mode 100644 index 000000000..8d266d665 --- /dev/null +++ b/tests/async/treturn_await.nim @@ -0,0 +1,23 @@ + +# bug #4371 + +import strutils, asyncdispatch, asynchttpserver + +type + List[A] = ref object + value: A + next: List[A] + StrPair* = tuple[k, v: string] + Context* = object + position*: int + accept*: bool + headers*: List[StrPair] + Handler* = proc(req: ref Request, ctx: Context): Future[Context] + +proc logging*(handler: Handler): auto = + proc h(req: ref Request, ctx: Context): Future[Context] {.async.} = + let ret = handler(req, ctx) + debugEcho "$3 $1 $2".format(req.reqMethod, req.url.path, req.hostname) + return await ret + + return h diff --git a/tests/async/twinasyncrw.nim b/tests/async/twinasyncrw.nim new file mode 100644 index 000000000..17b7d1cf5 --- /dev/null +++ b/tests/async/twinasyncrw.nim @@ -0,0 +1,257 @@ +discard """ + file: "twinasyncrw.nim" + output: "5000" +""" +when defined(windows): + import asyncdispatch, nativesockets, net, strutils, os, winlean + + var msgCount = 0 + + const + swarmSize = 50 + messagesToSend = 100 + + var clientCount = 0 + + proc winConnect*(socket: AsyncFD, address: string, port: Port, + domain = Domain.AF_INET): Future[void] = + var retFuture = newFuture[void]("winConnect") + proc cb(fd: AsyncFD): bool = + var ret = SocketHandle(fd).getSockOptInt(cint(SOL_SOCKET), cint(SO_ERROR)) + if ret == 0: + # We have connected. + retFuture.complete() + return true + else: + retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + return true + + var aiList = getAddrInfo(address, port, domain) + var success = false + var lastError: OSErrorCode = OSErrorCode(0) + var it = aiList + while it != nil: + var ret = nativesockets.connect(socket.SocketHandle, it.ai_addr, it.ai_addrlen.Socklen) + if ret == 0: + # Request to connect completed immediately. + success = true + retFuture.complete() + break + else: + lastError = osLastError() + if lastError.int32 == WSAEWOULDBLOCK: + success = true + addWrite(socket, cb) + break + else: + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + return retFuture + + proc winRecv*(socket: AsyncFD, size: int, + flags = {SocketFlag.SafeDisconn}): Future[string] = + var retFuture = newFuture[string]("recv") + + var readBuffer = newString(size) + + proc cb(sock: AsyncFD): bool = + result = true + let res = recv(sock.SocketHandle, addr readBuffer[0], size.cint, + flags.toOSFlags()) + if res < 0: + let lastError = osLastError() + if flags.isDisconnectionError(lastError): + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + elif res == 0: + # Disconnected + retFuture.complete("") + else: + readBuffer.setLen(res) + retFuture.complete(readBuffer) + # TODO: The following causes a massive slowdown. + #if not cb(socket): + addRead(socket, cb) + return retFuture + + proc winRecvInto*(socket: AsyncFD, buf: cstring, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = + var retFuture = newFuture[int]("winRecvInto") + + proc cb(sock: AsyncFD): bool = + result = true + let res = nativesockets.recv(sock.SocketHandle, buf, size.cint, + flags.toOSFlags()) + if res < 0: + let lastError = osLastError() + if flags.isDisconnectionError(lastError): + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + retFuture.complete(res) + # TODO: The following causes a massive slowdown. + #if not cb(socket): + addRead(socket, cb) + return retFuture + + proc winSend*(socket: AsyncFD, data: string, + flags = {SocketFlag.SafeDisconn}): Future[void] = + var retFuture = newFuture[void]("winSend") + + var written = 0 + + proc cb(sock: AsyncFD): bool = + result = true + let netSize = data.len-written + var d = data.cstring + let res = nativesockets.send(sock.SocketHandle, addr d[written], netSize.cint, 0) + if res < 0: + let lastError = osLastError() + if flags.isDisconnectionError(lastError): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + written.inc(res) + if res != netSize: + result = false # We still have data to send. + else: + retFuture.complete() + # TODO: The following causes crashes. + #if not cb(socket): + addWrite(socket, cb) + return retFuture + + proc winAcceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): + Future[tuple[address: string, client: AsyncFD]] = + var retFuture = newFuture[tuple[address: string, + client: AsyncFD]]("winAcceptAddr") + proc cb(sock: AsyncFD): bool = + result = true + if not retFuture.finished: + var sockAddress = Sockaddr() + var addrLen = sizeof(sockAddress).Socklen + var client = nativesockets.accept(sock.SocketHandle, + cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + if client == osInvalidSocket: + retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + else: + retFuture.complete((getAddrString(cast[ptr SockAddr](addr sockAddress)), client.AsyncFD)) + + addRead(socket, cb) + return retFuture + + proc winAccept*(socket: AsyncFD, + flags = {SocketFlag.SafeDisconn}): Future[AsyncFD] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection. + ## The future will complete when the connection is successfully accepted. + var retFut = newFuture[AsyncFD]("winAccept") + var fut = winAcceptAddr(socket, flags) + fut.callback = + proc (future: Future[tuple[address: string, client: AsyncFD]]) = + assert future.finished + if future.failed: + retFut.fail(future.error) + else: + retFut.complete(future.read.client) + return retFut + + + proc winRecvLine*(socket: AsyncFD): Future[string] {.async.} = + ## Reads a line of data from ``socket``. Returned future will complete once + ## a full line is read or an error occurs. + ## + ## If a full line is read ``\r\L`` is not + ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## will be set to it. + ## + ## If the socket is disconnected, ``line`` will be set to ``""``. + ## + ## If the socket is disconnected in the middle of a line (before ``\r\L`` + ## is read) then line will be set to ``""``. + ## The partial line **will be lost**. + ## + ## **Warning**: This assumes that lines are delimited by ``\r\L``. + ## + ## **Note**: This procedure is mostly used for testing. You likely want to + ## use ``asyncnet.recvLine`` instead. + + template addNLIfEmpty(): stmt = + if result.len == 0: + result.add("\c\L") + + result = "" + var c = "" + while true: + c = await winRecv(socket, 1) + if c.len == 0: + return "" + if c == "\r": + c = await winRecv(socket, 1) + assert c == "\l" + addNLIfEmpty() + return + elif c == "\L": + addNLIfEmpty() + return + add(result, c) + + proc sendMessages(client: AsyncFD) {.async.} = + for i in 0 .. <messagesToSend: + await winSend(client, "Message " & $i & "\c\L") + + proc launchSwarm(port: Port) {.async.} = + for i in 0 .. <swarmSize: + var sock = newNativeSocket() + setBlocking(sock, false) + + await winConnect(AsyncFD(sock), "localhost", port) + await sendMessages(AsyncFD(sock)) + discard closeSocket(sock) + + proc readMessages(client: AsyncFD) {.async.} = + while true: + var line = await winRecvLine(client) + if line == "": + closeSocket(client) + clientCount.inc + break + else: + if line.startswith("Message "): + msgCount.inc + else: + doAssert false + + proc createServer(port: Port) {.async.} = + var server = newNativeSocket() + setBlocking(server, false) + block: + var name = Sockaddr_in() + name.sin_family = toInt(Domain.AF_INET).int16 + name.sin_port = htons(uint16(port)) + name.sin_addr.s_addr = htonl(INADDR_ANY) + if bindAddr(server, cast[ptr SockAddr](addr(name)), + sizeof(name).Socklen) < 0'i32: + raiseOSError(osLastError()) + + discard server.listen() + while true: + asyncCheck readMessages(await winAccept(AsyncFD(server))) + + asyncCheck createServer(Port(10335)) + asyncCheck launchSwarm(Port(10335)) + while true: + poll() + if clientCount == swarmSize: break + + assert msgCount == swarmSize * messagesToSend + echo msgCount +else: + echo(5000) diff --git a/tests/ccgbugs/tcapture_static.nim b/tests/ccgbugs/tcapture_static.nim new file mode 100644 index 000000000..2afb8de47 --- /dev/null +++ b/tests/ccgbugs/tcapture_static.nim @@ -0,0 +1,13 @@ +discard """ + output: '''hi''' +""" + +# bug #4551 + +proc foo() = + let arr = ["hi"] + for i, v in arr: + let bar = proc = + echo v + bar() +foo() diff --git a/tests/ccgbugs/tclosureeq.nim b/tests/ccgbugs/tclosureeq.nim new file mode 100644 index 000000000..0486a9559 --- /dev/null +++ b/tests/ccgbugs/tclosureeq.nim @@ -0,0 +1,19 @@ +discard """ + output: '''true +true''' +""" + +# bug #4186 +type + Predicate[T] = proc(item: T): bool + +proc a[T](): Predicate[T] = + return nil + +proc b[T](): Predicate[T] = + return a[T]() + +echo b[int]() == nil # ok + +let x = b[int]() +echo x == nil #won't compile diff --git a/tests/ccgbugs/tescaping_temps.nim b/tests/ccgbugs/tescaping_temps.nim new file mode 100644 index 000000000..ef078913b --- /dev/null +++ b/tests/ccgbugs/tescaping_temps.nim @@ -0,0 +1,20 @@ + +# bug #4505 + +proc f(t: tuple[]) = discard +f((block: ())) + +# bug #4230 +# If we make `test` function return nothing - the bug disappears +proc test(dothejob: proc()): int {.discardable.} = + dothejob() + +test proc() = + let f = 15 + if f > 10: + test proc() = discard + # If we remove elif branch of the condition - the bug disappears + elif f < 3: + test proc() = discard + else: + test proc() = discard diff --git a/tests/ccgbugs/tinefficient_const_table.nim b/tests/ccgbugs/tinefficient_const_table.nim new file mode 100644 index 000000000..149b8bcff --- /dev/null +++ b/tests/ccgbugs/tinefficient_const_table.nim @@ -0,0 +1,27 @@ +discard """ + output: '''a +long +list +of +words''' + cmd: r"nim c --hints:on $options -d:release $file" + ccodecheck: "! @'genericSeqAssign'" +""" + + +# bug #4354 +import tables +import sets +import strutils + +#const FRUITS = ["banana", "apple", "grapes"] +#let FRUITS = ["banana", "apple", "grapes"].toSet +const FRUITS = {"banana":0, "apple":0, "grapes":0}.toTable + +proc main() = + let L = "a long list of words".split() + for word in L: + if word notin FRUITS: + echo(word) + +main() diff --git a/tests/ccgbugs/tmissingvolatile.nim b/tests/ccgbugs/tmissingvolatile.nim index 4d25e5c22..d61778ed4 100644 --- a/tests/ccgbugs/tmissingvolatile.nim +++ b/tests/ccgbugs/tmissingvolatile.nim @@ -1,7 +1,7 @@ discard """ output: "1" cmd: r"nim c --hints:on $options -d:release $file" - ccodecheck: "'NI volatile state;'" + ccodecheck: "'NI volatile state0;'" """ # bug #1539 diff --git a/tests/ccgbugs/tuplecast.nim b/tests/ccgbugs/tuplecast.nim new file mode 100644 index 000000000..d60e8c490 --- /dev/null +++ b/tests/ccgbugs/tuplecast.nim @@ -0,0 +1,8 @@ + +# bug #4345 + +# only needs to compile +proc f(): tuple[a, b: uint8] = (1'u8, 2'u8) + +let a, b = f() +let c = cast[int](b) diff --git a/tests/ccgbugs/tweakopenarray.nim b/tests/ccgbugs/tweakopenarray.nim new file mode 100644 index 000000000..51d781331 --- /dev/null +++ b/tests/ccgbugs/tweakopenarray.nim @@ -0,0 +1,12 @@ +# bug #4089 + +type + Proc = proc(args: openArray[Bar]): Bar + + Foo = object + p: Proc + + Bar = object + f: Foo + +proc bar(val: Foo): Bar = Bar() diff --git a/tests/ccgbugs/twrong_rc_for_refarray.nim b/tests/ccgbugs/twrong_rc_for_refarray.nim new file mode 100644 index 000000000..99bdac5e1 --- /dev/null +++ b/tests/ccgbugs/twrong_rc_for_refarray.nim @@ -0,0 +1,26 @@ +discard """ + output: '''m[0][0] = 1.0 +m[0][0] = 2.0''' +""" +# bug #4653 +type + Vector = ref array[2, float64] + Matrix = ref array[2, Vector] + +proc newVector(): Vector = + new(result) + +proc newMatrix(): Matrix = + new(result) + for ix in 0 .. 1: + result[ix] = newVector() + +let m = newMatrix() + +m[0][0] = 1.0 +echo "m[0][0] = ", m[0][0] + +GC_fullCollect() + +m[0][0] = 2.0 +echo "m[0][0] = ", m[0][0] diff --git a/tests/ccgbugs/twrong_string_asgn.nim b/tests/ccgbugs/twrong_string_asgn.nim index b62e70e7c..669b7f8f5 100644 --- a/tests/ccgbugs/twrong_string_asgn.nim +++ b/tests/ccgbugs/twrong_string_asgn.nim @@ -16,4 +16,4 @@ x.callback = proc () = finished = true -while not finished: discard +while not finished: poll() diff --git a/tests/closure/tdeeplynested.nim b/tests/closure/tdeeplynested.nim new file mode 100644 index 000000000..ddf4fa6a4 --- /dev/null +++ b/tests/closure/tdeeplynested.nim @@ -0,0 +1,20 @@ +discard """ + output: '''int: 108''' +""" + +# bug #4070 + +proc id(f: (proc())): auto = + return f + +proc foo(myinteger: int): (iterator(): int) = + return iterator(): int {.closure.} = + proc bar() = + proc kk() = + echo "int: ", myinteger + + kk() + + id(bar)() + +discard foo(108)() diff --git a/tests/closure/tflatmap.nim b/tests/closure/tflatmap.nim new file mode 100644 index 000000000..240756424 --- /dev/null +++ b/tests/closure/tflatmap.nim @@ -0,0 +1,24 @@ + +# bug #3995 + +import future + +type + RNG* = tuple[] + Rand*[A] = (RNG) -> (A, RNG) + +proc nextInt*(r: RNG): (int, RNG) = + (1, ()) + +proc flatMap[A,B](f: Rand[A], g: A -> Rand[B]): Rand[B] = + (rng: RNG) => ( + let (a, rng2) = f(rng); + let g1 = g(a); + g1(rng2) + ) + +proc map[A,B](s: Rand[A], f: A -> B): Rand[B] = + let g: A -> Rand[B] = (a: A) => ((rng: RNG) => (f(a), rng)) + flatMap(s, g) + +let f = nextInt.map(i => i - i mod 2) diff --git a/tests/closure/uclosures.nim b/tests/closure/uclosures.nim index 817bfec6b..f259cfeb9 100644 --- a/tests/closure/uclosures.nim +++ b/tests/closure/uclosures.nim @@ -1,12 +1,23 @@ +# This test is included from within tunittests import unittest -test "loop variables are captured by copy": +test "loop variables are captured by ref": var funcs: seq[proc (): int {.closure.}] = @[] for i in 0..10: let ii = i funcs.add do -> int: return ii * ii + check funcs[0]() == 100 + check funcs[3]() == 100 + +test "loop variables in closureScope are captured by copy": + var funcs: seq[proc (): int {.closure.}] = @[] + + for i in 0..10: + closureScope: + let ii = i + funcs.add do -> int: return ii * ii + check funcs[0]() == 0 check funcs[3]() == 9 - diff --git a/tests/converter/tconverter_with_varargs.nim b/tests/converter/tconverter_with_varargs.nim new file mode 100644 index 000000000..6d7e31e85 --- /dev/null +++ b/tests/converter/tconverter_with_varargs.nim @@ -0,0 +1,18 @@ + +# bug #888 + +type + PyRef = object + PPyRef* = ref PyRef + +converter to_py*(i: int) : PPyRef = nil + +when false: + proc to_tuple*(vals: openarray[PPyRef]): PPyRef = + discard + +proc abc(args: varargs[PPyRef]) = + #let args_tup = to_tuple(args) + discard + +abc(1, 2) diff --git a/tests/converter/texplicit_conversion.nim b/tests/converter/texplicit_conversion.nim new file mode 100644 index 000000000..6b2e96faf --- /dev/null +++ b/tests/converter/texplicit_conversion.nim @@ -0,0 +1,13 @@ +discard """ + output: "234" +""" + +# bug #4432 + +import strutils + +converter toInt(s: string): int = + result = parseInt(s) + +let x = (int)"234" +echo x diff --git a/tests/converter/tor_in_converter.nim b/tests/converter/tor_in_converter.nim new file mode 100644 index 000000000..5674526a1 --- /dev/null +++ b/tests/converter/tor_in_converter.nim @@ -0,0 +1,23 @@ +discard """ + output: '''test +test''' +""" +# bug #4537 + +# nim js --d:nodejs + +type + Str = distinct string + +when true: + # crashes + converter convert(s: string | cstring): Str = Str($s) +else: + # works! + converter convert(s: string): Str = Str($s) + converter convert(s: cstring): Str = Str($s) + +proc echoStr(s: Str) = echo s.string + +echoStr("test") +echoStr("test".cstring) diff --git a/tests/cpp/treturn_array.nim b/tests/cpp/treturn_array.nim new file mode 100644 index 000000000..ba4fbd6cc --- /dev/null +++ b/tests/cpp/treturn_array.nim @@ -0,0 +1,10 @@ + +# bug #2259 +type Mat4f* = array[0..15, float] + +proc get_rot_mat*(): Mat4f = discard +var mat: Mat4f = get_rot_mat() + +# bug #1389 +proc calcSizes(): array[2, int] = discard +let sizes: array[2, int] = calcSizes() diff --git a/tests/cpp/ttemplatetype.nim b/tests/cpp/ttemplatetype.nim new file mode 100644 index 000000000..7f56a225d --- /dev/null +++ b/tests/cpp/ttemplatetype.nim @@ -0,0 +1,9 @@ +type + Map {.importcpp: "std::map", header: "<map>".} [T,U] = object + +proc cInitMap(T: typedesc, U: typedesc): Map[T,U] {.importcpp: "std::map<'*1,'*2>()", nodecl.} + +proc initMap[T, U](): Map[T, U] = + result = cInitMap(T, U) + +var x: Map[cstring, cint] = initMap[cstring, cint]() diff --git a/tests/distinct/tnil.nim b/tests/distinct/tnil.nim new file mode 100644 index 000000000..ed0ac995a --- /dev/null +++ b/tests/distinct/tnil.nim @@ -0,0 +1,47 @@ +discard """ + file: "tnil.nim" + output: '''0x1 + +nil + +nil + +''' +""" + +type + MyPointer = distinct pointer + MyString = distinct string + MyStringNotNil = distinct (string not nil) + MyInt = distinct int + +proc foo(a: MyPointer) = + echo a.repr + +foo(cast[MyPointer](1)) +foo(cast[MyPointer](nil)) +foo(nil) + +var p: MyPointer +p = cast[MyPointer](1) +p = cast[MyPointer](nil) +p = nil.MyPointer +p = nil + +var c: MyString +c = "Test".MyString +c = nil.MyString +c = nil + +p = nil +doAssert(compiles(c = p) == false) + +var n: MyStringNotNil = "Test".MyStringNotNil # Cannot prove warning ... +n = "Test".MyStringNotNil +doAssert(compiles(n = nil.MyStringNotNil) == false) +doAssert(compiles(n = nil.MyStringNotNil) == false) +doAssert(compiles(n = nil) == false) + +var i: MyInt +i = 1.MyInt +doAssert(compiles(i = nil) == false) diff --git a/tests/enum/tenum.nim b/tests/enum/tenum.nim index b081212e6..6d9bdd539 100644 --- a/tests/enum/tenum.nim +++ b/tests/enum/tenum.nim @@ -6,3 +6,9 @@ type var en: E en = a + +# Bug #4066 +import macros +macro genEnum(): untyped = newNimNode(nnkEnumTy).add(newEmptyNode(), newIdentNode("geItem1")) +type GeneratedEnum = genEnum() +doAssert(type(geItem1) is GeneratedEnum) diff --git a/tests/float/tfloat4.nim b/tests/float/tfloat4.nim index 960c4e5f7..006b4d88f 100644 --- a/tests/float/tfloat4.nim +++ b/tests/float/tfloat4.nim @@ -1,3 +1,8 @@ +discard """ + file: "tfloat4.nim" + output: "passed all tests." + exitcode: 0 +""" import math, strutils proc c_sprintf(buf, fmt: cstring) {.importc:"sprintf", header: "<stdio.h>", varargs.} @@ -11,8 +16,9 @@ proc floatToStr(f: float64): string = return add(result, ch) + let testFloats = [ - "0", "-1", "1", "1.", ".3", "3.3", "-.3", "-99.99", + "0", "-0", "0.", "0.0", "-0.", "-0.0", "-1", "1", "1.", ".3", "3.3", "-.3", "-99.99", "1.1e10", "-2e100", "1.234e-10", "1.234e+10", "-inf", "inf", "+inf", "3.14159265358979323846264338327950288", @@ -25,18 +31,20 @@ let testFloats = [ ] for num in testFloats: - assert num.parseFloat.floatToStr.parseFloat == num.parseFloat + doAssert num.parseFloat.floatToStr.parseFloat == num.parseFloat -assert "0".parseFloat == 0.0 -assert "-.1".parseFloat == -0.1 -assert "2.5e1".parseFloat == 25.0 -assert "1e10".parseFloat == 10_000_000_000.0 -assert "0.000_005".parseFloat == 5.000_000e-6 -assert "1.234_567e+2".parseFloat == 123.4567 -assert "1e1_00".parseFloat == "1e100".parseFloat -assert "3.1415926535897932384626433".parseFloat == +doAssert "0".parseFloat == 0.0 +doAssert "-.1".parseFloat == -0.1 +doAssert "2.5e1".parseFloat == 25.0 +doAssert "1e10".parseFloat == 10_000_000_000.0 +doAssert "0.000_005".parseFloat == 5.000_000e-6 +doAssert "1.234_567e+2".parseFloat == 123.4567 +doAssert "1e1_00".parseFloat == "1e100".parseFloat +doAssert "3.1415926535897932384626433".parseFloat == 3.1415926535897932384626433 -assert "2.71828182845904523536028747".parseFloat == +doAssert "2.71828182845904523536028747".parseFloat == 2.71828182845904523536028747 -assert 0.00097656250000000021684043449710088680149056017398834228515625 == +doAssert 0.00097656250000000021684043449710088680149056017398834228515625 == "0.00097656250000000021684043449710088680149056017398834228515625".parseFloat + +echo("passed all tests.") diff --git a/tests/float/tfloat5.nim b/tests/float/tfloat5.nim new file mode 100644 index 000000000..aa7dc6c53 --- /dev/null +++ b/tests/float/tfloat5.nim @@ -0,0 +1,15 @@ +discard """ + file: "tfloat5.nim" + output: '''0 : 0.0 +0 : 0.0 +0 : 0.0 +0 : 0.0''' +""" + +import parseutils + +var f: float +echo "*".parseFloat(f), " : ", f +echo "/".parseFloat(f), " : ", f +echo "+".parseFloat(f), " : ", f +echo "-".parseFloat(f), " : ", f diff --git a/tests/float/tfloat6.nim b/tests/float/tfloat6.nim new file mode 100644 index 000000000..721abd721 --- /dev/null +++ b/tests/float/tfloat6.nim @@ -0,0 +1,21 @@ +discard """ + file: "tfloat6.nim" + output: '''1e-06 : 1e-06 +1e-06 : 1e-06 +0.001 : 0.001 +1e-06 : 1e-06 +1e-06 : 1e-06 +10.000001 : 10.000001 +100.000001 : 100.000001''' +""" + +import strutils + +echo "0.00_0001".parseFloat(), " : ", 1E-6 +echo "0.00__00_01".parseFloat(), " : ", 1E-6 +echo "0.0_01".parseFloat(), " : ", 0.001 +echo "0.00_000_1".parseFloat(), " : ", 1E-6 +echo "0.00000_1".parseFloat(), " : ", 1E-6 + +echo "1_0.00_0001".parseFloat(), " : ", 10.000001 +echo "1__00.00_0001".parseFloat(), " : ", 1_00.000001 diff --git a/tests/float/tfloat7.nim b/tests/float/tfloat7.nim new file mode 100644 index 000000000..2337d1dd4 --- /dev/null +++ b/tests/float/tfloat7.nim @@ -0,0 +1,26 @@ +discard """ + file: "tfloat6.nim" + output: '''passed. +passed. +passed. +passed. +passed. +passed. +passed.''' +""" + +import strutils +template expect_fail(x: expr) = + try: + discard x + echo("expected to fail!") + except ValueError: + echo("passed.") + +expect_fail("1_0._00_0001".parseFloat()) +expect_fail("_1_0_00.0001".parseFloat()) +expect_fail("10.00.01".parseFloat()) +expect_fail("10.00E_01".parseFloat()) +expect_fail("10.00E_01".parseFloat()) +expect_fail("10.00E".parseFloat()) +expect_fail("10.00A".parseFloat()) diff --git a/tests/gc/thavlak.nim b/tests/gc/thavlak.nim new file mode 100644 index 000000000..efab49e36 --- /dev/null +++ b/tests/gc/thavlak.nim @@ -0,0 +1,457 @@ +discard """ + output: '''Welcome to LoopTesterApp, Nim edition +Constructing Simple CFG... +15000 dummy loops +Constructing CFG... +Performing Loop Recognition +1 Iteration +Another 50 iterations... +.................................................. +Found 1 loops (including artificial root node) (50)''' +""" + +# bug #3184 + +import tables +import sequtils +import sets + +type + BasicBlock = object + inEdges: seq[ref BasicBlock] + outEdges: seq[ref BasicBlock] + name: int + +proc newBasicBlock(name: int): ref BasicBlock = + new(result) + result.inEdges = newSeq[ref BasicBlock]() + result.outEdges = newSeq[ref BasicBlock]() + result.name = name + +proc hash(x: ref BasicBlock): int {.inline.} = + result = x.name + +type + BasicBlockEdge = object + fr: ref BasicBlock + to: ref BasicBlock + + Cfg = object + basicBlockMap: Table[int, ref BasicBlock] + edgeList: seq[BasicBlockEdge] + startNode: ref BasicBlock + +proc newCfg(): Cfg = + result.basicBlockMap = initTable[int, ref BasicBlock]() + result.edgeList = newSeq[BasicBlockEdge]() + +proc createNode(self: var Cfg, name: int): ref BasicBlock = + result = self.basicBlockMap.getOrDefault(name) + if result == nil: + result = newBasicBlock(name) + self.basicBlockMap.add name, result + + if self.startNode == nil: + self.startNode = result + +proc addEdge(self: var Cfg, edge: BasicBlockEdge) = + self.edgeList.add(edge) + +proc getNumNodes(self: Cfg): int = + self.basicBlockMap.len + +proc newBasicBlockEdge(cfg: var Cfg, fromName: int, toName: int): BasicBlockEdge = + result.fr = cfg.createNode(fromName) + result.to = cfg.createNode(toName) + result.fr.outEdges.add(result.to) + result.to.inEdges.add(result.fr) + cfg.addEdge(result) + +type + SimpleLoop = object + basicBlocks: seq[ref BasicBlock] # TODO: set here + children: seq[ref SimpleLoop] # TODO: set here + parent: ref SimpleLoop + header: ref BasicBlock + isRoot: bool + isReducible: bool + counter: int + nestingLevel: int + depthLevel: int + +proc newSimpleLoop(): ref SimpleLoop = + new(result) + result.basicBlocks = newSeq[ref BasicBlock]() + result.children = newSeq[ref SimpleLoop]() + result.parent = nil + result.header = nil + result.isRoot = false + result.isReducible = true + result.counter = 0 + result.nestingLevel = 0 + result.depthLevel = 0 + +proc addNode(self: ref SimpleLoop, bb: ref BasicBlock) = + self.basicBlocks.add bb + +proc addChildLoop(self: ref SimpleLoop, loop: ref SimpleLoop) = + self.children.add loop + +proc setParent(self: ref SimpleLoop, parent: ref SimpleLoop) = + self.parent = parent + self.parent.addChildLoop(self) + +proc setHeader(self: ref SimpleLoop, bb: ref BasicBlock) = + self.basicBlocks.add(bb) + self.header = bb + +proc setNestingLevel(self: ref SimpleLoop, level: int) = + self.nestingLevel = level + if level == 0: self.isRoot = true + +var loop_counter: int = 0 + +type + Lsg = object + loops: seq[ref SimpleLoop] + root: ref SimpleLoop + +proc createNewLoop(self: var Lsg): ref SimpleLoop = + result = newSimpleLoop() + loop_counter += 1 + result.counter = loop_counter + +proc addLoop(self: var Lsg, l: ref SimpleLoop) = + self.loops.add l + +proc newLsg(): Lsg = + result.loops = newSeq[ref SimpleLoop]() + result.root = result.createNewLoop() + result.root.setNestingLevel(0) + result.addLoop(result.root) + +proc getNumLoops(self: Lsg): int = + self.loops.len + +type + UnionFindNode = object + parent: ref UnionFindNode + bb: ref BasicBlock + l: ref SimpleLoop + dfsNumber: int + +proc newUnionFindNode(): ref UnionFindNode = + new(result) + when false: + result.parent = nil + result.bb = nil + result.l = nil + result.dfsNumber = 0 + +proc initNode(self: ref UnionFindNode, bb: ref BasicBlock, dfsNumber: int) = + self.parent = self + self.bb = bb + self.dfsNumber = dfsNumber + +proc findSet(self: ref UnionFindNode): ref UnionFindNode = + var nodeList = newSeq[ref UnionFindNode]() + result = self + + while result != result.parent: + var parent = result.parent + if parent != parent.parent: nodeList.add result + result = parent + + for iter in nodeList: iter.parent = result.parent + +proc union(self: ref UnionFindNode, unionFindNode: ref UnionFindNode) = + self.parent = unionFindNode + + +const + BB_TOP = 0 # uninitialized + BB_NONHEADER = 1 # a regular BB + BB_REDUCIBLE = 2 # reducible loop + BB_SELF = 3 # single BB loop + BB_IRREDUCIBLE = 4 # irreducible loop + BB_DEAD = 5 # a dead BB + BB_LAST = 6 # Sentinel + + # # Marker for uninitialized nodes. + UNVISITED = -1 + + # # Safeguard against pathologic algorithm behavior. + MAXNONBACKPREDS = (32 * 1024) + +type + HavlakLoopFinder = object + cfg: Cfg + lsg: Lsg + +proc newHavlakLoopFinder(cfg: Cfg, lsg: Lsg): HavlakLoopFinder = + result.cfg = cfg + result.lsg = lsg + +proc isAncestor(w: int, v: int, last: seq[int]): bool = + w <= v and v <= last[w] + +proc dfs(currentNode: ref BasicBlock, nodes: var seq[ref UnionFindNode], number: var Table[ref BasicBlock, int], last: var seq[int], current: int): int = + var stack = @[(currentNode, current)] + while stack.len > 0: + let (currentNode, current) = stack.pop() + nodes[current].initNode(currentNode, current) + number[currentNode] = current + + result = current + for target in currentNode.outEdges: + if number[target] == UNVISITED: + stack.add((target, result+1)) + #result = dfs(target, nodes, number, last, result + 1) + last[number[currentNode]] = result + +proc findLoops(self: var HavlakLoopFinder): int = + var startNode = self.cfg.startNode + if startNode == nil: return 0 + var size = self.cfg.getNumNodes + + var nonBackPreds = newSeq[HashSet[int]]() + var backPreds = newSeq[seq[int]]() + var number = initTable[ref BasicBlock, int]() + var header = newSeq[int](size) + var types = newSeq[int](size) + var last = newSeq[int](size) + var nodes = newSeq[ref UnionFindNode]() + + for i in 1..size: + nonBackPreds.add initSet[int](1) + backPreds.add newSeq[int]() + nodes.add newUnionFindNode() + + # Step a: + # - initialize all nodes as unvisited. + # - depth-first traversal and numbering. + # - unreached BB's are marked as dead. + # + for v in self.cfg.basicBlockMap.values: number[v] = UNVISITED + var res = dfs(startNode, nodes, number, last, 0) + + # Step b: + # - iterate over all nodes. + # + # A backedge comes from a descendant in the DFS tree, and non-backedges + # from non-descendants (following Tarjan). + # + # - check incoming edges 'v' and add them to either + # - the list of backedges (backPreds) or + # - the list of non-backedges (nonBackPreds) + # + for w in 0 .. <size: + header[w] = 0 + types[w] = BB_NONHEADER + + var nodeW = nodes[w].bb + if nodeW != nil: + for nodeV in nodeW.inEdges: + var v = number[nodeV] + if v != UNVISITED: + if isAncestor(w, v, last): + backPreds[w].add v + else: + nonBackPreds[w].incl v + else: + types[w] = BB_DEAD + + # Start node is root of all other loops. + header[0] = 0 + + # Step c: + # + # The outer loop, unchanged from Tarjan. It does nothing except + # for those nodes which are the destinations of backedges. + # For a header node w, we chase backward from the sources of the + # backedges adding nodes to the set P, representing the body of + # the loop headed by w. + # + # By running through the nodes in reverse of the DFST preorder, + # we ensure that inner loop headers will be processed before the + # headers for surrounding loops. + + for w in countdown(size - 1, 0): + # this is 'P' in Havlak's paper + var nodePool = newSeq[ref UnionFindNode]() + + var nodeW = nodes[w].bb + if nodeW != nil: # dead BB + # Step d: + for v in backPreds[w]: + if v != w: + nodePool.add nodes[v].findSet + else: + types[w] = BB_SELF + + # Copy nodePool to workList. + # + var workList = newSeq[ref UnionFindNode]() + for x in nodePool: workList.add x + + if nodePool.len != 0: types[w] = BB_REDUCIBLE + + # work the list... + # + while workList.len > 0: + var x = workList[0] + workList.del(0) + + # Step e: + # + # Step e represents the main difference from Tarjan's method. + # Chasing upwards from the sources of a node w's backedges. If + # there is a node y' that is not a descendant of w, w is marked + # the header of an irreducible loop, there is another entry + # into this loop that avoids w. + # + + # The algorithm has degenerated. Break and + # return in this case. + # + var nonBackSize = nonBackPreds[x.dfsNumber].len + if nonBackSize > MAXNONBACKPREDS: return 0 + + for iter in nonBackPreds[x.dfsNumber]: + var y = nodes[iter] + var ydash = y.findSet + + if not isAncestor(w, ydash.dfsNumber, last): + types[w] = BB_IRREDUCIBLE + nonBackPreds[w].incl ydash.dfsNumber + else: + if ydash.dfsNumber != w and not nodePool.contains(ydash): + workList.add ydash + nodePool.add ydash + + # Collapse/Unionize nodes in a SCC to a single node + # For every SCC found, create a loop descriptor and link it in. + # + if (nodePool.len > 0) or (types[w] == BB_SELF): + var l = self.lsg.createNewLoop + + l.setHeader(nodeW) + l.isReducible = types[w] != BB_IRREDUCIBLE + + # At this point, one can set attributes to the loop, such as: + # + # the bottom node: + # iter = backPreds(w).begin(); + # loop bottom is: nodes(iter).node; + # + # the number of backedges: + # backPreds(w).size() + # + # whether this loop is reducible: + # types(w) != BB_IRREDUCIBLE + # + nodes[w].l = l + + for node in nodePool: + # Add nodes to loop descriptor. + header[node.dfsNumber] = w + node.union(nodes[w]) + + # Nested loops are not added, but linked together. + var node_l = node.l + if node_l != nil: + node_l.setParent(l) + else: + l.addNode(node.bb) + + self.lsg.addLoop(l) + + result = self.lsg.getNumLoops + + +type + LoopTesterApp = object + cfg: Cfg + lsg: Lsg + +proc newLoopTesterApp(): LoopTesterApp = + result.cfg = newCfg() + result.lsg = newLsg() + +proc buildDiamond(self: var LoopTesterApp, start: int): int = + var bb0 = start + var x1 = newBasicBlockEdge(self.cfg, bb0, bb0 + 1) + var x2 = newBasicBlockEdge(self.cfg, bb0, bb0 + 2) + var x3 = newBasicBlockEdge(self.cfg, bb0 + 1, bb0 + 3) + var x4 = newBasicBlockEdge(self.cfg, bb0 + 2, bb0 + 3) + result = bb0 + 3 + +proc buildConnect(self: var LoopTesterApp, start1: int, end1: int) = + var x1 = newBasicBlockEdge(self.cfg, start1, end1) + +proc buildStraight(self: var LoopTesterApp, start: int, n: int): int = + for i in 0..n-1: + self.buildConnect(start + i, start + i + 1) + result = start + n + +proc buildBaseLoop(self: var LoopTesterApp, from1: int): int = + var header = self.buildStraight(from1, 1) + var diamond1 = self.buildDiamond(header) + var d11 = self.buildStraight(diamond1, 1) + var diamond2 = self.buildDiamond(d11) + var footer = self.buildStraight(diamond2, 1) + + self.buildConnect(diamond2, d11) + self.buildConnect(diamond1, header) + self.buildConnect(footer, from1) + result = self.buildStraight(footer, 1) + +proc run(self: var LoopTesterApp) = + echo "Welcome to LoopTesterApp, Nim edition" + echo "Constructing Simple CFG..." + + var x1 = self.cfg.createNode(0) + var x2 = self.buildBaseLoop(0) + var x3 = self.cfg.createNode(1) + self.buildConnect(0, 2) + + echo "15000 dummy loops" + + for i in 1..15000: + var h = newHavlakLoopFinder(self.cfg, newLsg()) + var res = h.findLoops + + echo "Constructing CFG..." + var n = 2 + + for parlooptrees in 1..10: + var x6 = self.cfg.createNode(n + 1) + self.buildConnect(2, n + 1) + n += 1 + for i in 1..100: + var top = n + n = self.buildStraight(n, 1) + for j in 1..25: n = self.buildBaseLoop(n) + var bottom = self.buildStraight(n, 1) + self.buildConnect n, top + n = bottom + self.buildConnect(n, 1) + + echo "Performing Loop Recognition\n1 Iteration" + + var h = newHavlakLoopFinder(self.cfg, newLsg()) + var loops = h.findLoops + + echo "Another 50 iterations..." + + var sum = 0 + for i in 1..50: + write stdout, "." + flushFile(stdout) + var hlf = newHavlakLoopFinder(self.cfg, newLsg()) + sum += hlf.findLoops + #echo getOccupiedMem() + echo "\nFound ", loops, " loops (including artificial root node) (", sum, ")" + +var l = newLoopTesterApp() +l.run diff --git a/tests/gc/tlists.nim b/tests/gc/tlists.nim new file mode 100644 index 000000000..26b32396c --- /dev/null +++ b/tests/gc/tlists.nim @@ -0,0 +1,37 @@ +discard """ + output: '''Success''' +""" + +# bug #3793 + +import os +import math +import lists +import strutils + +proc mkleak() = + # allocate 10 MB via linked lists + let numberOfLists = 100 + for i in countUp(1, numberOfLists): + var leakList = initDoublyLinkedList[string]() + let numberOfLeaks = 50000 + for j in countUp(1, numberOfLeaks): + let leakSize = 200 + let leaked = newString(leakSize) + leakList.append(leaked) + +proc mkManyLeaks() = + for i in 0..0: + when false: echo getOccupiedMem() + mkleak() + when false: echo getOccupiedMem() + # Force a full collection. This should free all of the + # lists and bring the memory usage down to a few MB's. + GC_fullCollect() + when false: echo getOccupiedMem() + if getOccupiedMem() > 8 * 200 * 50_000 * 2: + echo GC_getStatistics() + quit "leaking" + echo "Success" + +mkManyLeaks() diff --git a/tests/generics/t88.nim b/tests/generics/t88.nim new file mode 100644 index 000000000..35bdeb8f1 --- /dev/null +++ b/tests/generics/t88.nim @@ -0,0 +1,33 @@ +# Issue 88 + +type + BaseClass[V] = object of RootObj + b: V + +proc new[V](t: typedesc[BaseClass], v: V): BaseClass[V] = + BaseClass[V](b: v) + +proc baseMethod[V](v: BaseClass[V]): V = v.b +proc overridedMethod[V](v: BaseClass[V]): V = v.baseMethod + +type + ChildClass[V] = object of BaseClass[V] + c: V + +proc new[V](t: typedesc[ChildClass], v1, v2: V): ChildClass[V] = + ChildClass[V](b: v1, c: v2) + +proc overridedMethod[V](v: ChildClass[V]): V = v.c + +let c = ChildClass[string].new("Base", "Child") + +assert c.baseMethod == "Base" +assert c.overridedMethod == "Child" + + +# bug #4528 +type GenericBase[T] = ref object of RootObj +type GenericSubclass[T] = ref object of GenericBase[T] +proc foo[T](g: GenericBase[T]) = discard +var bar: GenericSubclass[int] +foo(bar) diff --git a/tests/generics/tforward_generic.nim b/tests/generics/tforward_generic.nim new file mode 100644 index 000000000..169279cb3 --- /dev/null +++ b/tests/generics/tforward_generic.nim @@ -0,0 +1,28 @@ +discard """ + output: '''b() +720 120.0''' +""" + +# bug #3055 +proc b(t: int | string) +proc a(t: int) = b(t) +proc b(t: int | string) = echo "b()" +a(1) + +# test recursive generics still work: +proc fac[T](x: T): T = + if x == 0: return 1 + else: return fac(x-1)*x + +echo fac(6), " ", fac(5.0) + +when false: + # This still doesn't work... + # test recursive generic with forwarding: + proc fac2[T](x: T): T + + echo fac2(6), " ", fac2(5.0) + + proc fac2[T](x: T): T = + if x == 0: return 1 + else: return fac2(x-1)*x diff --git a/tests/generics/tgenerictmpl2.nim b/tests/generics/tgenerictmpl2.nim new file mode 100644 index 000000000..0ecaf9ded --- /dev/null +++ b/tests/generics/tgenerictmpl2.nim @@ -0,0 +1,31 @@ +discard """ + output: '''1 +1 +1 +1 +999 +999 +999 +2''' +""" + +# test if we can pass explicit generic arguments to generic templates +# based on bug report #3496 + +proc tproc[T](t: T = 999) = echo t +template ttmpl[T](t: T = 999) = echo t + +tproc(1) +tproc[int](1) +ttmpl(1) +ttmpl[int](1) #<- crash case #1 + +tproc[int]() +discard tproc[int] +ttmpl[int]() #<- crash case #2 +ttmpl[int] #<- crash case #3 + +# but still allow normal use of [] on non-generic templates + +template tarr: expr = [1, 2, 3, 4] +echo tarr[1] diff --git a/tests/generics/ttable_alias.nim b/tests/generics/ttable_alias.nim new file mode 100644 index 000000000..992ca85e0 --- /dev/null +++ b/tests/generics/ttable_alias.nim @@ -0,0 +1,7 @@ +# bug #4589 + +import tables +type SimpleTable*[TKey, TVal] = TableRef[TKey, TVal] +template newSimpleTable*(TKey, TVal: typedesc): SimpleTable[TKey, TVal] = newTable[TKey, TVal]() +var fontCache : SimpleTable[string, SimpleTable[int32, int]] +fontCache = newSimpleTable(string, SimpleTable[int32, int]) diff --git a/tests/generics/ttempl_in_generic.nim b/tests/generics/ttempl_in_generic.nim new file mode 100644 index 000000000..f04b9d216 --- /dev/null +++ b/tests/generics/ttempl_in_generic.nim @@ -0,0 +1,8 @@ + +# bug #4600 +template foo(x: untyped): untyped = echo 1 +template foo(x,y: untyped): untyped = echo 2 + +proc bar1[T](x: T) = foo(x) +proc bar2(x: float) = foo(x,x) +proc bar3[T](x: T) = foo(x,x) diff --git a/tests/generics/twrong_explicit_typeargs.nim b/tests/generics/twrong_explicit_typeargs.nim new file mode 100644 index 000000000..e47b38e99 --- /dev/null +++ b/tests/generics/twrong_explicit_typeargs.nim @@ -0,0 +1,16 @@ +discard """ + errormsg: "cannot instantiate: 'newImage[string]'" + line: 16 +""" + +# bug #4084 +type + Image[T] = object + data: seq[T] + +proc newImage[T: int32|int64](w, h: int): ref Image[T] = + new(result) + result.data = newSeq[T](w * h) + +var correct = newImage[int32](320, 200) +var wrong = newImage[string](320, 200) diff --git a/tests/iter/tcomplex_openarray.nim b/tests/iter/tcomplex_openarray.nim new file mode 100644 index 000000000..6fc191e90 --- /dev/null +++ b/tests/iter/tcomplex_openarray.nim @@ -0,0 +1,33 @@ + +# bug #3221 + +import algorithm, math, sequtils + + +iterator permutations[T](ys: openarray[T]): seq[T] = + var + d = 1 + c = newSeq[int](ys.len) + xs = newSeq[T](ys.len) + for i, y in ys: xs[i] = y + yield xs + block outer: + while true: + while d > 1: + dec d + c[d] = 0 + while c[d] >= d: + inc d + if d >= ys.len: break outer + let i = if (d and 1) == 1: c[d] else: 0 + swap xs[i], xs[d] + yield xs + inc c[d] + +proc dig_vectors(): void = + var v_nums: seq[int] + v_nums = newSeq[int](1) + for perm in permutations(toSeq(0 .. 1)): + v_nums[0] = 1 + +dig_vectors() diff --git a/tests/js/tclosures.nim b/tests/js/tclosures.nim new file mode 100644 index 000000000..0ec4f4743 --- /dev/null +++ b/tests/js/tclosures.nim @@ -0,0 +1,51 @@ +discard """ + action: run +""" + +import math, random, strutils +const consolePrefix = "jsCallbacks" + +asm """ + var callback = [] + function regCallback (fn) { callback.push (fn); } + function runCallbacks () { + var result = "\n" + var n = 0 + for (var fn in callback) { + n += 1 + result += "("+String (n)+")" + result += callback [fn] () + result += "\n" + } + return result + } + function print (text) { console.log (text); } +""" + +proc consoleprint (str:cstring): void {.importc: "print", noDecl.} +proc print* (a: varargs[string, `$`]) = consoleprint "$1: $2" % [consolePrefix, join (a, " ")] + +type CallbackProc {.importc.} = proc () : cstring + +proc regCallback (fn:CallbackProc) {.importc.} +proc runCallbacks ():cstring {.importc.} + +proc `*` (s:string, n:Natural) : string = s.repeat (n) + +proc outer (i:Natural) : (string, int) = + let c = $char (random (93) + 33) + let n = random (40) + let s = c * n + proc inner () : cstring = ("[$1]" % $n) & s & " <--" + regCallback (inner) + return (s, n) + +var expected = "\n" +for i in 1 .. 10: + let (s, n) = outer (i) + expected &= ("($1)[$2]" % [$i, $n]) & s & " <--" + expected &= "\n" + +let results = runCallbacks () + +doAssert(expected == results) diff --git a/tests/js/test2.nim b/tests/js/test2.nim index f6976d058..0f460d6f8 100644 --- a/tests/js/test2.nim +++ b/tests/js/test2.nim @@ -1,7 +1,8 @@ discard """ output: '''foo js 3.14 -7''' +7 +1''' """ # This file tests the JavaScript generator @@ -29,3 +30,13 @@ proc test(x: C, T: typedesc): T = cast[T](x) echo 7.test(int8) + +# #4222 +const someConst = [ "1"] + +proc procThatRefersToConst() # Forward decl +procThatRefersToConst() # Call bar before it is defined + +proc procThatRefersToConst() = + var i = 0 # Use a var index, otherwise nim will constfold foo[0] + echo someConst[i] # JS exception here: foo is still not initialized (undefined) diff --git a/tests/js/testtojsstr.nim b/tests/js/testtojsstr.nim new file mode 100644 index 000000000..03ac89e20 --- /dev/null +++ b/tests/js/testtojsstr.nim @@ -0,0 +1,8 @@ +discard """ + output = "И\n" +""" + +let s: string = "И\n" +let cs = s.cstring + +echo $s diff --git a/tests/js/tstring_assignment.nim b/tests/js/tstring_assignment.nim new file mode 100644 index 000000000..bdd93e6b5 --- /dev/null +++ b/tests/js/tstring_assignment.nim @@ -0,0 +1,11 @@ +discard """ + output: '''true''' +""" + +# bug #4471 +when true: + let s1 = "123" + var s2 = s1 + s2.setLen(0) + # fails - s1.len == 0 + echo s1.len == 3 diff --git a/tests/lexer/tstrlits.nim b/tests/lexer/tstrlits.nim index f5b7ce937..cc8872f60 100644 --- a/tests/lexer/tstrlits.nim +++ b/tests/lexer/tstrlits.nim @@ -1,6 +1,6 @@ discard """ file: "tstrlits.nim" - output: "a\"\"long string\"\"\"\"\"abc\"def" + output: "a\"\"long string\"\"\"\"\"abc\"def_'2'●" """ # Test the new different string literals @@ -11,9 +11,13 @@ const raw = r"abc""def" + escaped = "\x5f'\50'\u25cf" + + stdout.write(rawQuote) stdout.write(tripleEmpty) stdout.write(raw) +stdout.write(escaped) #OUT a""long string"""""abc"def diff --git a/tests/lookups/test.nim b/tests/lookups/test.nim new file mode 100644 index 000000000..a17d235a4 --- /dev/null +++ b/tests/lookups/test.nim @@ -0,0 +1,17 @@ +# This file needs to be called 'test' nim to provoke a clash +# with the unittest.test name. Issue # + +import unittest, macros + +# bug #4555 + +macro memo(n: untyped): typed = + result = n + +proc fastFib(n: int): int {.memo.} = 40 +proc fib(n: int): int = 40 + +suite "memoization": + test "recursive function memoization": + check fastFib(40) == fib(40) + diff --git a/tests/lookups/tprefer_proc.nim b/tests/lookups/tprefer_proc.nim new file mode 100644 index 000000000..57ee8e539 --- /dev/null +++ b/tests/lookups/tprefer_proc.nim @@ -0,0 +1,4 @@ + +# bug #4353 +import random +echo random[int](low(int) .. high(int)) diff --git a/tests/macros/tcomplexecho.nim b/tests/macros/tcomplexecho.nim new file mode 100644 index 000000000..f7f933c1c --- /dev/null +++ b/tests/macros/tcomplexecho.nim @@ -0,0 +1,42 @@ +discard """ + output: '''3 +OK +56 +123 +56 +61''' +""" + +import macros + +# Bug from the forum +macro addEcho1(s: untyped): stmt = + s.body.add(newCall("echo", newStrLitNode("OK"))) + result = s + +proc f1() {.addEcho1.} = + let i = 1+2 + echo i + +f1() + +# bug #537 +proc test(): seq[NimNode] {.compiletime.} = + result = @[] + result.add parseExpr("echo 56") + result.add parseExpr("echo 123") + result.add parseExpr("echo 56") + +proc foo(): seq[NimNode] {.compiletime.} = + result = @[] + result.add test() + result.add parseExpr("echo(5+56)") + +macro bar(): stmt = + result = newNimNode(nnkStmtList) + let x = foo() + for xx in x: + result.add xx + echo treeRepr(result) + +bar() diff --git a/tests/macros/tdumptree.nim b/tests/macros/tdumptree.nim index e5160b7ba..58b011b45 100644 --- a/tests/macros/tdumptree.nim +++ b/tests/macros/tdumptree.nim @@ -3,7 +3,7 @@ msg: '''StmtList VarSection IdentDefs Ident !"x" - nil + Empty Call DotExpr Ident !"foo" diff --git a/tests/macros/tgettypeinst.nim b/tests/macros/tgettypeinst.nim new file mode 100644 index 000000000..22e03a119 --- /dev/null +++ b/tests/macros/tgettypeinst.nim @@ -0,0 +1,122 @@ +discard """ +""" + +import macros, strUtils + +proc symToIdent(x: NimNode): NimNode = + case x.kind: + of nnkCharLit..nnkUInt64Lit: + result = newNimNode(x.kind) + result.intVal = x.intVal + of nnkFloatLit..nnkFloat64Lit: + result = newNimNode(x.kind) + result.floatVal = x.floatVal + of nnkStrLit..nnkTripleStrLit: + result = newNimNode(x.kind) + result.strVal = x.strVal + of nnkIdent, nnkSym: + result = newIdentNode($x) + else: + result = newNimNode(x.kind) + for c in x: + result.add symToIdent(c) + +macro testX(x,inst0: typed; recurse: static[bool]; implX: stmt): typed = + let inst = x.getTypeInst + let impl = x.getTypeImpl + let inst0r = inst0.symToIdent.treeRepr + let instr = inst.symToIdent.treeRepr + #echo inst0r + #echo instr + doAssert(instr == inst0r) + var impl0 = + if implX.kind == nnkNilLit: inst0 + else: implX[0][2] + let impl0r = impl0.symToIdent.treerepr + let implr = impl.symToIdent.treerepr + #echo impl0r + #echo implr + doAssert(implr == impl0r) + template echoString(s:string) = echo s.replace("\n","\n ") + result = newStmtList() + #result.add getAst(echoString(" " & inst0.repr)) + #result.add getAst(echoString(" " & inst.repr)) + #result.add getAst(echoString(" " & impl0.repr)) + #result.add getAst(echoString(" " & impl.repr)) + if recurse: + template testDecl(n, m :typed) = + testV(n, false): + type _ = m + result.add getAst(testDecl(inst.symToIdent, impl.symToIdent)) + +template testV(inst, recurse, impl) = + block: + #echo "testV(" & astToStr(inst) & ", " & $recurse & "):" & astToStr(impl) + var x: inst + testX(x, inst, recurse, impl) +template testT(inst, recurse) = + block: + type myType = inst + testV(myType, recurse): + type _ = inst + +template test(inst) = + testT(inst, false) + testV(inst, true, nil) +template test(inst, impl) = testV(inst, true, impl) + +type + Model = object of RootObj + User = object of Model + name : string + password : string + + Tree = object of RootObj + value : int + left,right : ref Tree + + MyEnum = enum + valueA, valueB, valueC + + MySet = set[MyEnum] + MySeq = seq[int] + MyIntPtr = ptr int + MyIntRef = ref int + + GenericObject[T] = object + value:T + Foo[N:static[int],T] = object + Bar[N:static[int],T] = object + #baz:Foo[N+1,GenericObject[T]] + baz:Foo[N,GenericObject[T]] + +test(bool) +test(char) +test(int) +test(float) +test(ptr int) +test(ref int) +test(array[1..10,Bar[2,Foo[3,float]]]) +test(distinct Bar[2,Foo[3,float]]) +test(tuple[a:int,b:Foo[-1,float]]) +#test(MyEnum): +# type _ = enum +# valueA, valueB, valueC +test(set[MyEnum]) +test(seq[int]) +test(Bar[2,Foo[3,float]]): + type _ = object + baz: Foo[2, GenericObject[Foo[3, float]]] +test(Model): + type _ = object of RootObj +test(User): + type _ = object of Model + name: string + password: string +test(Tree): + type _ = object of RootObj + value: int + left: ref Tree + right: ref Tree +test(proc (a: int, b: Foo[2,float])) +test(proc (a: int, b: Foo[2,float]): Bar[3,int]) diff --git a/tests/macros/tvarargsuntyped.nim b/tests/macros/tvarargsuntyped.nim new file mode 100644 index 000000000..b7d2bc001 --- /dev/null +++ b/tests/macros/tvarargsuntyped.nim @@ -0,0 +1,79 @@ +discard """ + output: '''Let's go! +(left: 2, r: 7, x: 8, height: 4, s: test, width: 3, y: 9, top: 1, g: 7, b: 8) +(left: 2, r: 7, x: 8, height: 4, s: text, width: 3, y: 9, top: 1, g: 7, b: 8) +(left: 2, r: 7, x: 8, height: 4, s: text, width: 3, y: 9, top: 4, g: 7, b: 8) +(left: 2, r: 7, x: 8, height: 4, s: test, width: 3, y: 9, top: 1, g: 7, b: 8)''' +""" + +import macros + +proc internalBar(top, left, width, height: cint, s: string, x, y: int, r,g,b: int) = + echo locals() + +# we need these dummy constructors due to the wrong implementation +# of 'varargs[untyped]' in the compiler: + +proc point(x, y: int): int = discard +proc color(r, g, b: int): int = discard +proc rect(a, b, c, d: int): int = discard + +template declareUnpackingMacro(nimname,extname) = + macro nimname(n: varargs[untyped]): untyped = + var s: string = astToStr(extname) & "(" + var first = true + echo repr n + for x in n.children: + var unpack = false + if x.kind in nnkCallKinds: + case $x[0] + of "point": + expectLen(x, 3) + unpack = true + of "rect": + expectLen(x, 5) + unpack = true + of "color": + expectLen(x, 4) + unpack = true + else: discard + if unpack: + for i in 1..<x.len: + if first: + first = false + else: + add(s, ", ") + add(s, repr(x[i])) + else: + if first: + first = false + else: + add(s, ", ") + add(s, repr(x)) + + add(s, ")") + echo s + result = parseStmt(s) + +declareUnpackingMacro(bar,internalBar) + +type MyInt = distinct int + +proc myInt(i: int): MyInt = cast[MyInt](i) + +converter toCInt(mi: MyInt): cint = cast[cint](mi) + +echo "Let's go!" + +bar(rect(1, 2, 3, 4), "test", point(8, 9), color(7,7,8)) + +bar(1,2,3,4,"text",8,9,7,7,8) + +bar(myInt(4),2,3,4,"text",8,9,7,7,8) + +let top: cint = 1 +let left: cint = 2 +let width: cint = 3 +let height: cint = 4 + +bar(rect(top, left, width, height), "test", point(8, 9), color(7,7,8)) diff --git a/tests/macros/typesafeprintf.nim b/tests/macros/typesafeprintf.nim new file mode 100644 index 000000000..2f4622f3e --- /dev/null +++ b/tests/macros/typesafeprintf.nim @@ -0,0 +1,48 @@ +discard """ + output: '''test 10''' +""" + +# bug #1152 + +import macros, typetraits +proc printfImpl(formatstr: cstring) {.importc: "printf", varargs.} + +iterator tokenize(format: string): char = + var i = 0 + while true: + case format[i] + of '%': + case format[i+1] + of '\0': break + else: yield format[i+1] + i.inc + of '\0': break + else: discard + i.inc + +macro printf(formatString: string{lit}, args: varargs[typed]): untyped = + var i = 0 + let err = getType(bindSym"ValueError") + for c in tokenize(formatString.strVal): + var expectedType = case c + of 'c': getType(bindSym"char") + of 'd', 'i', 'x', 'X': getType(bindSym"int") + of 'f', 'e', 'E', 'g', 'G': getType(bindSym"float") + of 's': getType(bindSym"string") + of 'p': getType(bindSym"pointer") + else: err + + var actualType = getType(args[i]) + inc i + + if sameType(expectedType, err): + error c & " is not a valid format character" + elif not sameType(expectedType, actualType): + error "type mismatch for argument " & $i & ". expected type: " & + $expectedType & ", actual type: " & $actualType + + # keep the original callsite, but use cprintf instead + result = callsite() + result[0] = bindSym"printfImpl" + +printf("test %d\n", 10) diff --git a/tests/manyloc/keineschweine/lib/vehicles.nim b/tests/manyloc/keineschweine/lib/vehicles.nim index 94ebf9f57..ddfb43b38 100644 --- a/tests/manyloc/keineschweine/lib/vehicles.nim +++ b/tests/manyloc/keineschweine/lib/vehicles.nim @@ -1,6 +1,6 @@ import sfml, chipmunk, - sg_assets, sfml_stuff, keineschweine + sg_assets, sfml_stuff, "../keineschweine" proc accel*(obj: PVehicle, dt: float) = diff --git a/tests/manyloc/nake/nakefile.nim b/tests/manyloc/nake/nakefile.nim index e91b86986..2055d7834 100644 --- a/tests/manyloc/nake/nakefile.nim +++ b/tests/manyloc/nake/nakefile.nim @@ -1,5 +1,5 @@ import nake -import httpclient, zip/zipfiles, times, math, sequtils +import httpclient, zip/zipfiles, times, random, sequtils nakeImports randomize() @@ -145,7 +145,7 @@ task "download", "download game assets": echo "Extracted the libs dir. Copy the ones you need to this dir." task "zip-lib", "zip up the libs dir": - var z: TZipArchive + var z: ZipArchive if not z.open("libs-" & getDateStr() & ".zip", fmReadWrite): quit "Could not open zip" for file in walkDirRec("libs", {pcFile, pcDir}): diff --git a/tests/manyloc/named_argument_bug/main.nim.cfg b/tests/manyloc/named_argument_bug/main.nim.cfg index 27cf8e688..7df7a0e97 100644 --- a/tests/manyloc/named_argument_bug/main.nim.cfg +++ b/tests/manyloc/named_argument_bug/main.nim.cfg @@ -1,2 +1,3 @@ # this file only exists to mark 'main.nim' as the main file +--path:"$projectpath" diff --git a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim index 22d36ef4d..7d787c07b 100644 --- a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim +++ b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim @@ -6,7 +6,7 @@ export opengl type - EGL* = object of E_Base + EGL* = object of Exception EGL_code* = object of EGL code*: EGL_err EGL_err {.pure.} = enum diff --git a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/shader.nim b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/shader.nim index 89bb76064..970885b3d 100644 --- a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/shader.nim +++ b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/shader.nim @@ -8,7 +8,7 @@ type TShaderType* {.pure.} = enum frag = GL_FRAGMENT_SHADER, vert = GL_VERTEX_SHADER - E_Shader* = object of E_Base + E_Shader* = object of Exception E_UnknownShaderType* = object of E_Shader converter pathToShaderType*(s: string): TShaderType = diff --git a/tests/manyloc/named_argument_bug/tri_engine/gfx/tex.nim b/tests/manyloc/named_argument_bug/tri_engine/gfx/tex.nim index e5043ae34..282a1ac99 100644 --- a/tests/manyloc/named_argument_bug/tri_engine/gfx/tex.nim +++ b/tests/manyloc/named_argument_bug/tri_engine/gfx/tex.nim @@ -25,7 +25,7 @@ proc whiteTex*(): TTex = setTexParams() var pixel = [255'u8, 255'u8, 255'u8, 255'u8] - ?glTexImage2D(GLtexture2D, 0, GL_RGBA, 1, 1, 0, GL_BGRA, cGLUnsignedByte, pixel[0].addr) + ?glTexImage2D(GLtexture2D, 0, GLint GL_RGBA, 1, 1, 0, GL_BGRA, cGLUnsignedByte, pixel[0].addr) ?glBindTexture(GLtexture2D, 0) result = gWhiteTex diff --git a/tests/manyloc/named_argument_bug/tri_engine/math/circle.nim b/tests/manyloc/named_argument_bug/tri_engine/math/circle.nim index 7e7517998..b95cfa379 100644 --- a/tests/manyloc/named_argument_bug/tri_engine/math/circle.nim +++ b/tests/manyloc/named_argument_bug/tri_engine/math/circle.nim @@ -1,6 +1,6 @@ import - tri_engine/config, - tri_engine/math/vec + ../config, + vec type TCircle* = tuple[p: TV2[TR], r: TR] diff --git a/tests/manyloc/named_argument_bug/tri_engine/math/vec.nim b/tests/manyloc/named_argument_bug/tri_engine/math/vec.nim index 3b57acb8e..926958fe4 100644 --- a/tests/manyloc/named_argument_bug/tri_engine/math/vec.nim +++ b/tests/manyloc/named_argument_bug/tri_engine/math/vec.nim @@ -1,6 +1,6 @@ import macros, - tri_engine/config + "../config" type TV2*[T:SomeNumber=TR] = array[0..1, T] diff --git a/tests/metatype/tautoproc.nim b/tests/metatype/tautoproc.nim index ef5377096..2c8f6a3f7 100644 --- a/tests/metatype/tautoproc.nim +++ b/tests/metatype/tautoproc.nim @@ -1,5 +1,5 @@ discard """ - output: "empty" + output: "void" """ # bug #898 diff --git a/tests/metatype/tmodulo.nim b/tests/metatype/tmodulo.nim new file mode 100644 index 000000000..08bcc7935 --- /dev/null +++ b/tests/metatype/tmodulo.nim @@ -0,0 +1,20 @@ +discard """ + output: '''1 mod 7''' +""" + +# bug #3706 + +type Modulo[M: static[int]] = distinct int + +proc modulo(a: int, M: static[int]): Modulo[M] = Modulo[M](a %% M) + +proc `+`[M: static[int]](a, b: Modulo[M]): Modulo[M] = (a.int + b.int).modulo(M) + +proc `$`*[M: static[int]](a: Modulo[M]): string = $(a.int) & " mod " & $(M) + +when isMainModule: + let + a = 3.modulo(7) + b = 5.modulo(7) + echo a + b + diff --git a/tests/metatype/tvoid_must_not_match.nim b/tests/metatype/tvoid_must_not_match.nim new file mode 100644 index 000000000..240c3f751 --- /dev/null +++ b/tests/metatype/tvoid_must_not_match.nim @@ -0,0 +1,21 @@ +discard """ + errormsg: "type mismatch: got (Future[system.int], void)" + line: 20 +""" + +type + Future[T] = object + value: T + +proc complete[T](x: T) = + echo "completed" + let y = x + + +proc complete*[T](future: var Future[T], val: T) = + future.value = val + +var a: Future[int] + +complete(a): + echo "yielding void" diff --git a/tests/method/tgeneric_methods.nim b/tests/method/tgeneric_methods.nim new file mode 100644 index 000000000..76a68fbb0 --- /dev/null +++ b/tests/method/tgeneric_methods.nim @@ -0,0 +1,24 @@ +discard """ + output: "wow2" +""" +type + First[T] = ref object of RootObj + value: T + + Second[T] = ref object of First[T] + value2: T + +method wow[T](y: int; x: First[T]) {.base.} = + echo "wow1" + +method wow[T](y: int; x: Second[T]) = + echo "wow2" + +var + x: Second[int] +new(x) + +proc takeFirst(x: First[int]) = + wow(2, x) + +takeFirst(x) diff --git a/tests/method/tmultim6.nim b/tests/method/tmultim6.nim index 97ed2845c..2c622b832 100644 --- a/tests/method/tmultim6.nim +++ b/tests/method/tmultim6.nim @@ -4,27 +4,27 @@ discard """ # Test multi methods type - TThing = object {.inheritable.} - TUnit[T] = object of TThing + Thing = object {.inheritable.} + Unit[T] = object of Thing x: T - TParticle = object of TThing + Particle = object of Thing a, b: int -method collide(a, b: TThing) {.base, inline.} = +method collide(a, b: Thing) {.base, inline.} = quit "to override!" -method collide[T](a: TThing, b: TUnit[T]) {.inline.} = +method collide[T](a: Thing, b: Unit[T]) {.inline.} = write stdout, "collide: thing, unit | " -method collide[T](a: TUnit[T], b: TThing) {.inline.} = +method collide[T](a: Unit[T], b: Thing) {.inline.} = write stdout, "collide: unit, thing | " -proc test(a, b: TThing) {.inline.} = +proc test(a, b: Thing) {.inline.} = collide(a, b) var - a: TThing - b, c: TUnit[string] -collide(b, TThing(c)) + a: Thing + b, c: Unit[string] +collide(b, Thing(c)) test(b, c) collide(a, b) diff --git a/tests/modules/texplicit_system_import.nim b/tests/modules/texplicit_system_import.nim new file mode 100644 index 000000000..bc4d018bf --- /dev/null +++ b/tests/modules/texplicit_system_import.nim @@ -0,0 +1,9 @@ +##. +import system except `+` +discard """ + errormsg: "undeclared identifier: '+'" + line: 9 +""" +# Testament requires that the initial """ occurs before the 40th byte +# in the file. No kidding... +echo 4+5 diff --git a/tests/newconfig/mymath.nim b/tests/newconfig/mymath.nim new file mode 100644 index 000000000..5668b448b --- /dev/null +++ b/tests/newconfig/mymath.nim @@ -0,0 +1,4 @@ + + +proc ln*(x: float): float = + return 0.5 diff --git a/tests/newconfig/tfoo.nim b/tests/newconfig/tfoo.nim index d593d4a75..2e10167b1 100644 --- a/tests/newconfig/tfoo.nim +++ b/tests/newconfig/tfoo.nim @@ -1,10 +1,12 @@ discard """ cmd: "nim default $file" - output: '''hello world!''' + output: '''hello world! 0.5''' msg: '''[NimScript] exec: gcc -v''' """ when not defined(definedefine): {.fatal: "wrong nim script configuration".} -echo "hello world!" +import math + +echo "hello world! ", ln 2.0 diff --git a/tests/newconfig/tfoo.nims b/tests/newconfig/tfoo.nims index 519a868d5..057c0ed92 100644 --- a/tests/newconfig/tfoo.nims +++ b/tests/newconfig/tfoo.nims @@ -8,6 +8,11 @@ import ospaths --forceBuild +warning("uninit", off) +hint("processing", off) +#--verbosity:2 +patchFile("stdlib", "math", "mymath") + task listDirs, "lists every subdirectory": for x in listDirs("."): echo "DIR ", x diff --git a/tests/objects/tobject3.nim b/tests/objects/tobject3.nim index 2d9c8d023..15dd8ea24 100644 --- a/tests/objects/tobject3.nim +++ b/tests/objects/tobject3.nim @@ -49,7 +49,7 @@ proc makeDesktop(): PDesktop = new(TDesktop) proc makeWindow(): PWindow = new(TWindow) -proc thisCausesError(a: var PView, b: PView) = +proc thisCausesError(a: PView, b: PView) = discard var dd = makeDesktop() diff --git a/tests/osproc/passenv.nim b/tests/osproc/passenv.nim new file mode 100644 index 000000000..815f7536f --- /dev/null +++ b/tests/osproc/passenv.nim @@ -0,0 +1,32 @@ +discard """ + file: "passenv.nim" + output: "123" + targets: "c c++ objc" +""" + +import osproc, os, strtabs + +# Checks that the environment is passed correctly in startProcess +# To do that launches a copy of itself with a new environment. + +if paramCount() == 0: + # Parent process + + let env = newStringTable() + env["A"] = "1" + env["B"] = "2" + env["C"] = "3" + + let p = startProcess( + getAppFilename(), + args = @["child"], + env = env, + options = {poStdErrToStdOut, poUsePath, poParentStreams} + ) + + discard p.waitForExit + +else: + # Child process + # should output "123" + echo getEnv("A") & getEnv("B") & getEnv("C") diff --git a/tests/overload/tissue4475.nim b/tests/overload/tissue4475.nim new file mode 100644 index 000000000..34618cac5 --- /dev/null +++ b/tests/overload/tissue4475.nim @@ -0,0 +1,6 @@ +# Bug: https://github.com/nim-lang/Nim/issues/4475 +# Fix: https://github.com/nim-lang/Nim/pull/4477 + +proc test(x: varargs[string], y: int) = discard + +test(y = 1) diff --git a/tests/overload/tstmtoverload.nim b/tests/overload/tstmtoverload.nim index f1944b637..75584bcab 100644 --- a/tests/overload/tstmtoverload.nim +++ b/tests/overload/tstmtoverload.nim @@ -10,7 +10,7 @@ template test(loopCount: int, extraI: int, testBody: stmt): stmt = template test(loopCount: int, extraF: float, testBody: stmt): stmt = block: - test(loopCount, round(extraF), testBody) + test(loopCount, round(extraF).int, testBody) template test(loopCount: int, testBody: stmt): stmt = block: diff --git a/tests/overload/tvart_varargs.nim b/tests/overload/tvart_varargs.nim new file mode 100644 index 000000000..c0c460c76 --- /dev/null +++ b/tests/overload/tvart_varargs.nim @@ -0,0 +1,18 @@ + +# bug #4545 +type SomeObject = object + a : int + +type AbstractObject = object + objet: ptr SomeObject + +proc convert(this: var SomeObject): AbstractObject = + AbstractObject(objet: this.addr) + +proc varargProc(args: varargs[AbstractObject, convert]): int = + for arg in args: + result += arg.objet.a + +var obj = SomeObject(a: 17) + +discard varargProc(obj) diff --git a/tests/parallel/twrong_refcounts.nim b/tests/parallel/twrong_refcounts.nim index db32a96d8..57e0588a0 100644 --- a/tests/parallel/twrong_refcounts.nim +++ b/tests/parallel/twrong_refcounts.nim @@ -2,7 +2,7 @@ discard """ output: "Success" """ -import math, threadPool +import math, random, threadPool # --- diff --git a/tests/parser/tdo.nim b/tests/parser/tdo.nim new file mode 100644 index 000000000..7bd1f7411 --- /dev/null +++ b/tests/parser/tdo.nim @@ -0,0 +1,79 @@ +discard """ + output: '''true +true +true +true inner B''' +""" + +template withValue(a, b, c, d, e: untyped) = + if c: + d + else: + e + +template withValue(a, b, c, d: untyped) = + if c: + d + +const + EVENT_READ = 1 + EVENT_WRITE = 2 + FLAG_HANDLE = 3 + EVENT_MASK = 3 + +var s: string + +proc main = + var value = false + var fd = 8888 + var event = 0 + s.withValue(fd, value) do: + if value: + var oe = (EVENT_MASK) + if (oe xor event) != 0: + if (oe and EVENT_READ) != 0 and (event and EVENT_READ) == 0: + discard + if (oe and EVENT_WRITE) != 0 and (event and EVENT_WRITE) == 0: + discard + if (oe and EVENT_READ) == 0 and (event and EVENT_READ) != 0: + discard + if (oe and EVENT_WRITE) == 0 and (event and EVENT_WRITE) != 0: + discard + else: + raise newException(ValueError, "error") + do: + raise newException(ValueError, "Descriptor is not registered in queue") + +proc main2 = + var unused = 8 + # test 'then' branch: + s.withValue(unused, true) do: + echo "true" + do: + echo "false" + + # test overloading: + s.withValue(unused, false) do: + echo "cannot come here" + + # test 'else' branch: + s.withValue(unused, false) do: + echo "false" + do: + echo "true" + + # test proper nesting: + s.withValue(unused, false) do: + echo "false" + s.withValue(unused, false) do: + echo "false inner A" + do: + echo "true inner A" + do: + echo "true" + s.withValue(unused, false) do: + echo "false inner B" + do: + echo "true inner B" + +main2() diff --git a/tests/stdlib/nre/find.nim b/tests/stdlib/nre/find.nim index 05bfb848a..94fdd0bc1 100644 --- a/tests/stdlib/nre/find.nim +++ b/tests/stdlib/nre/find.nim @@ -1,4 +1,6 @@ -import unittest, sequtils, nre, optional_nonstrict +import unittest, sequtils +import nre except toSeq +import optional_nonstrict suite "find": test "find text": diff --git a/tests/stdlib/thtmlparser2814.nim b/tests/stdlib/thtmlparser2814.nim new file mode 100644 index 000000000..968d390f1 --- /dev/null +++ b/tests/stdlib/thtmlparser2814.nim @@ -0,0 +1,44 @@ +discard """ + output: true +""" +import htmlparser +import xmltree +import strutils +from streams import newStringStream + + +## builds the two cases below and test that +## ``//[dd,li]`` has "<p>that</p>" as children +## +## <dl> +## <dt>this</dt> +## <dd> +## <p>that</p> +## </dd> +## </dl> + +## +## <ul> +## <li> +## <p>that</p> +## </li> +## </ul> + + +for ltype in [["dl","dd"], ["ul","li"]]: + let desc_item = if ltype[0]=="dl": "<dt>this</dt>" else: "" + let item = "$1<$2><p>that</p></$2>" % [desc_item, ltype[1]] + let list = """ <$1> + $2 +</$1> """ % [ltype[0], item] + + var errors : seq[string] = @[] + + let parseH = parseHtml(newStringStream(list),"statichtml", errors =errors) + + if $parseH.findAll(ltype[1])[0].child("p") != "<p>that</p>": + echo "case " & ltype[0] & " failed !" + quit(2) + + +echo "true" diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim index 1ac9c8092..538582ba8 100644 --- a/tests/stdlib/tmath.nim +++ b/tests/stdlib/tmath.nim @@ -1,4 +1,4 @@ -import math +import math, random import unittest import sets diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index cae388792..1ddaacfcb 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -1,12 +1,88 @@ -# test some things of the os module +discard """ + output: '''true +true +true +true +true +true +true +true +true +All: +__really_obscure_dir_name/are.x +__really_obscure_dir_name/created +__really_obscure_dir_name/dirs +__really_obscure_dir_name/files.q +__really_obscure_dir_name/some +__really_obscure_dir_name/test +__really_obscure_dir_name/testing.r +__really_obscure_dir_name/these.txt +Files: +__really_obscure_dir_name/are.x +__really_obscure_dir_name/files.q +__really_obscure_dir_name/testing.r +__really_obscure_dir_name/these.txt +Dirs: +__really_obscure_dir_name/created +__really_obscure_dir_name/dirs +__really_obscure_dir_name/some +__really_obscure_dir_name/test +false +false +false +false +false +false +false +false +false +''' +""" +# test os path creation, iteration, and deletion import os -proc walkDirTree(root: string) = - for k, f in walkDir(root): - case k - of pcFile, pcLinkToFile: echo(f) - of pcDir: walkDirTree(f) - of pcLinkToDir: discard +let files = @["these.txt", "are.x", "testing.r", "files.q"] +let dirs = @["some", "created", "test", "dirs"] -walkDirTree(".") +let dname = "__really_obscure_dir_name" + +createDir(dname) +echo dirExists(dname) + +# Test creating files and dirs +for dir in dirs: + createDir(dname/dir) + echo dirExists(dname/dir) + +for file in files: + let fh = open(dname/file, fmReadWrite) + fh.close() + echo fileExists(dname/file) + +echo "All:" + +for path in walkPattern(dname/"*"): + echo path + +echo "Files:" + +for path in walkFiles(dname/"*"): + echo path + +echo "Dirs:" + +for path in walkDirs(dname/"*"): + echo path + +# Test removal of files dirs +for dir in dirs: + removeDir(dname/dir) + echo dirExists(dname/dir) + +for file in files: + removeFile(dname/file) + echo fileExists(dname/file) + +removeDir(dname) +echo dirExists(dname) diff --git a/tests/stdlib/tparscfg.nim b/tests/stdlib/tparscfg.nim index 4c11ccf61..7022d071b 100644 --- a/tests/stdlib/tparscfg.nim +++ b/tests/stdlib/tparscfg.nim @@ -1,25 +1,37 @@ +import parsecfg -import - os, parsecfg, strutils, streams +## Creating a configuration file. +var dict1=newConfig() +dict1.setSectionKey("","charset","utf-8") +dict1.setSectionKey("Package","name","hello") +dict1.setSectionKey("Package","--threads","on") +dict1.setSectionKey("Author","name","lihf8515") +dict1.setSectionKey("Author","qq","10214028") +dict1.setSectionKey("Author","email","lihaifeng@wxm.com") +dict1.writeConfig("config.ini") + +## Reading a configuration file. +var dict2 = loadConfig("config.ini") +var charset = dict2.getSectionValue("","charset") +var threads = dict2.getSectionValue("Package","--threads") +var pname = dict2.getSectionValue("Package","name") +var name = dict2.getSectionValue("Author","name") +var qq = dict2.getSectionValue("Author","qq") +var email = dict2.getSectionValue("Author","email") +echo charset +echo threads +echo pname +echo name +echo qq +echo email + +## Modifying a configuration file. +var dict3 = loadConfig("config.ini") +dict3.setSectionKey("Author","name","lhf") +dict3.writeConfig("config.ini") + +## Deleting a section key in a configuration file. +var dict4 = loadConfig("config.ini") +dict4.delSectionKey("Author","email") +dict4.writeConfig("config.ini") -var f = newFileStream(paramStr(1), fmRead) -if f != nil: - var p: TCfgParser - open(p, f, paramStr(1)) - while true: - var e = next(p) - case e.kind - of cfgEof: - echo("EOF!") - break - of cfgSectionStart: ## a ``[section]`` has been parsed - echo("new section: " & e.section) - of cfgKeyValuePair: - echo("key-value-pair: " & e.key & ": " & e.value) - of cfgOption: - echo("command: " & e.key & ": " & e.value) - of cfgError: - echo(e.msg) - close(p) -else: - echo("cannot open: " & paramStr(1)) diff --git a/tests/stdlib/tparseuints.nim b/tests/stdlib/tparseuints.nim new file mode 100644 index 000000000..5be3bcbd0 --- /dev/null +++ b/tests/stdlib/tparseuints.nim @@ -0,0 +1,11 @@ +discard """ + action: run +""" +import unittest, strutils + +suite "parseutils": + test "uint": + check: parseBiggestUInt("0") == 0'u64 + check: parseBiggestUInt("18446744073709551615") == 0xFFFF_FFFF_FFFF_FFFF'u64 + expect(ValueError): + discard parseBiggestUInt("18446744073709551616") diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim new file mode 100644 index 000000000..c702ccc2a --- /dev/null +++ b/tests/stdlib/trstgen.nim @@ -0,0 +1,139 @@ +# tests for rstgen module. + +import ../../lib/packages/docutils/rstgen +import unittest + +suite "YAML syntax highlighting": + test "Basics": + let input = """.. code-block:: yaml + %YAML 1.2 + --- + a string: string + a list: + - item 1 + - item 2 + a map: + ? key + : value + ...""" + let output = rstTohtml(input, {}, defaultConfig()) + assert output == """<pre class = "listing"><span class="Directive">%YAML 1.2</span> +<span class="Keyword">---</span> +<span class="StringLit">a string</span><span class="Punctuation">:</span> <span class="StringLit">string</span> +<span class="StringLit">a list</span><span class="Punctuation">:</span> + <span class="Punctuation">-</span> <span class="StringLit">item 1</span> + <span class="Punctuation">-</span> <span class="StringLit">item 2</span> +<span class="StringLit">a map</span><span class="Punctuation">:</span> +<span class="Punctuation">?</span> <span class="StringLit">key</span> +<span class="Punctuation">:</span> <span class="StringLit">value</span> +<span class="Keyword">...</span></pre>""" + + test "Block scalars": + let input = """.. code-block:: yaml + a literal block scalar: | + some text + # not a comment + # a comment, since less indented + # another comment + a folded block scalar: >2 + some text + # not a comment since indented as specified + # a comment + another literal block scalar: + |+ # comment after header + allowed, since more indented than parent""" + let output = rstToHtml(input, {}, defaultConfig()) + assert output == """<pre class = "listing"><span class="StringLit">a literal block scalar</span><span class="Punctuation">:</span> <span class="Command">|</span><span class="Command"></span><span class="LongStringLit"> + some text + # not a comment + </span><span class="Comment"># a comment, since less indented</span> + <span class="Comment"># another comment</span> +<span class="StringLit">a folded block scalar</span><span class="Punctuation">:</span> <span class="Command">>2</span><span class="Command"></span><span class="LongStringLit"> + some text + # not a comment since indented as specified + </span><span class="Comment"># a comment</span> +<span class="StringLit">another literal block scalar</span><span class="Punctuation">:</span> + <span class="Command">|+</span> <span class="Comment"># comment after header</span><span class="LongStringLit"> + allowed, since more indented than parent</span></pre>""" + + test "Directives": + let input = """.. code-block:: yaml + %YAML 1.2 + --- + %not a directive + ... + %a directive + ... + a string + % not a directive + ... + %TAG ! !foo:""" + let output = rstToHtml(input, {}, defaultConfig()) + assert output == """<pre class = "listing"><span class="Directive">%YAML 1.2</span> +<span class="Keyword">---</span> +<span class="StringLit">%not a directive</span> +<span class="Keyword">...</span> +<span class="Directive">%a directive</span> +<span class="Keyword">...</span> +<span class="StringLit">a string</span> +<span class="StringLit">% not a directive</span> +<span class="Keyword">...</span> +<span class="Directive">%TAG ! !foo:</span></pre>""" + + test "Flow Style and Numbers": + let input = """.. code-block:: yaml + { + "quoted string": 42, + 'single quoted string': false, + [ list, "with", 'entries' ]: 73.32e-73, + more numbers: [-783, 11e78], + not numbers: [ 42e, 0023, +32.37, 8 ball] + }""" + let output = rstToHtml(input, {}, defaultConfig()) + assert output == """<pre class = "listing"><span class="Punctuation">{</span> + <span class="StringLit">"</span><span class="StringLit">quoted string"</span><span class="Punctuation">:</span> <span class="DecNumber">42</span><span class="Punctuation">,</span> + <span class="StringLit">'single quoted string'</span><span class="Punctuation">:</span> <span class="StringLit">false</span><span class="Punctuation">,</span> + <span class="Punctuation">[</span> <span class="StringLit">list</span><span class="Punctuation">,</span> <span class="StringLit">"</span><span class="StringLit">with"</span><span class="Punctuation">,</span> <span class="StringLit">'entries'</span> <span class="Punctuation">]</span><span class="Punctuation">:</span> <span class="FloatNumber">73.32e-73</span><span class="Punctuation">,</span> + <span class="StringLit">more numbers</span><span class="Punctuation">:</span> <span class="Punctuation">[</span><span class="DecNumber">-783</span><span class="Punctuation">,</span> <span class="FloatNumber">11e78</span><span class="Punctuation">]</span><span class="Punctuation">,</span> + <span class="StringLit">not numbers</span><span class="Punctuation">:</span> <span class="Punctuation">[</span> <span class="StringLit">42e</span><span class="Punctuation">,</span> <span class="StringLit">0023</span><span class="Punctuation">,</span> <span class="StringLit">+32.37</span><span class="Punctuation">,</span> <span class="StringLit">8 ball</span><span class="Punctuation">]</span> +<span class="Punctuation">}</span></pre>""" + + test "Anchors, Aliases, Tags": + let input = """.. code-block:: yaml + --- !!map + !!str string: !<tag:yaml.org,2002:int> 42 + ? &anchor !!seq []: + : !localtag foo + alias: *anchor + """ + let output = rstToHtml(input, {}, defaultConfig()) + assert output == """<pre class = "listing"><span class="Keyword">---</span> <span class="TagStart">!!map</span> +<span class="TagStart">!!str</span> <span class="StringLit">string</span><span class="Punctuation">:</span> <span class="TagStart">!<tag:yaml.org,2002:int></span> <span class="DecNumber">42</span> +<span class="Punctuation">?</span> <span class="Label">&anchor</span> <span class="TagStart">!!seq</span> <span class="Punctuation">[</span><span class="Punctuation">]</span><span class="Punctuation">:</span> +<span class="Punctuation">:</span> <span class="TagStart">!localtag</span> <span class="StringLit">foo</span> +<span class="StringLit">alias</span><span class="Punctuation">:</span> <span class="Reference">*anchor</span></pre>""" + + test "Edge cases": + let input = """.. code-block:: yaml + ... + %a string: + a:string:not:a:map + ... + not a list: + -2 + -3 + -4 + example.com/not/a#comment: + ?not a map key + """ + let output = rstToHtml(input, {}, defaultConfig()) + assert output == """<pre class = "listing"><span class="Keyword">...</span> + <span class="StringLit">%a string</span><span class="Punctuation">:</span> + <span class="StringLit">a:string:not:a:map</span> +<span class="Keyword">...</span> +<span class="StringLit">not a list</span><span class="Punctuation">:</span> + <span class="DecNumber">-2</span> + <span class="DecNumber">-3</span> + <span class="DecNumber">-4</span> +<span class="StringLit">example.com/not/a#comment</span><span class="Punctuation">:</span> + <span class="StringLit">?not a map key</span></pre>""" \ No newline at end of file diff --git a/tests/stdlib/tsplit.nim b/tests/stdlib/tsplit.nim index 5a1cd2f5f..44da58aca 100644 --- a/tests/stdlib/tsplit.nim +++ b/tests/stdlib/tsplit.nim @@ -9,7 +9,7 @@ for w in split("|abc|xy|z", {'|'}): s.add("#") s.add(w) -if s == "#abc#xy#z": +if s == "##abc#xy#z": echo "true" else: echo "false" diff --git a/tests/stdlib/tunittest.nim b/tests/stdlib/tunittest.nim index 4b210c23b..e87cd3508 100644 --- a/tests/stdlib/tunittest.nim +++ b/tests/stdlib/tunittest.nim @@ -26,7 +26,7 @@ test "unittest multiple requires": require(true) -import math +import math, random from strutils import parseInt proc defectiveRobot() = randomize() @@ -83,3 +83,9 @@ suite "suite with both": test "unittest with both 2": check c == 2 + +suite "bug #4494": + test "Uniqueness check": + var tags = @[1, 2, 3, 4, 5] + check: + allIt(0..3, tags[it] != tags[it + 1]) diff --git a/tests/template/mlt.nim b/tests/template/mlt.nim new file mode 100644 index 000000000..e567cf085 --- /dev/null +++ b/tests/template/mlt.nim @@ -0,0 +1,3 @@ + +type Point* = ref object of RootObj +proc `>`*(p1, p2: Point): bool = false diff --git a/tests/template/t2do.nim b/tests/template/t2do.nim index b87e3328c..ec364c5f3 100644 --- a/tests/template/t2do.nim +++ b/tests/template/t2do.nim @@ -15,8 +15,9 @@ template toFloatHelper(result: expr; tooSmall, tooLarge: stmt) {.immediate.} = tooLarge proc toFloat*(a: int): float = - toFloatHelper(result) - do: raise newException(ValueError, "number too small"): - raise newException(ValueError, "number too large") + toFloatHelper(result) do: + raise newException(ValueError, "number too small") + do: + raise newException(ValueError, "number too large") echo toFloat(8) diff --git a/tests/template/tlt.nim b/tests/template/tlt.nim new file mode 100644 index 000000000..75c7dd991 --- /dev/null +++ b/tests/template/tlt.nim @@ -0,0 +1,7 @@ + +import mlt +# bug #4564 +type Bar* = ref object of RootObj +proc foo(a: Bar): int = 0 +var a: Bar +let b = a.foo() > 0 diff --git a/tests/template/typedescids.nim b/tests/template/typedescids.nim new file mode 100644 index 000000000..ebed49b17 --- /dev/null +++ b/tests/template/typedescids.nim @@ -0,0 +1,17 @@ +discard """ + output: '''2 3''' +""" + +# bug #4097 + +var i {.compileTime.} = 2 + +template defineId*(t: typedesc): stmt = + const id {.genSym.} = i + static: inc(i) + proc idFor*(T: typedesc[t]): int {.inline, raises: [].} = id + +defineId(int8) +defineId(int16) + +echo idFor(int8), " ", idFor(int16) diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims new file mode 100644 index 000000000..436e990ef --- /dev/null +++ b/tests/test_nimscript.nims @@ -0,0 +1,25 @@ +# This nimscript is used to test if the following modules can be imported +# http://nim-lang.org/docs/nims.html + +import algorithm +import base64 +import colors +import hashes +import lists +import math +# import marshal +import options +import ospaths +# import parsecfg +# import parseopt +import parseutils +# import pegs +import queues +import sequtils +import strutils +import subexes +import tables +import unicode +import uri + +echo "Nimscript imports are successful." diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index 150c55edc..e534cc161 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -152,6 +152,9 @@ proc gcTests(r: var TResults, cat: Category, options: string) = testWithoutBoehm "closureleak" testWithoutMs "refarrayleak" + testWithoutBoehm "tlists" + testWithoutBoehm "thavlak" + test "stackrefleak" test "cyclecollector" @@ -223,7 +226,7 @@ proc jsTests(r: var TResults, cat: Category, options: string) = "varres/tvartup", "misc/tints", "misc/tunsignedinc"]: test "tests/" & testfile & ".nim" - for testfile in ["pure/strutils", "pure/json"]: + for testfile in ["pure/strutils", "pure/json", "pure/random", "pure/times"]: test "lib/" & testfile & ".nim" # ------------------------- manyloc ------------------------------------------- @@ -257,12 +260,14 @@ proc compileExample(r: var TResults, pattern, options: string, cat: Category) = testNoSpec r, makeTest(test, options, cat) proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = + var disabledSet = disabledFiles.toSet() for test in os.walkFiles(pattern): - let contents = readFile(test).string - if contents.contains("when isMainModule"): - testSpec r, makeTest(test, options, cat, actionRunNoSpec) - else: - testNoSpec r, makeTest(test, options, cat, actionCompile) + if test notin disabledSet: + let contents = readFile(test).string + if contents.contains("when isMainModule"): + testSpec r, makeTest(test, options, cat, actionRunNoSpec) + else: + testNoSpec r, makeTest(test, options, cat, actionCompile) # ----------------------------- nimble ---------------------------------------- type PackageFilter = enum diff --git a/tests/testament/specs.nim b/tests/testament/specs.nim index bab17d2cd..b9519c70f 100644 --- a/tests/testament/specs.nim +++ b/tests/testament/specs.nim @@ -107,6 +107,15 @@ proc specDefaults*(result: var TSpec) = result.tline = 0 result.tcolumn = 0 +proc parseTargets*(value: string): set[TTarget] = + for v in value.normalize.split: + case v + of "c": result.incl(targetC) + of "cpp", "c++": result.incl(targetCpp) + of "objc": result.incl(targetObjC) + of "js": result.incl(targetJS) + else: echo "target ignored: " & v + proc parseSpec*(filename: string): TSpec = specDefaults(result) result.file = filename @@ -146,7 +155,11 @@ proc parseSpec*(filename: string): TSpec = result.nimout = e.value of "disabled": if parseCfgBool(e.value): result.err = reIgnored - of "cmd": result.cmd = e.value + of "cmd": + if e.value.startsWith("nim "): + result.cmd = "compiler" / e.value + else: + result.cmd = e.value of "ccodecheck": result.ccodeCheck = e.value of "target", "targets": for v in e.value.normalize.split: diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index b1e8ac099..83e59a6c1 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -12,7 +12,7 @@ import parseutils, strutils, pegs, os, osproc, streams, parsecfg, json, marshal, backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, compiler/nodejs, re, times + algorithm, compiler/nodejs, re, times, sets const resultsFile = "testresults.html" @@ -33,6 +33,7 @@ Options: --print also print results to the console --failing only show failing/ignored tests --pedantic return non-zero status code if there are failures + --targets:"c c++ js objc" run tests for specified targets (default: all) """ % resultsFile type @@ -60,6 +61,8 @@ let pegSuccess = peg"'Hint: operation successful'.*" pegOfInterest = pegLineError / pegOtherError +var targets = {low(TTarget)..high(TTarget)} + proc callCompiler(cmdTemplate, filename, options: string, target: TTarget): TSpec = let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], @@ -275,7 +278,13 @@ proc analyzeAndConsolidateOutput(s: string): string = proc testSpec(r: var TResults, test: TTest) = # major entry point for a single test + if test.target notin targets: + r.addResult(test, "", "", reIgnored) + inc(r.skipped) + return + let tname = test.name.addFileExt(".nim") + #echo "TESTING ", tname inc(r.total) var expected: TSpec if test.action != actionRunNoSpec: @@ -336,7 +345,7 @@ proc testSpec(r: var TResults, test: TTest) = reExitCodesDiffer) return - if bufB != expectedOut: + if bufB != expectedOut and expected.action != actionRunNoSpec: if not (expected.substr and expectedOut in bufB): given.err = reOutputsDiffer r.addResult(test, expected.outp, bufB, reOutputsDiffer) @@ -379,6 +388,10 @@ proc makeTest(test, options: string, cat: Category, action = actionCompile, result = TTest(cat: cat, name: test, options: options, target: target, action: action, startTime: epochTime()) +const + # array of modules disabled from compilation test of stdlib. + disabledFiles = ["-"] + include categories # proc runCaasTests(r: var TResults) = @@ -394,6 +407,7 @@ proc main() = var optPrintResults = false var optFailing = false var optPedantic = false + var p = initOptParser() p.next() while p.kind == cmdLongoption: @@ -401,6 +415,7 @@ proc main() = of "print", "verbose": optPrintResults = true of "failing": optFailing = true of "pedantic": optPedantic = true + of "targets": targets = parseTargets(p.val.string) else: quit Usage p.next() if p.kind != cmdArgument: quit Usage diff --git a/tests/threads/ttryrecv.nim b/tests/threads/ttryrecv.nim index be79fadae..4a98e6c27 100644 --- a/tests/threads/ttryrecv.nim +++ b/tests/threads/ttryrecv.nim @@ -4,7 +4,7 @@ discard """ # bug #1816 -from math import random +from random import random from os import sleep type PComm = ptr Channel[int] diff --git a/tests/trmacros/tpatterns.nim b/tests/trmacros/tpatterns.nim index 6bc8772e3..907973637 100644 --- a/tests/trmacros/tpatterns.nim +++ b/tests/trmacros/tpatterns.nim @@ -1,6 +1,7 @@ discard """ output: '''48 -hel''' +hel +lo''' """ template optZero{x+x}(x: int): int = x*3 @@ -15,3 +16,8 @@ s[0] = "hello" s[0] = substr(s[0], 0, 2) echo s[0] + +# Test varargs matching +proc someVarargProc(k: varargs[string]) = doAssert(false) # this should not get called +template someVarargProcSingleArg{someVarargProc([a])}(a: string) = echo a +someVarargProc("lo") diff --git a/tests/trmacros/tstatic_t_bug.nim b/tests/trmacros/tstatic_t_bug.nim new file mode 100644 index 000000000..cdfa53514 --- /dev/null +++ b/tests/trmacros/tstatic_t_bug.nim @@ -0,0 +1,24 @@ +discard """ + output: "optimized" +""" +# bug #4227 +type Vector64[N: static[int]] = array[N, int] + +proc `*`*[N: static[int]](a: Vector64[N]; b: float64): Vector64[N] = + result = a + +proc `+=`*[N: static[int]](a: var Vector64[N]; b: Vector64[N]) = + echo "regular" + +proc linearCombinationMut[N: static[int]](a: float64, v: var Vector64[N], w: Vector64[N]) {. inline .} = + echo "optimized" + +template rewriteLinearCombinationMut*{v += `*`(w, a)}(a: float64, v: var Vector64, w: Vector64): auto = + linearCombinationMut(a, v, w) + +proc main() = + const scaleVal = 9.0 + var a, b: Vector64[7] + a += b * scaleval + +main() diff --git a/tests/tuples/tconver_tuple.nim b/tests/tuples/tconver_tuple.nim new file mode 100644 index 000000000..306da77fe --- /dev/null +++ b/tests/tuples/tconver_tuple.nim @@ -0,0 +1,23 @@ +# Bug 4479 + +type + MyTuple = tuple + num: int + strings: seq[string] + ints: seq[int] + +var foo = MyTuple(( + num: 7, + strings: @[], + ints: @[], +)) + +var bar = ( + num: 7, + strings: @[], + ints: @[], +).MyTuple + +var fooUnnamed = MyTuple((7, @[], @[])) +var n = 7 +var fooSym = MyTuple((num: n, strings: @[], ints: @[])) diff --git a/tests/tuples/tuple_with_nil.nim b/tests/tuples/tuple_with_nil.nim index 442fbab92..9b5d583d3 100644 --- a/tests/tuples/tuple_with_nil.nim +++ b/tests/tuples/tuple_with_nil.nim @@ -4,7 +4,7 @@ import parseutils import unicode import math import fenv -import unsigned +#import unsigned import pegs import streams diff --git a/tests/tuples/twrong_generic_caching.nim b/tests/tuples/twrong_generic_caching.nim new file mode 100644 index 000000000..32ef344d2 --- /dev/null +++ b/tests/tuples/twrong_generic_caching.nim @@ -0,0 +1,4 @@ + +import parsecfg + +import asynchttpserver diff --git a/tests/typerel/tclosure_nil_as_default.nim b/tests/typerel/tclosure_nil_as_default.nim new file mode 100644 index 000000000..fe9f42b14 --- /dev/null +++ b/tests/typerel/tclosure_nil_as_default.nim @@ -0,0 +1,11 @@ + +# bug #4328 +type + foo[T] = object + z: T + +proc test[T](x: foo[T], p: proc(a: T) = nil) = + discard + +var d: foo[int] +d.test() # <- param omitted diff --git a/tests/typerel/temptynode.nim b/tests/typerel/temptynode.nim new file mode 100644 index 000000000..b32b16121 --- /dev/null +++ b/tests/typerel/temptynode.nim @@ -0,0 +1,16 @@ +discard """ + line: 16 + errormsg: "type mismatch: got (void)" +""" + +# bug #950 + +import macros + +proc blah(x: proc (a, b: int): int) = + echo x(5, 5) + +macro test(): stmt = + result = newNimNode(nnkEmpty) + +blah(test()) diff --git a/tests/typerel/tgeneric_subtype_regression.nim b/tests/typerel/tgeneric_subtype_regression.nim new file mode 100644 index 000000000..e279c0ad4 --- /dev/null +++ b/tests/typerel/tgeneric_subtype_regression.nim @@ -0,0 +1,19 @@ +discard """ + errormsg: "type mismatch: got (FooRef[system.string])" + line: 15 +""" + +# bug #4478 + +type + Foo[T] = object + FooRef[T] = ref Foo[T] + +proc takeFoo[T](foo: Foo[T]): int = discard + +proc g(x: FooRef[string]) = + echo x.takeFoo() != 8 + +var x: FooRef[string] + +g(x) diff --git a/tests/types/tassignemptytuple.nim b/tests/types/tassignemptytuple.nim new file mode 100644 index 000000000..bdfc653a5 --- /dev/null +++ b/tests/types/tassignemptytuple.nim @@ -0,0 +1,11 @@ +discard """ + file: "tassignemptytuple.nim" + line: 11 + errormsg: "cannot infer the type of the tuple" +""" + +var + foo: seq[int] + bar: tuple[a: seq[int], b: set[char]] + +(foo, bar) = (@[], (@[], {})) diff --git a/tests/types/typeof_produces_alias.nim b/tests/types/typeof_produces_alias.nim new file mode 100644 index 000000000..44cb00c94 --- /dev/null +++ b/tests/types/typeof_produces_alias.nim @@ -0,0 +1,25 @@ + +# bug #4124 + +import sequtils + +type + Foo = distinct string + +var + foo: Foo + +type + Alias = (type(foo)) +var + a: Alias + +a = foo + +when true: + var xs = @[1,2,3] + + proc asFoo(i: string): Foo = + Foo(i) + + var xx = xs.mapIt(asFoo($(it + 5))) diff --git a/tests/usingstmt/tthis.nim b/tests/usingstmt/tthis.nim new file mode 100644 index 000000000..83d75d08c --- /dev/null +++ b/tests/usingstmt/tthis.nim @@ -0,0 +1,15 @@ + +# bug #4177 + +type + Parent = object of RootObj + parentField: int + Child = object of Parent + childField: int + +{.this: self.} +proc sumFields(self: Child): int = + result = parentField + childField # Error: undeclared identifier: 'parentField' + +proc sumFieldsWorks(self: Child): int = + result = self.parentField + childField diff --git a/tests/vm/meta.nim b/tests/vm/meta.nim new file mode 100644 index 000000000..2aa01b5b3 --- /dev/null +++ b/tests/vm/meta.nim @@ -0,0 +1,240 @@ +# +# meta.nim +# + +import tables +import macros + +type + NodeSeq* = seq[NimNode] + Ident* = tuple[name: string, exported: bool] + Bracket* = seq[Ident] + Field* = tuple[identifier: Ident, type_name: string, default: string] + FieldSeq* = seq[Field] + TypeDef* = object + identifier*: Ident + fields*: FieldSeq + is_ref*: bool + object_type*: string + base_type*: string + TypeDefSeq* = seq[TypeDef] + Proc* = tuple[identifier: Ident, params: FieldSeq, + returns: Ident, generics: FieldSeq, body: NimNode] + ProcSeq* = seq[Proc] + +# Ident procs +proc newIdent*(name: string, exported = false): Ident = + result.name = name + result.exported = exported + +proc newIdent*(node: NimNode): Ident = + case node.kind: + of nnkPostfix: + result = newIdent(node[1]) + result.exported = true + of nnkIdent, nnkSym: + result.name = $(node) + else: + let msg = "newIdent cannot initialize from node kind: " & $(node.kind) + raise newException(ValueError, msg) + +proc render*(i: Ident): NimNode {.compileTime.} = + if i.name == nil: + return newNimNode(nnkEmpty) + + if i.exported: + result = newNimNode(nnkPostfix) + result.add(ident "*") + result.add(ident i.name) + else: + result = ident i.name + +proc `$`*(identifier: Ident): string = identifier.name + +converter toString*(x: Ident): string = x.name + +proc newBracket*(node: NimNode): Bracket = + result = @[] + case node.kind: + of nnkBracket: + for child in node: + if child.kind != nnkIdent: + let msg = "Bracket members can only be nnkIdent not kind: " & $(node.kind) + raise newException(ValueError, msg) + result.add(newIdent(child)) + else: + let msg = "newBracket must initialize from node kind nnkBracket not: " & $(node.kind) + raise newException(ValueError, msg) + +# Field procs +proc newField*(identifier: Ident, type_name: string, default: string = nil): Field = + result.identifier = identifier + result.type_name = type_name + result.default = default + +proc newField*(node: NimNode): Field = + case node.kind: + of nnkIdentDefs: + if node.len > 3: + let msg = "newField cannot initialize from nnkIdentDefs with multiple names" + raise newException(ValueError, msg) + result.identifier = newIdent(node[0]) + result.type_name = $(node[1]) + case node[2].kind: + of nnkIdent: + result.default = $(node[2]) + else: + result.default = nil + else: + let msg = "newField cannot initialize from node kind: " & $(node.kind) + raise newException(ValueError, msg) + +# FieldSeq procs +proc newFieldSeq*(node: NimNode): FieldSeq = + result = @[] + case node.kind: + of nnkIdentDefs: + let + type_name = $(node[node.len - 2]) + default_node = node[node.len - 1] + var default: string + case default_node.kind: + of nnkIdent: + default = $(default_node) + else: + default = nil + for i in 0..node.len - 3: + let name = newIdent(node[i]) + result.add(newField(name, type_name, default)) + of nnkRecList, nnkVarSection, nnkGenericParams: + for child in node: + result = result & newFieldSeq(child) + else: + let msg = "newFieldSeq cannot initialize from node kind: " & $(node.kind) + raise newException(ValueError, msg) + +proc render*(f: Field): NimNode {.compileTime.} = + let identifier = f.identifier.render() + let type_name = if f.type_name != nil: ident(f.type_name) else: newEmptyNode() + let default = if f.default != nil: ident(f.default) else: newEmptyNode() + newIdentDefs(identifier, type_name, default) + +proc render*(fs: FieldSeq): NimNode {.compileTime.} = + result = newNimNode(nnkRecList) + for field in fs: + result.add(field.render()) + +# TypeDef procs +proc newTypeDef*(identifier: Ident, is_ref = false, + object_type = "object", + base_type: string = nil): TypeDef {.compileTime.} = + result.identifier = identifier + result.fields = @[] + result.is_ref = is_ref + result.object_type = "object" + result.base_type = base_type + +proc newTypeDef*(node: NimNode): TypeDef {.compileTime.} = + case node.kind: + of nnkTypeDef: + result.identifier = newIdent($(node[0])) + var object_node: NimNode + case node[2].kind: + of nnkRefTy: + object_node = node[2][0] + result.is_ref = true + of nnkObjectTy: + object_node = node[2] + result.is_ref = false + else: + let msg = "newTypeDef could not parse RefTy/ObjectTy, found: " & $(node[2].kind) + raise newException(ValueError, msg) + case object_node[1].kind: + of nnkOfInherit: + result.base_type = $(object_node[1][0]) + else: + result.base_type = "object" + result.fields = newFieldSeq(object_node[2]) + else: + let msg = "newTypeDef cannot initialize from node kind: " & $(node.kind) + raise newException(ValueError, msg) + +proc render*(typedef: TypeDef): NimNode {.compileTime.} = + result = newNimNode(nnkTypeDef) + result.add(typedef.identifier.render) + result.add(newEmptyNode()) + let object_node = newNimNode(nnkObjectTy) + object_node.add(newEmptyNode()) + if typedef.base_type == nil: + object_node.add(newEmptyNode()) + else: + var base_type = newNimNode(nnkOfInherit) + base_type.add(ident(typedef.base_type)) + object_node.add(base_type) + let fields = typedef.fields.render() + object_node.add(fields) + if typedef.is_ref: + let ref_node = newNimNode(nnkRefTy) + ref_node.add(object_node) + result.add(ref_node) + else: + result.add(object_node) + +proc newTypeDefSeq*(node: NimNode): TypeDefSeq = + result = @[] + case node.kind: + of nnkTypeSection: + for child in node: + result.add(newTypeDef(child)) + else: + let msg = "newTypeSection could not parse TypeDef, found: " & $(node.kind) + raise newException(ValueError, msg) + +proc render*(typeseq: TypeDefSeq): NimNode {.compileTime.} = + result = newNimNode(nnkTypeSection) + for typedef in typeseq: + result.add(typedef.render()) + +proc newProc*(identifier: Ident, params: FieldSeq = nil, + returns: Ident, generics: FieldSeq = nil): Proc = + result.identifier = identifier + result.params = params + result.returns = returns + result.generics = generics + +proc newProc*(node: NimNode): Proc = + case node.kind: + of nnkProcDef, nnkMethodDef: + result.identifier = newIdent(node[0]) + case node[2].kind: + of nnkGenericParams: + result.generics = newFieldSeq(node[2]) + else: result.generics = nil + let formal_params = node[3] + case formal_params[0].kind: + of nnkIdent: + result.returns = newIdent(formal_params[0]) + else: discard + result.params = @[] + for i in 1..formal_params.len - 1: + let param = formal_params[i] + for field in newFieldSeq(param): + result.params.add(field) + result.body = node[6] + else: + let msg = "newProc cannot initialize from node kind: " & $(node.kind) + raise newException(ValueError, msg) + +proc render*(procdef: Proc): NimNode {.compileTime.} = + result = newNimNode(nnkProcDef) + result.add(procdef.identifier.render()) + result.add(newEmptyNode()) + result.add(newEmptyNode()) + let formal_params = newNimNode(nnkFormalParams) + formal_params.add(procdef.returns.render()) + for param in procdef.params: + formal_params.add(param.render()) + result.add(formal_params) + result.add(newEmptyNode()) + result.add(newEmptyNode()) + result.add(procdef.body) diff --git a/tests/vm/tanonproc.nim b/tests/vm/tanonproc.nim new file mode 100644 index 000000000..474b768ca --- /dev/null +++ b/tests/vm/tanonproc.nim @@ -0,0 +1,52 @@ +discard """ + output: '''`Test`''' +""" + +# bug #3561 + +import macros, future, strutils + +type + Option[T] = ref object + case valid: bool + of true: + data: T + else: + discard + +proc some[T](v: T): Option[T] = Option[T](valid: true, data: v) +proc none[T](v: T): Option[T] = Option[T](valid: false) +proc none(T: typedesc): Option[T] = Option[T](valid: false) + +proc map[T,U](o: Option[T], f: T -> U): Option[U] = + case o.valid + of true: + f(o.data).some + else: + U.none + +proc notEmpty(o: Option[string]): Option[string] = + case o.valid + of true: + if o.data.strip == "": string.none else: o.data.strip.some + else: + o + +proc getOrElse[T](o: Option[T], def: T): T = + case o.valid + of true: + o.data + else: + def + +proc quoteStr(s: string): Option[string] = + s.some.notEmpty.map(v => "`" & v & "`") + +macro str(s: string): stmt = + let x = s.strVal + let y = quoteStr(x) + let sn = newStrLitNode(y.getOrElse("NONE")) + result = quote do: + echo `sn` + +str"Test" diff --git a/tests/vm/tcomponent.nim b/tests/vm/tcomponent.nim new file mode 100644 index 000000000..efeba2a6d --- /dev/null +++ b/tests/vm/tcomponent.nim @@ -0,0 +1,132 @@ +discard """ + output: '''`:)` @ 0,0 +FOO: blah''' +""" + +# +# magic.nim +# + +# bug #3729 + +import macros, sequtils, tables +import strutils +import future, meta + +type + Component = object + fields: FieldSeq + field_index: seq[string] + procs: ProcSeq + procs_index: seq[string] + + Registry = object + field_index: seq[string] + procs_index: seq[string] + components: Table[string, Component] + builtin: Component + +proc newRegistry(): Registry = + result.field_index = @[] + result.procs_index = @[] + result.components = initTable[string, Component]() + +var registry {.compileTime.} = newRegistry() + +proc validateComponent(r: var Registry, name: string, c: Component) = + if r.components.hasKey(name): + let msg = "`component` macro cannot consume duplicated identifier: " & name + raise newException(ValueError, msg) + + for field_name in c.field_index: + if r.field_index.contains(field_name): + let msg = "`component` macro cannot delcare duplicated field: " & field_name + raise newException(ValueError, msg) + r.field_index.add(field_name) + + for proc_name in c.procs_index: + if r.procs_index.contains(proc_name): + let msg = "`component` macro cannot delcare duplicated proc: " & proc_name + raise newException(ValueError, msg) + r.procs_index.add(proc_name) + +proc addComponent(r: var Registry, name: string, c: Component) = + r.validateComponent(name, c) + r.components.add(name, c) + +proc parse_component(body: NimNode): Component = + result.field_index = @[] + result.procs_index = @[] + for node in body: + case node.kind: + of nnkVarSection: + result.fields = newFieldSeq(node) + for field in result.fields: + result.field_index.add(field.identifier.name) + of nnkMethodDef, nnkProcDef: + let new_proc = meta.newProc(node) + result.procs = result.procs & @[new_proc] + for procdef in result.procs: + result.procs_index.add(procdef.identifier.name) + else: discard + +macro component*(name: expr, body: stmt): stmt {.immediate.} = + let component = parse_component(body) + registry.addComponent($name, component) + parseStmt("discard") + +macro component_builtins(body: stmt): stmt {.immediate.} = + let builtin = parse_component(body) + registry.field_index = builtin.field_index + registry.procs_index = builtin.procs_index + registry.builtin = builtin + +proc bind_methods*(component: var Component, identifier: Ident): seq[NimNode] = + result = @[] + for procdef in component.procs.mitems: + let this_field = newField(newIdent("this"), identifier) + procdef.params.insert(this_field, 0) + result.add(procdef.render()) + +macro bind_components*(type_name, component_names: expr): stmt {.immediate.} = + result = newStmtList() + let identifier = newIdent(type_name) + let components = newBracket(component_names) + var entity_type = newTypeDef(identifier, true, "object", "RootObj") + entity_type.fields = registry.builtin.fields + for component_name, component in registry.components: + if components.contains(newIdent(component_name)): + entity_type.fields = entity_type.fields & component.fields + # TODO why doesn't the following snippet work instead of the one above? + # for name in components: + # echo "Registering $1 to $2" % [name.name, identifier.name] + # let component = registry.components[name.name] + # entity_type.fields = entity_type.fields & component.fields + let type_section: TypeDefSeq = @[entity_type] + result.add type_section.render + var builtin = registry.builtin + let builtin_methods = bind_methods(builtin, identifier) + for builtin_proc in builtin_methods: + result.add(builtin_proc) + echo "SIGSEV here" + for component in registry.components.mvalues(): + for method_proc in bind_methods(component, identifier): + result.add(method_proc) + +component_builtins: + proc foo(msg: string) = + echo "FOO: $1" % msg + +component position: + var x*, y*: int + +component name: + var name*: string + proc render*(x, y: int) = echo "`$1` @ $2,$3" % [this.name, $x, $y] + +bind_components(Entity, [position, name]) + +var e = new(Entity) +e.name = ":)" +e.render(e.x, e.y) +e.foo("blah") diff --git a/tests/vm/tconst_float_as_int.nim b/tests/vm/tconst_float_as_int.nim new file mode 100644 index 000000000..ed84ec194 --- /dev/null +++ b/tests/vm/tconst_float_as_int.nim @@ -0,0 +1,3 @@ + +# bug #4619 +const x: float = 0 diff --git a/tests/vm/tinheritance.nim b/tests/vm/tinheritance.nim new file mode 100644 index 000000000..d465e22b9 --- /dev/null +++ b/tests/vm/tinheritance.nim @@ -0,0 +1,29 @@ +discard """ + msg: '''Hello fred , managed by sally +Hello sally , managed by bob''' +""" +# bug #3973 + +type + EmployeeCode = enum + ecCode1, + ecCode2 + + Person* = object of RootObj + name* : string + last_name*: string + + Employee* = object of Person + empl_code* : EmployeeCode + mgr_name* : string + +proc test() = + var + empl1 = Employee(name: "fred", last_name: "smith", mgr_name: "sally", empl_code: ecCode1) + empl2 = Employee(name: "sally", last_name: "jones", mgr_name: "bob", empl_code: ecCode2) + + echo "Hello ", empl1.name, " , managed by ", empl1.mgr_name + echo "Hello ", empl2.name, " , managed by ", empl2.mgr_name + +static: + test() diff --git a/tests/vm/tmitems.nim b/tests/vm/tmitems.nim new file mode 100644 index 000000000..4ee225eed --- /dev/null +++ b/tests/vm/tmitems.nim @@ -0,0 +1,31 @@ +discard """ + msg: '''13''' + output: '''3 +3 +3''' +""" +# bug #3731 +var list {.compileTime.} = newSeq[int]() + +macro calc*(): stmt {.immediate.} = + list.add(1) + for c in list.mitems: + c = 13 + + for c in list: + echo c + +calc() + +# bug #3859 +import macros +macro m: stmt = + var s = newseq[NimNode](3) + # var s: array[3,NimNode] # not working either + for i in 0..<s.len: s[i] = newLit(3) # works + #for x in s.mitems: x = newLit(3) + result = newStmtList() + for i in s: + result.add newCall(bindsym"echo", i) + +m() diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim new file mode 100644 index 000000000..b7112b099 --- /dev/null +++ b/tests/vm/tvmmisc.nim @@ -0,0 +1,16 @@ + + +# bug #4462 +import macros + +proc foo(t: typedesc) {.compileTime.} = + echo getType(t).treeRepr + +static: + foo(int) + +# #4412 +proc default[T](t: typedesc[T]): T {.inline.} = discard + +static: + var x = default(type(0)) diff --git a/tests/vm/twrong_concat.nim b/tests/vm/twrong_concat.nim new file mode 100644 index 000000000..538ea2527 --- /dev/null +++ b/tests/vm/twrong_concat.nim @@ -0,0 +1,28 @@ +discard """ + output: '''success''' +""" + +# bug #3804 + +#import sequtils + +type AnObj = ref object + field: string + +#proc aBug(objs: seq[AnObj]) {.compileTime.} = +# discard objs.mapIt(it.field & " bug") + +proc sameBug(objs: seq[AnObj]) {.compileTime.} = + var strSeq = newSeq[string](objs.len) + strSeq[0] = objs[0].field & " bug" + +static: + var objs: seq[AnObj] = @[] + objs.add(AnObj(field: "hello")) + + sameBug(objs) + # sameBug(objs) + echo objs[0].field + assert(objs[0].field == "hello") # fails, because (objs[0].field == "hello bug") - mutated! + +echo "success" diff --git a/todo.txt b/todo.txt index a6312468a..106f2bb34 100644 --- a/todo.txt +++ b/todo.txt @@ -1,29 +1,26 @@ +version 1.0 battle plan +======================= -nim c --gc:v2 -r -d:useSysAssert -d:useGcAssert -d:smokeCycles -d:useRealtimeGc tests/gc/gctest +- iters for js +- fix "high priority" bugs +- try to fix as many compiler crashes as reasonable -- document ``this`` pragma -- document and stress test ``.partial`` object declarations -essential for 1.0 -================= +Not critical for 1.0 +==================== + +- get GC:v2 stable: nim c --gc:v2 -r -d:useSysAssert -d:useGcAssert -d:smokeCycles -d:useRealtimeGc tests/gc/gctest -- introduce newSeqOfCap(10) -- annotation support for getType() -- overloading of `()` needs to be in .experimental - find a solution for the x.f[T](y) gotcha -- ``concept`` needs to be refined, a nice name for the feature is not enough. +- implement ``.delegate`` for .experimental + - Destructors need to be refined. +- annotation support for getType() +- ``concept`` needs to be refined, a nice name for the feature is not enough. - make '--implicitStatic:on' the default; then we can also clean up the 'static[T]' mess in the compiler! - -- Deprecate ``immediate`` for templates and macros -- document NimMain and check whether it works for threading - ``not`` or ``~`` for the effects system - - -Not critical for 1.0 -==================== - +- document and stress test ``.partial`` object declarations - add "all threads are blocked" detection to 'spawn' - figure out why C++ bootstrapping is so much slower - The bitwise 'not' operator cold be renamed to 'bnot' to @@ -62,7 +59,6 @@ Bugs GC == -- hybrid GC - use big blocks in the allocator - provide tool/API to track leaks/object counts - resizing of strings/sequences could take into account the memory that diff --git a/tools/heapdumprepl.nim b/tools/heapdumprepl.nim new file mode 100644 index 000000000..ffa153f5d --- /dev/null +++ b/tools/heapdumprepl.nim @@ -0,0 +1,150 @@ + +include prelude +import intsets + +type + NodeKind = enum + internal, local, localInvalid, global, globalInvalid + Color = enum + white, grey, black + Node = ref object + id, rc: int + kids: seq[int] + k: NodeKind + col: Color + Graph = object + nodes: Table[int, Node] + roots: Table[int, NodeKind] + +proc add(father: Node; son: int) = + if father.kids.isNil: father.kids = @[] + father.kids.add(son) + +proc renderNode(g: Graph; id: int) = + let n = g.nodes[id] + echo n[] + +proc toNodeId(aliases: var Table[string,int]; s: string): int = + result = aliases.getOrDefault(s) + if result == 0: + if s.startsWith("x"): + discard s.parseHex(result, 1) + else: + result = s.parseInt + +proc parseHex(s: string): int = + discard parseutils.parseHex(s, result, 0) + +proc reachable(g: Graph; stack: var seq[int]; goal: int): bool = + var t = initIntSet() + while stack.len > 0: + let it = stack.pop + if not t.containsOrIncl(it): + if it == goal: return true + if it in g.nodes: + for kid in g.nodes[it].kids: + stack.add(kid) + +const Help = """ +quit -- quits this REPL +locals, l -- output the list of local stack roots +globals, g -- output the list of global roots +alias name addr -- give addr a name. start 'addr' with 'x' for hexidecimal + notation +print name|addr -- print a node by name or address +reachable,r l|g|node dest -- outputs TRUE or FALSE depending on whether + dest is reachable by (l)ocals, (g)lobals or by the + other given node. Nodes can be node names or node + addresses. +""" + +proc repl(g: Graph) = + var aliases = initTable[string,int]() + while true: + let line = stdin.readLine() + let data = line.split() + if data.len == 0: continue + case data[0] + of "quit": + break + of "help": + echo Help + of "locals", "l": + for k,v in g.roots: + if v == local: renderNode(g, k) + of "globals", "g": + for k,v in g.roots: + if v == global: renderNode(g, k) + of "alias", "a": + # generate alias + if data.len == 3: + aliases[data[1]] = toNodeId(aliases, data[2]) + of "print", "p": + if data.len == 2: + renderNode(g, toNodeId(aliases, data[1])) + of "reachable", "r": + if data.len == 3: + var stack: seq[int] = @[] + case data[1] + of "locals", "l": + for k,v in g.roots: + if v == local: stack.add k + of "globals", "g": + for k,v in g.roots: + if v == global: stack.add k + else: + stack.add(toNodeId(aliases, data[1])) + let goal = toNodeId(aliases, data[2]) + echo reachable(g, stack, goal) + else: discard + +proc importData(input: string): Graph = + #c_fprintf(file, "%s %p %d rc=%ld color=%c\n", + # msg, c, kind, c.refcount shr rcShift, col) + # cell 0x10a908190 22 rc=2 color=w + var i: File + var + nodes = initTable[int, Node]() + roots = initTable[int, NodeKind]() + if open(i, input): + var currNode: Node + for line in lines(i): + let data = line.split() + if data.len == 0: continue + case data[0] + of "end": + currNode = nil + of "cell": + let rc = parseInt(data[3].substr("rc=".len)) + let col = case data[4].substr("color=".len) + of "b": black + of "w": white + of "g": grey + else: (assert(false); grey) + let id = parseHex(data[1]) + currNode = Node(id: id, + k: roots.getOrDefault(id), + rc: rc, col: col) + nodes[currNode.id] = currNode + of "child": + assert currNode != nil + currNode.add parseHex(data[1]) + of "global_root": + roots[data[1].parseHex] = global + of "global_root_invalid": + roots[data[1].parseHex] = globalInvalid + of "onstack": + roots[data[1].parseHex] = local + of "onstack_invalid": + roots[data[1].parseHex] = localInvalid + else: discard + close(i) + else: + quit "error: cannot open " & input + shallowCopy(result.nodes, nodes) + shallowCopy(result.roots, roots) + +if paramCount() == 1: + repl(importData(paramStr(1))) +else: + quit "usage: heapdumprepl inputfile" diff --git a/tools/niminst/buildsh.tmpl b/tools/niminst/buildsh.tmpl index aa3bce3c3..13dfe5226 100644 --- a/tools/niminst/buildsh.tmpl +++ b/tools/niminst/buildsh.tmpl @@ -1,5 +1,5 @@ #? stdtmpl(subsChar='?') | standard -#proc generateBuildShellScript(c: ConfigData): string = +#proc generateBuildShellScript(c: ConfigData): string = # result = "#! /bin/sh\n# Generated from niminst\n" & # "# Template is in tools/niminst/buildsh.tmpl\n" & # "# To regenerate run ``niminst csource`` or ``koch csource``\n" @@ -29,8 +29,8 @@ done CC="gcc" LINKER="gcc" -COMP_FLAGS="?{c.ccompiler.flags}$extraBuildArgs" -LINK_FLAGS="?{c.linker.flags}" +COMP_FLAGS="${CPPFLAGS:-} ${CFLAGS:-} ?{c.ccompiler.flags}$extraBuildArgs" +LINK_FLAGS="${LDFLAGS:-} ?{c.linker.flags}" PS4="" # add(result, "# platform detection\n") ucpu=`uname -m` @@ -51,8 +51,8 @@ ucpu=`echo $ucpu | tr "[:upper:]" "[:lower:]"` uos=`echo $uos | tr "[:upper:]" "[:lower:]"` case $uos in - *linux* ) - myos="linux" + *linux* ) + myos="linux" LINK_FLAGS="$LINK_FLAGS -ldl -lm" ;; *dragonfly* ) @@ -66,14 +66,14 @@ case $uos in LINK_FLAGS="$LINK_FLAGS -lm" ;; *openbsd* ) - myos="openbsd" + myos="openbsd" LINK_FLAGS="$LINK_FLAGS -lm" ;; *netbsd* ) myos="netbsd" LINK_FLAGS="$LINK_FLAGS -lm" ;; - *darwin* ) + *darwin* ) myos="macosx" CC="clang" LINKER="clang" @@ -84,41 +84,45 @@ case $uos in ;; *aix* ) myos="aix" - LINK_FLAGS="$LINK_FLAGS -ldl -lm" + LINK_FLAGS="$LINK_FLAGS -ldl -lm" ;; - *solaris* | *sun* ) + *solaris* | *sun* ) myos="solaris" LINK_FLAGS="$LINK_FLAGS -ldl -lm -lsocket -lnsl" ;; *haiku* ) myos="haiku" ;; - *) + *) echo 2>&1 "Error: unknown operating system: $uos" exit 1 ;; esac case $ucpu in - *i386* | *i486* | *i586* | *i686* | *bepc* | *i86pc* ) + *i386* | *i486* | *i586* | *i686* | *bepc* | *i86pc* ) mycpu="i386" ;; - *amd*64* | *x86-64* | *x86_64* ) + *amd*64* | *x86-64* | *x86_64* ) mycpu="amd64" ;; - *sparc*|*sun* ) - mycpu="sparc" ;; - *ppc64* ) + *sparc*|*sun* ) + mycpu="sparc" + if [ "$(isainfo -b)" = "64" ]; then + mycpu="sparc64" + fi + ;; + *ppc64* ) if [ "$myos" = "linux" ] ; then COMP_FLAGS="$COMP_FLAGS -m64" LINK_FLAGS="$LINK_FLAGS -m64" fi mycpu="powerpc64" ;; - *power*|*ppc* ) + *power*|*ppc* ) mycpu="powerpc" ;; - *mips* ) + *mips* ) mycpu="mips" ;; *arm*|*armv6l* ) mycpu="arm" ;; - *) + *) echo 2>&1 "Error: unknown processor: $ucpu" exit 1 ;; @@ -128,7 +132,7 @@ esac case $myos in # for osA in 1..c.oses.len: -?{c.oses[osA-1]}) +?{c.oses[osA-1]}) case $mycpu in # for cpuA in 1..c.cpus.len: ?{c.cpus[cpuA-1]}) @@ -149,7 +153,7 @@ case $myos in esac ;; # end for -*) +*) echo 2>&1 "Error: no C code generated for: [$myos: $mycpu]" exit 1 ;; diff --git a/tools/niminst/deinstall.tmpl b/tools/niminst/deinstall.tmpl index 7349abcb4..3cdfbf45d 100644 --- a/tools/niminst/deinstall.tmpl +++ b/tools/niminst/deinstall.tmpl @@ -1,5 +1,5 @@ #? stdtmpl(subsChar='?') | standard -#proc generateDeinstallScript(c: ConfigData): string = +#proc generateDeinstallScript(c: ConfigData): string = # result = "#! /bin/sh\n# Generated by niminst\n" # var proj = c.name.toLower @@ -12,7 +12,7 @@ if [ $# -eq 1 ] ; then echo " /usr/bin" echo " /usr/local/bin" echo " /opt" - echo " <some other dir> (treated like '/opt')" + echo " <some other dir> (treated similar '/opt')" exit 1 ;; "/usr/bin") @@ -21,6 +21,7 @@ if [ $# -eq 1 ] ; then libdir=/usr/lib/?proj docdir=/usr/share/?proj/doc datadir=/usr/share/?proj/data + nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" ;; "/usr/local/bin") bindir=/usr/local/bin @@ -28,6 +29,15 @@ if [ $# -eq 1 ] ; then libdir=/usr/local/lib/?proj docdir=/usr/local/share/?proj/doc datadir=/usr/local/share/?proj/data + nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" + ;; + "/opt") + bindir="/opt/?proj/bin" + configdir="/opt/?proj/config" + libdir="/opt/?proj/lib" + docdir="/opt/?proj/doc" + datadir="/opt/?proj/data" + nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" ;; *) bindir="$1/?proj/bin" @@ -35,6 +45,7 @@ if [ $# -eq 1 ] ; then libdir="$1/?proj/lib" docdir="$1/?proj/doc" datadir="$1/?proj/data" + nimbleDir="$1/?proj" ;; esac echo "removing files..." @@ -43,7 +54,7 @@ if [ $# -eq 1 ] ; then #let f = ff.toUnix rm -f $bindir/?f.skipRoot #end for -#for ff in items(c.cat[fcConfig]): +#for ff in items(c.cat[fcConfig]): #let f = ff.toUnix rm -f $configdir/?f.skipRoot #end for @@ -51,6 +62,12 @@ if [ $# -eq 1 ] ; then rm -rf $datadir rm -rf $libdir + ## Nimble pkg stuff + #for f in items(c.cat[fcNimble]): + rm -f $nimbleDir/?f.toUnix + #end for + rm -f $nimbleDir/?{c.nimblePkgName}.nimble + echo "deinstallation successful" else echo "?c.displayName deinstallation script" @@ -59,6 +76,6 @@ else echo " /usr/bin" echo " /usr/local/bin" echo " /opt" - echo " <some other dir> (treated like '/opt')" + echo " <some other dir> (treated similar '/opt')" exit 1 fi diff --git a/tools/niminst/install.tmpl b/tools/niminst/install.tmpl index 14d88e07d..3f17840a8 100644 --- a/tools/niminst/install.tmpl +++ b/tools/niminst/install.tmpl @@ -1,14 +1,42 @@ #? stdtmpl(subsChar = '?') | standard -#proc generateInstallScript(c: ConfigData): string = +#proc generateInstallScript(c: ConfigData): string = # result = "#! /bin/sh\n# Generated by niminst\n" # var proj = c.name.toLower -set -e +## Current directory you start script from +BASE_DIR=$(pwd) + +## The following one-liner takes directory path which contains install script. +## `command -v -- "$0"` takes path if script sourced from interactive shell +## `dirname` returns relative directory path to install script +## `cd -P` dive into directory to use `pwd` +## `pwd -P` prints full path to install script directory path +## -P option allows to use symlinks in path +## Good explanation can be found here: +## http://stackoverflow.com/questions/29832037/how-to-get-script-directory-in-posix-sh +NIM_DIR=$(cd -P -- "$(dirname -- "$(command -v -- "$0")")" && pwd -P) + +go_back() { + cd $BASE_DIR +} + +## Go to base dir on exit +trap go_back EXIT + +install_error() { + echo "Nim installation failed!" + exit 1 +} + +## Exit if any command failed +trap install_error ERR ## `set -e` alternative + +cd $NIM_DIR if [ $# -eq 1 ] ; then # if c.cat[fcUnixBin].len > 0: if test -f ?{c.cat[fcUnixBin][0].toUnix} - then + then echo "?c.displayName build detected" else echo "Please build ?c.displayName before installing it" @@ -23,7 +51,7 @@ if [ $# -eq 1 ] ; then echo " /usr/bin" echo " /usr/local/bin" echo " /opt" - echo " <some other dir> (treated like '/opt')" + echo " <some other dir> (treated similar to '/opt')" echo "To deinstall, use the command:" echo "sh deinstall.sh DIR" exit 1 @@ -34,6 +62,7 @@ if [ $# -eq 1 ] ; then libdir=/usr/lib/?proj docdir=/usr/share/?proj/doc datadir=/usr/share/?proj/data + nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" ;; "/usr/local/bin") bindir=/usr/local/bin @@ -41,6 +70,18 @@ if [ $# -eq 1 ] ; then libdir=/usr/local/lib/?proj docdir=/usr/local/share/?proj/doc datadir=/usr/local/share/?proj/data + nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" + ;; + "/opt") + bindir="/opt/?proj/bin" + configdir="/opt/?proj/config" + libdir="/opt/?proj/lib" + docdir="/opt/?proj/doc" + datadir="/opt/?proj/data" + nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" + mkdir -p /opt/?proj + mkdir -p $bindir + mkdir -p $configdir ;; *) bindir="$1/?proj/bin" @@ -48,25 +89,29 @@ if [ $# -eq 1 ] ; then libdir="$1/?proj/lib" docdir="$1/?proj/doc" datadir="$1/?proj/data" - + nimbleDir="$1/?proj" mkdir -p $1/?proj mkdir -p $bindir mkdir -p $configdir ;; esac + mkdir -p $libdir mkdir -p $docdir + mkdir -p $nimbleDir/ echo "copying files..." #var createdDirs = newStringTable() -#for cat in fcConfig..fcLib: +#for cat in {fcConfig..fcLib, fcNimble}: # for f in items(c.cat[cat]): # var mk = splitFile(f.skipRoot).dir -# if mk.len > 0: +# if cat != fcNimble: # mk = unixDirVars[cat] & "/" & mk -# if not createdDirs.hasKey(mk): -# createdDirs[mk] = "true" +# else: +# mk = "$nimbleDir" / splitFile(f).dir +# end if +# if mk.len > 0 and not createdDirs.hasKey(mk): +# createdDirs[mk] = "true" mkdir -p ?{mk.toUnix} -# end if # end if # end for #end for @@ -75,11 +120,11 @@ if [ $# -eq 1 ] ; then cp ?f.toUnix $bindir/?f.skipRoot.toUnix chmod 755 $bindir/?f.skipRoot.toUnix #end for -#for f in items(c.cat[fcConfig]): +#for f in items(c.cat[fcConfig]): cp ?f.toUnix $configdir/?f.skipRoot.toUnix chmod 644 $configdir/?f.skipRoot.toUnix #end for -#for f in items(c.cat[fcData]): +#for f in items(c.cat[fcData]): if [ -f ?f.toUnix ]; then cp ?f.toUnix $datadir/?f.skipRoot.toUnix chmod 644 $datadir/?f.skipRoot.toUnix @@ -95,7 +140,13 @@ if [ $# -eq 1 ] ; then cp ?f.toUnix $libdir/?f.skipRoot.toUnix chmod 644 $libdir/?f.skipRoot.toUnix #end for - +#for f in items(c.cat[fcNimble]): + cp ?f.toUnix $nimbleDir/?f.toUnix + chmod 644 $nimbleDir/?f.toUnix +#end for +cp ?{c.nimblePkgName}.nimble $nimbleDir/?{c.nimblePkgName}.nimble +chmod 644 $nimbleDir/?{c.nimblePkgName}.nimble + echo "installation successful" else echo "?c.displayName installation script" @@ -104,7 +155,7 @@ else echo " /usr/bin" echo " /usr/local/bin" echo " /opt" - echo " <some other dir> (treated like '/opt')" + echo " <some other dir> (treated similar to '/opt')" echo "To deinstall, use the command:" echo "sh deinstall.sh DIR" exit 1 diff --git a/tools/niminst/makefile.tmpl b/tools/niminst/makefile.tmpl index 6615ddc02..5c95ccda9 100644 --- a/tools/niminst/makefile.tmpl +++ b/tools/niminst/makefile.tmpl @@ -1,13 +1,13 @@ #? stdtmpl(subsChar='?') | standard -#proc generateMakefile(c: ConfigData): string = +#proc generateMakefile(c: ConfigData): string = # result = "# Generated from niminst\n" & # "# Template is in tools/niminst/makefile.tmpl\n" & # "# To regenerate run ``niminst csource`` or ``koch csource``\n" CC = gcc LINKER = gcc -COMP_FLAGS = ?{c.ccompiler.flags} -LINK_FLAGS = ?{c.linker.flags} +COMP_FLAGS = $(CPPFLAGS) $(CFLAGS) ?{c.ccompiler.flags} +LINK_FLAGS = $(LDFLAGS) ?{c.linker.flags} binDir = ?{firstBinPath(c).toUnix} koch := $(shell sh -c 'test -s ../koch.nim && echo "yes"') diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index d1216701f..4c8dfcddf 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -18,7 +18,7 @@ import const maxOS = 20 # max number of OSes - maxCPU = 10 # max number of CPUs + maxCPU = 20 # max number of CPUs buildShFile = "build.sh" buildBatFile32 = "build.bat" buildBatFile64 = "build64.bat" @@ -48,7 +48,8 @@ type fcWindows, # files only for Windows fcUnix, # files only for Unix; must be after ``fcWindows`` fcUnixBin, # binaries for Unix - fcDocStart # links to documentation for Windows installer + fcDocStart, # links to documentation for Windows installer + fcNimble # nimble package files to copy to /opt/nimble/pkgs/pkg-ver ConfigData = object of RootObj actions: set[Action] @@ -65,6 +66,7 @@ type app: AppType nimArgs: string debOpts: TDebOptions + nimblePkgName: string const unixDirVars: array[fcConfig..fcLib, string] = [ @@ -200,22 +202,63 @@ proc parseCmdLine(c: var ConfigData) = if c.infile.len == 0: quit(Usage) if c.mainfile.len == 0: c.mainfile = changeFileExt(c.infile, "nim") -proc walkDirRecursively(s: var seq[string], root: string) = +proc eqT(a, b: string; t: proc (a: char): char{.nimcall.}): bool = + ## equality under a transformation ``t``. candidate for the stdlib? + var i = 0 + var j = 0 + while i < a.len and j < b.len: + let aa = t a[i] + let bb = t b[j] + if aa == '\0': + inc i + if bb == '\0': inc j + elif bb == '\0': inc j + else: + if aa != bb: return false + inc i + inc j + result = i >= a.len and j >= b.len + +proc tPath(c: char): char = + if c == '\\': '/' + else: c + +proc ignoreFile(f, explicit: string, allowHtml: bool): bool = + let (_, name, ext) = splitFile(f) + let html = if not allowHtml: ".html" else: "" + result = (ext in ["", ".exe", ".idx", ".o", ".obj", ".dylib"] or + ext == html or name[0] == '.') and not eqT(f, explicit, tPath) + +proc walkDirRecursively(s: var seq[string], root, explicit: string, + allowHtml: bool) = + let tail = splitPath(root).tail + if tail == "nimcache" or tail[0] == '.': + return + let allowHtml = allowHtml or tail == "doc" for k, f in walkDir(root): - case k - of pcFile, pcLinkToFile: add(s, unixToNativePath(f)) - of pcDir: walkDirRecursively(s, f) - of pcLinkToDir: discard + if f[0] == '.' and root[0] != '.': + discard "skip .git directories etc" + else: + case k + of pcFile, pcLinkToFile: + if not ignoreFile(f, explicit, allowHtml): + add(s, unixToNativePath(f)) + of pcDir: + walkDirRecursively(s, f, explicit, allowHtml) + of pcLinkToDir: discard proc addFiles(s: var seq[string], patterns: seq[string]) = for p in items(patterns): if existsDir(p): - walkDirRecursively(s, p) + walkDirRecursively(s, p, p, false) else: var i = 0 - for f in walkFiles(p): - add(s, unixToNativePath(f)) - inc(i) + for f in walkPattern(p): + if existsDir(f): + walkDirRecursively(s, f, p, false) + elif not ignoreFile(f, p, false): + add(s, unixToNativePath(f)) + inc(i) if i == 0: echo("[Warning] No file found that matches: " & p) proc pathFlags(p: var CfgParser, k, v: string, @@ -362,6 +405,14 @@ proc parseIniFile(c: var ConfigData) = else: file.add(v[i]) inc(i) else: quit(errorStr(p, "unknown variable: " & k.key)) + of "nimble": + case normalize(k.key) + of "pkgname": + c.nimblePkgName = v + of "pkgfiles": + addFiles(c.cat[fcNimble], split(v, {';'})) + else: + quit(errorStr(p, "invalid key: " & k.key)) else: quit(errorStr(p, "invalid section: " & section)) of cfgOption: quit(errorStr(p, "syntax error")) @@ -554,8 +605,13 @@ when haveZipLib: for k, f in walkDir("build" / dir): if k == pcFile: addFile(z, proj / dir / extractFilename(f), f) - for cat in items({fcConfig..fcOther, fcUnix}): + for cat in items({fcConfig..fcOther, fcUnix, fcNimble}): for f in items(c.cat[cat]): addFile(z, proj / f, f) + + # Copy the .nimble file over + let nimbleFile = c.nimblePkgName & ".nimble" + processFile(z, proj / nimbleFile, nimbleFile) + close(z) else: quit("Cannot open for writing: " & n) @@ -587,16 +643,24 @@ proc xzDist(c: var ConfigData) = for k, f in walkDir("build" / dir): if k == pcFile: processFile(z, proj / dir / extractFilename(f), f) - for cat in items({fcConfig..fcOther, fcUnix}): + for cat in items({fcConfig..fcOther, fcUnix, fcNimble}): + echo("Current category: ", cat) for f in items(c.cat[cat]): processFile(z, proj / f, f) - let oldDir = getCurrentDir() - setCurrentDir(tmpDir) - try: - if execShellCmd("XZ_OPT=-9 tar Jcf $1.tar.xz $1" % proj) != 0: - echo("External program failed") - finally: - setCurrentDir(oldDir) + # Copy the .nimble file over + let nimbleFile = c.nimblePkgName & ".nimble" + processFile(z, proj / nimbleFile, nimbleFile) + + when true: + let oldDir = getCurrentDir() + setCurrentDir(tmpDir) + try: + if execShellCmd("XZ_OPT=-9 gtar Jcf $1.tar.xz $1 --exclude=.DS_Store" % proj) != 0: + # try old 'tar' without --exclude feature: + if execShellCmd("XZ_OPT=-9 tar Jcf $1.tar.xz $1" % proj) != 0: + echo("External program failed") + finally: + setCurrentDir(oldDir) # -- prepare build files for .deb creation diff --git a/tools/niminst/nsis.tmpl b/tools/niminst/nsis.tmpl index abf462388..0897d36a8 100644 --- a/tools/niminst/nsis.tmpl +++ b/tools/niminst/nsis.tmpl @@ -35,7 +35,7 @@ ; Default installation folder ; This is changed later (in .onInit) to the root directory, if possible. - InstallDir "$LOCALAPPDATA\?{c.name}-?{c.version}" + InstallDir "$PROGRAMFILES?{when sizeof(int) == 8: "64" else: ""}\?{c.name}-?{c.version}" ; Get installation folder from registry if available InstallDirRegKey HKCU "Software\c.name\c.version" "" diff --git a/tools/nimweb.nim b/tools/nimweb.nim index d94a75162..4cf7020c2 100644 --- a/tools/nimweb.nim +++ b/tools/nimweb.nim @@ -9,7 +9,9 @@ import os, strutils, times, parseopt, parsecfg, streams, strtabs, tables, - re, htmlgen, macros, md5, osproc + re, htmlgen, macros, md5, osproc, parsecsv, algorithm + +from xmltree import escape type TKeyValPair = tuple[key, id, val: string] @@ -25,9 +27,18 @@ type numProcessors: int # Set by parallelBuild:n, only works for values > 0. gaId: string # google analytics ID, nil means analytics are disabled TRssItem = object - year, month, day, title: string + year, month, day, title, url, content: string TAction = enum - actAll, actOnlyWebsite, actPdf + actAll, actOnlyWebsite, actPdf, actJson2 + + Sponsor = object + logo: string + name: string + url: string + thisMonth: int + allTime: int + since: string + level: int var action: TAction @@ -80,9 +91,10 @@ Compile_options: will be passed to the Nim compiler """ - rYearMonthDayTitle = r"(\d{4})-(\d{2})-(\d{2})\s+(.*)" + rYearMonthDay = r"(\d{4})_(\d{2})_(\d{2})" rssUrl = "http://nim-lang.org/news.xml" rssNewsUrl = "http://nim-lang.org/news.html" + sponsors = "web/sponsors.csv" validAnchorCharacters = Letters + Digits @@ -145,6 +157,7 @@ proc parseCmdLine(c: var TConfigData) = c.vars[substr(val, 0, idx-1)] = substr(val, idx+1) of "website": action = actOnlyWebsite of "pdf": action = actPdf + of "json2": action = actJson2 of "googleanalytics": c.gaId = val c.nimArgs.add("--doc.googleAnalytics:" & val & " ") @@ -165,10 +178,10 @@ proc walkDirRecursively(s: var seq[string], root, ext: string) = proc addFiles(s: var seq[string], dir, ext: string, patterns: seq[string]) = for p in items(patterns): + if existsFile(dir / addFileExt(p, ext)): + s.add(dir / addFileExt(p, ext)) if existsDir(dir / p): walkDirRecursively(s, dir / p, ext) - else: - add(s, dir / addFileExt(p, ext)) proc parseIniFile(c: var TConfigData) = var @@ -207,8 +220,8 @@ proc parseIniFile(c: var TConfigData) = of "ticker": c.ticker = v of "documentation": case normalize(k.key) - of "doc": addFiles(c.doc, "doc", ".txt", split(v, {';'})) - of "pdf": addFiles(c.pdf, "doc", ".txt", split(v, {';'})) + of "doc": addFiles(c.doc, "doc", ".rst", split(v, {';'})) + of "pdf": addFiles(c.pdf, "doc", ".rst", split(v, {';'})) of "srcdoc": addFiles(c.srcdoc, "lib", ".nim", split(v, {';'})) of "srcdoc2": addFiles(c.srcdoc2, "lib", ".nim", split(v, {';'})) of "webdoc": addFiles(c.webdoc, "lib", ".nim", split(v, {';'})) @@ -239,9 +252,22 @@ proc parseIniFile(c: var TConfigData) = # ------------------- main ---------------------------------------------------- + +proc exe(f: string): string = return addFileExt(f, ExeExt) + +proc findNim(): string = + var nim = "nim".exe + result = "bin" / nim + if existsFile(result): return + for dir in split(getEnv("PATH"), PathSep): + if existsFile(dir / nim): return dir / nim + # assume there is a symlink to the exe or something: + return nim + proc exec(cmd: string) = echo(cmd) - if os.execShellCmd(cmd) != 0: quit("external program failed") + let (_, exitCode) = osproc.execCmdEx(cmd) + if exitCode != 0: quit("external program failed") proc sexec(cmds: openarray[string]) = ## Serial queue wrapper around exec. @@ -263,9 +289,9 @@ proc buildDocSamples(c: var TConfigData, destPath: string) = ## it didn't make much sense to integrate into the existing generic ## documentation builders. const src = "doc"/"docgen_sample.nim" - exec("nim doc $# -o:$# $#" % + exec(findNim() & " doc $# -o:$# $#" % [c.nimArgs, destPath / "docgen_sample.html", src]) - exec("nim doc2 $# -o:$# $#" % + exec(findNim() & " doc2 $# -o:$# $#" % [c.nimArgs, destPath / "docgen_sample2.html", src]) proc pathPart(d: string): string = splitFile(d).dir.replace('\\', '/') @@ -276,30 +302,30 @@ proc buildDoc(c: var TConfigData, destPath: string) = commands = newSeq[string](len(c.doc) + len(c.srcdoc) + len(c.srcdoc2)) i = 0 for d in items(c.doc): - commands[i] = "nim rst2html $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % + commands[i] = findNim() & " rst2html $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % [c.nimArgs, c.gitRepo, c.gitCommit, d.pathPart, destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc for d in items(c.srcdoc): - commands[i] = "nim doc $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % + commands[i] = findNim() & " doc $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % [c.nimArgs, c.gitRepo, c.gitCommit, d.pathPart, destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc for d in items(c.srcdoc2): - commands[i] = "nim doc2 $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % + commands[i] = findNim() & " doc2 $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % [c.nimArgs, c.gitRepo, c.gitCommit, d.pathPart, destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc mexec(commands, c.numProcessors) - exec("nim buildIndex -o:$1/theindex.html $1" % [destPath]) + exec(findNim() & " buildIndex -o:$1/theindex.html $1" % [destPath]) proc buildPdfDoc(c: var TConfigData, destPath: string) = if os.execShellCmd("pdflatex -version") != 0: echo "pdflatex not found; no PDF documentation generated" else: for d in items(c.pdf): - exec("nim rst2tex $# $#" % [c.nimArgs, d]) + exec(findNim() & " rst2tex $# $#" % [c.nimArgs, d]) # call LaTeX twice to get cross references right: exec("pdflatex " & changeFileExt(d, "tex")) exec("pdflatex " & changeFileExt(d, "tex")) @@ -319,26 +345,26 @@ proc buildAddDoc(c: var TConfigData, destPath: string) = # build additional documentation (without the index): var commands = newSeq[string](c.webdoc.len) for i, doc in pairs(c.webdoc): - commands[i] = "nim doc2 $# --docSeeSrcUrl:$#/$#/$# -o:$# $#" % - [c.nimArgs, c.gitRepo, c.gitCommit, doc.pathPart, + commands[i] = findNim() & " doc2 $# --docSeeSrcUrl:$#/$# -o:$# $#" % + [c.nimArgs, c.gitRepo, c.gitCommit, destPath / changeFileExt(splitFile(doc).name, "html"), doc] mexec(commands, c.numProcessors) proc parseNewsTitles(inputFilename: string): seq[TRssItem] = - # parses the file for titles and returns them as TRssItem blocks. - let reYearMonthDayTitle = re(rYearMonthDayTitle) - var - input: File - line = "" - + # Goes through each news file, returns its date/title. result = @[] - if not open(input, inputFilename): - quit("Could not read $1 for rss generation" % [inputFilename]) - defer: input.close() - while input.readLine(line): - if line =~ reYearMonthDayTitle: - result.add(TRssItem(year: matches[0], month: matches[1], day: matches[2], - title: matches[3])) + let reYearMonthDay = re(rYearMonthDay) + for kind, path in walkDir(inputFilename): + let (dir, name, ext) = path.splitFile + if ext == ".rst": + let content = readFile(path) + let title = content.splitLines()[0] + let urlPath = "news/" & name & ".html" + if name =~ reYearMonthDay: + result.add(TRssItem(year: matches[0], month: matches[1], day: matches[2], + title: title, url: "http://nim-lang.org/" & urlPath, + content: content)) + result.reverse() proc genUUID(text: string): string = # Returns a valid RSS uuid, which is basically md5 with dashes and a prefix. @@ -354,7 +380,7 @@ proc genNewsLink(title: string): string = result = title result.insert("Z") for i in 1..len(result)-1: - let letter = result[i].toLower() + let letter = result[i].toLowerAscii() if letter in validAnchorCharacters: result[i] = letter else: @@ -382,34 +408,86 @@ proc generateRss(outputFilename: string, news: seq[TRssItem]) = output.write(updatedDate($now.year, $(int(now.month) + 1), $now.monthday)) for rss in news: - let joinedTitle = "$1-$2-$3 $4" % [rss.year, rss.month, rss.day, rss.title] output.write(entry( - title(joinedTitle), - id(genUUID(joinedTitle)), + title(xmltree.escape(rss.title)), + id(genUUID(rss.title)), link(`type` = "text/html", rel = "alternate", - href = genNewsLink(joinedTitle)), + href = rss.url), updatedDate(rss.year, rss.month, rss.day), "<author><name>Nim</name></author>", - content(joinedTitle, `type` = "text"), + content(xmltree.escape(rss.content), `type` = "text"), )) output.write("""</feed>""") proc buildNewsRss(c: var TConfigData, destPath: string) = - # generates an xml feed from the web/news.txt file + # generates an xml feed from the web/news.rst file let - srcFilename = "web" / "news.txt" + srcFilename = "web" / "news" destFilename = destPath / changeFileExt(splitFile(srcFilename).name, "xml") generateRss(destFilename, parseNewsTitles(srcFilename)) proc buildJS(destPath: string) = - exec("nim js -d:release --out:$1 web/nimblepkglist.nim" % + exec(findNim() & " js -d:release --out:$1 web/nimblepkglist.nim" % [destPath / "nimblepkglist.js"]) +proc readSponsors(sponsorsFile: string): seq[Sponsor] = + result = @[] + var fileStream = newFileStream(sponsorsFile, fmRead) + if fileStream == nil: quit("Cannot open sponsors.csv file: " & sponsorsFile) + var parser: CsvParser + open(parser, fileStream, sponsorsFile) + discard readRow(parser) # Skip the header row. + while readRow(parser): + result.add(Sponsor(logo: parser.row[0], name: parser.row[1], + url: parser.row[2], thisMonth: parser.row[3].parseInt, + allTime: parser.row[4].parseInt, + since: parser.row[5], level: parser.row[6].parseInt)) + parser.close() + +proc buildSponsors(c: var TConfigData, sponsorsFile: string, outputDir: string) = + let sponsors = generateSponsors(readSponsors(sponsorsFile)) + let outFile = outputDir / "sponsors.html" + var f: File + if open(f, outFile, fmWrite): + writeLine(f, generateHtmlPage(c, "", "Our Sponsors", sponsors, "")) + close(f) + else: + quit("[Error] Cannot write file: " & outFile) + +const + cmdRst2Html = " rst2html --compileonly $1 -o:web/$2.temp web/$2.rst" + +proc buildPage(c: var TConfigData, file, title, rss: string, assetDir = "") = + exec(findNim() & cmdRst2Html % [c.nimArgs, file]) + var temp = "web" / changeFileExt(file, "temp") + var content: string + try: + content = readFile(temp) + except IOError: + quit("[Error] cannot open: " & temp) + var f: File + var outfile = "web/upload/$#.html" % file + if not existsDir(outfile.splitFile.dir): + createDir(outfile.splitFile.dir) + if open(f, outfile, fmWrite): + writeLine(f, generateHTMLPage(c, file, title, content, rss, assetDir)) + close(f) + else: + quit("[Error] cannot write file: " & outfile) + removeFile(temp) + +proc buildNews(c: var TConfigData, newsDir: string, outputDir: string) = + for kind, path in walkDir(newsDir): + let (dir, name, ext) = path.splitFile + if ext == ".rst": + let title = readFile(path).splitLines()[0] + buildPage(c, tailDir(dir) / name, title, "", "../") + else: + echo("Skipping file in news directory: ", path) + proc buildWebsite(c: var TConfigData) = - const - cmd = "nim rst2html --compileonly $1 -o:web/$2.temp web/$2.txt" if c.ticker.len > 0: try: c.ticker = readFile("web" / c.ticker) @@ -419,25 +497,11 @@ proc buildWebsite(c: var TConfigData) = var file = c.tabs[i].val let rss = if file in ["news", "index"]: extractFilename(rssUrl) else: "" if '.' in file: continue - exec(cmd % [c.nimArgs, file]) - var temp = "web" / changeFileExt(file, "temp") - var content: string - try: - content = readFile(temp) - except IOError: - quit("[Error] cannot open: " & temp) - var f: File - var outfile = "web/upload/$#.html" % file - if not existsDir("web/upload"): - createDir("web/upload") - if open(f, outfile, fmWrite): - writeLine(f, generateHTMLPage(c, file, content, rss)) - close(f) - else: - quit("[Error] cannot write file: " & outfile) - removeFile(temp) + buildPage(c, file, if file == "question": "FAQ" else: file, rss) copyDir("web/assets", "web/upload/assets") buildNewsRss(c, "web/upload") + buildSponsors(c, sponsors, "web/upload") + buildNews(c, "web/news", "web/upload/news") proc main(c: var TConfigData) = buildWebsite(c) @@ -448,6 +512,19 @@ proc main(c: var TConfigData) = buildDocSamples(c, "doc") buildDoc(c, "doc") +proc json2(c: var TConfigData) = + const destPath = "web/json2" + var commands = newSeq[string](c.srcdoc2.len) + var i = 0 + for d in items(c.srcdoc2): + createDir(destPath / splitFile(d).dir) + commands[i] = findNim() & " jsondoc2 $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % + [c.nimArgs, c.gitRepo, c.gitCommit, d.pathPart, + destPath / changeFileExt(d, "json"), d] + i.inc + + mexec(commands, c.numProcessors) + var c: TConfigData initConfigData(c) parseCmdLine(c) @@ -456,3 +533,4 @@ case action of actOnlyWebsite: buildWebsite(c) of actPdf: buildPdfDoc(c, "doc") of actAll: main(c) +of actJson2: json2(c) diff --git a/tools/website.tmpl b/tools/website.tmpl index 50152a051..cf2c72a60 100644 --- a/tools/website.tmpl +++ b/tools/website.tmpl @@ -1,22 +1,33 @@ #? stdtmpl | standard -#proc generateHTMLPage(c: var TConfigData, currentTab, content, rss: string): string = +#proc generateHTMLPage(c: var TConfigData, currentTab, title, content, rss, +# rootDir = ""): string = # result = "" <!DOCTYPE html> <html> <head> <meta http-equiv="content-type" content="text/html; charset=utf-8"> - <title>$c.projectTitle</title> - <link rel="stylesheet" type="text/css" href="assets/style.css" /> - <link rel="shortcut icon" href="assets/images/favicon.ico"> + <title>${title} - $c.projectTitle</title> + <link rel="stylesheet" type="text/css" href="${rootDir}assets/style.css?t=2320" /> + <link rel="shortcut icon" href="${rootDir}assets/images/favicon.ico"> #if len(rss) > 0: <link href="$rss" title="Recent changes" type="application/atom+xml" rel="alternate"> #end if </head> <body> + <div id="bountysource"> + <a href="https://salt.bountysource.com/teams/nim"> + <div class="page-layout" style="padding: 2pt 2pt 2pt 30pt"> + <img src="${rootDir}assets/bountysource/bountysource.png" style="width: 20px; float: left;"> + <span style="margin-left: 10pt; float: left; margin-top: 2pt;">Fund Nim and help us develop it further!</span> + <img src="https://api.bountysource.com/badge/team?team_id=19072&style=raised" style="margin-top: 2pt; margin-left: 10pt"/> + </div> + </a> + </div> + <header id="head"> <div class="page-layout tall"> <div id="head-logo"></div> - <a id="head-logo-link" href="index.html"></a> + <a id="head-logo-link" href="${rootDir}index.html"></a> <nav id="head-links"> #for i in 0.. c.tabs.len-1: # let t = c.tabs[i].val @@ -30,7 +41,7 @@ # if t.contains('.'): href="${t}" title = "$c.projectName - $name">$name</a> # else: - href="${t}.html" title = "$c.projectName - $name">$name</a> + href="${rootDir}${t}.html" title = "$c.projectName - $name">$name</a> # end if # end if #end for @@ -50,11 +61,16 @@ <div id="slideshow"> <!-- slides --> <div id="slide0" class="active niaslide"> + <a href="sponsors.html"> + <img src="${rootDir}assets/bountysource/meet_sponsors.png" alt="Meet our BountySource sponsors!"/> + </a> + </div> + <div id="slide1" class="niaslide"> <a href="news.html#Z2016-01-27-nim-in-action-is-now-available"> - <img src="assets/niminaction/banner.jpg" alt="New in Manning Early Access Program: Nim in Action!"/> + <img src="${rootDir}assets/niminaction/banner.jpg" alt="New in Manning Early Access Program: Nim in Action!"/> </a> </div> - <div id="slide1" class="codeslide2"> + <div id="slide2" class="codeslide2"> <div> <h2>Nim is simple..</h2> <pre> @@ -88,7 +104,7 @@ p.greet() <span class="cmt"># or greet(p)</span> </pre> </div> </div> - <div id="slide2" class="codeslide3"> + <div id="slide3" class="codeslide3"> <div> <h2>C FFI is easy in Nim..</h2> <pre> @@ -125,6 +141,7 @@ runForever() <div id="slideControl0" onclick="slideshow_click(0)" class="active"></div> <div id="slideControl1" onclick="slideshow_click(1)"></div> <div id="slideControl2" onclick="slideshow_click(2)"></div> + <div id="slideControl3" onclick="slideshow_click(3)"></div> </div> # end <aside id="sidebar"> @@ -140,7 +157,7 @@ runForever() # if len(c.ticker) > 0: <h3 class="blue">Latest News</h3> <div id="sidebar-news"> - $c.ticker + ${c.ticker % rootDir} </div> # end if </aside> @@ -184,7 +201,7 @@ runForever() </footer> # if currentTab == "index": - <script src="assets/index.js"></script> + <script src="${rootDir}assets/index.js"></script> # end if # if c.gaId != nil: <script> @@ -199,3 +216,38 @@ runForever() # end if </body> </html> +#end proc +# +#proc generateSponsors(sponsors: seq[Sponsor]): string = +#result = "" +<h1 id="our-current-sponsors">Our Current Sponsors</h1> +<p>This page lists the companies and individuals that are, very kindly, contributing a +monthly amount to help sustain Nim's development. For more details take a +look at the <a href="https://salt.bountysource.com/teams/nim">Bountysource campaign</a>.</p> +<p class="lastUpdate">Last updated: ${getTime().getGMTime().format("dd/MM/yyyy")}</p> +<dl> +#for sponsor in sponsors: + <dt class="level-${sponsor.level}"> + #if sponsor.url.len > 0: + <a href="${sponsor.url}" target="_blank">${sponsor.name}</a> + #else: + ${sponsor.name} + #end if + </dt> + <dd class="logo"> + #if sponsor.logo.len > 0: + <a href="${sponsor.url}" target="_blank"> + <img alt="${sponsor.name}'s logo" src="${sponsor.logo}"/> + </a> + #end if + </dd> + <dd class="this_month"> + Donated <b>$$${sponsor.thisMonth}</b> this month + </dd> + <dd class="legend"> + Donated $$${sponsor.allTime} in total since ${sponsor.since} + </dd> +#end for +</dl> +# +#end proc diff --git a/web/assets/bountysource/bountysource.png b/web/assets/bountysource/bountysource.png new file mode 100644 index 000000000..d92d75df0 --- /dev/null +++ b/web/assets/bountysource/bountysource.png Binary files differdiff --git a/web/assets/bountysource/meet_sponsors.png b/web/assets/bountysource/meet_sponsors.png new file mode 100644 index 000000000..67d7a1042 --- /dev/null +++ b/web/assets/bountysource/meet_sponsors.png Binary files differdiff --git a/web/assets/bountysource/meet_sponsors.xcf b/web/assets/bountysource/meet_sponsors.xcf new file mode 100644 index 000000000..e36560491 --- /dev/null +++ b/web/assets/bountysource/meet_sponsors.xcf Binary files differdiff --git a/web/assets/bountysource/secondspectrum.png b/web/assets/bountysource/secondspectrum.png new file mode 100644 index 000000000..4dab35c6f --- /dev/null +++ b/web/assets/bountysource/secondspectrum.png Binary files differdiff --git a/web/assets/bountysource/xored.png b/web/assets/bountysource/xored.png new file mode 100644 index 000000000..7a07cdd31 --- /dev/null +++ b/web/assets/bountysource/xored.png Binary files differdiff --git a/web/assets/bountysource/xored.svg b/web/assets/bountysource/xored.svg new file mode 100644 index 000000000..17a4acde6 --- /dev/null +++ b/web/assets/bountysource/xored.svg @@ -0,0 +1 @@ +<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg width="100%" height="100%" viewBox="0 0 317 110" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:1.41421;"><g id="Layer 1"><path d="M102.691,70.309c-1.469,2.545 -1.59,5.494 -0.593,8.032l-0.026,-0.015c0.988,2.533 0.863,5.473 -0.602,8.011c-1.465,2.537 -3.949,4.115 -6.637,4.526l0.029,0.017c-2.698,0.405 -5.193,1.984 -6.663,4.53c-2.561,4.436 -1.041,10.109 3.395,12.67c4.436,2.561 10.109,1.041 12.67,-3.395c1.47,-2.546 1.59,-5.497 0.592,-8.035l0.029,0.016c-0.988,-2.533 -0.864,-5.473 0.601,-8.011c1.465,-2.537 3.949,-4.115 6.637,-4.526l-0.026,-0.015c2.696,-0.406 5.19,-1.985 6.659,-4.53c2.561,-4.436 1.041,-10.109 -3.395,-12.67c-4.436,-2.561 -10.109,-1.041 -12.67,3.395M15.736,20.106c-1.469,2.545 -1.589,5.494 -0.593,8.032l-0.025,-0.015c0.987,2.533 0.862,5.473 -0.603,8.01c-1.465,2.538 -3.949,4.116 -6.637,4.527l0.029,0.017c-2.697,0.404 -5.193,1.984 -6.663,4.529c-2.561,4.437 -1.041,10.109 3.395,12.67c4.437,2.562 10.109,1.042 12.671,-3.394c1.469,-2.546 1.589,-5.497 0.591,-8.035l0.029,0.016c-0.988,-2.533 -0.864,-5.473 0.602,-8.011c1.465,-2.538 3.949,-4.115 6.636,-4.527l-0.026,-0.015c2.696,-0.405 5.19,-1.984 6.659,-4.529c2.562,-4.436 1.042,-10.109 -3.395,-12.67c-4.436,-2.561 -10.108,-1.042 -12.67,3.395M88.199,45.207c-1.47,2.546 -1.604,5.489 -0.606,8.027c0.987,2.533 0.849,5.465 -0.617,8.003c-1.465,2.537 -3.935,4.123 -6.622,4.535c-2.697,0.405 -5.178,1.992 -6.648,4.537c-1.47,2.546 -1.604,5.488 -0.606,8.027c0.988,2.533 0.849,5.465 -0.616,8.003c-1.465,2.537 -3.935,4.123 -6.623,4.534c-2.697,0.405 -5.178,1.992 -6.647,4.538c-2.562,4.436 -1.042,10.109 3.395,12.67c4.436,2.561 10.108,1.041 12.67,-3.395c1.469,-2.545 1.603,-5.488 0.606,-8.026c-0.988,-2.533 -0.85,-5.465 0.615,-8.003c1.465,-2.537 3.935,-4.124 6.623,-4.535c2.697,-0.405 5.178,-1.992 6.648,-4.538c1.47,-2.545 1.603,-5.487 0.606,-8.025c-0.988,-2.533 -0.849,-5.466 0.616,-8.003c1.465,-2.538 3.935,-4.124 6.622,-4.535c2.698,-0.405 5.179,-1.993 6.649,-4.538c2.561,-4.437 1.041,-10.109 -3.395,-12.671c-4.437,-2.561 -10.109,-1.041 -12.67,3.395M44.721,20.106c-1.47,2.545 -1.604,5.488 -0.606,8.026c0.988,2.533 0.849,5.465 -0.616,8.003c-1.465,2.538 -3.935,4.124 -6.623,4.535c-2.697,0.405 -5.178,1.992 -6.647,4.537c-1.47,2.546 -1.604,5.489 -0.606,8.027c0.988,2.533 0.849,5.465 -0.616,8.003c-1.465,2.537 -3.936,4.123 -6.623,4.535c-2.697,0.405 -5.178,1.992 -6.648,4.537c-2.561,4.436 -1.041,10.109 3.395,12.67c4.437,2.561 10.109,1.041 12.67,-3.395c1.47,-2.545 1.604,-5.487 0.606,-8.025c-0.987,-2.533 -0.849,-5.466 0.616,-8.003c1.465,-2.538 3.935,-4.124 6.623,-4.535c2.697,-0.405 5.178,-1.993 6.648,-4.538c1.469,-2.546 1.603,-5.488 0.606,-8.026c-0.988,-2.533 -0.85,-5.465 0.615,-8.003c1.466,-2.538 3.935,-4.124 6.623,-4.535c2.697,-0.405 5.179,-1.992 6.648,-4.538c2.562,-4.436 1.042,-10.109 -3.395,-12.67c-4.436,-2.561 -10.108,-1.042 -12.67,3.395M86.376,16.711c4.436,2.561 5.956,8.234 3.395,12.67c-1.469,2.546 -3.965,4.125 -6.662,4.53l0.028,0.016c-2.687,0.411 -5.171,1.989 -6.636,4.527c-1.466,2.537 -1.59,5.478 -0.602,8.011l-0.028,-0.016c0.997,2.538 0.877,5.488 -0.592,8.034c-1.469,2.544 -3.963,4.123 -6.659,4.529l0.026,0.015c-2.688,0.411 -5.172,1.989 -6.637,4.527c-1.465,2.537 -1.59,5.478 -0.602,8.011l-0.028,-0.017c0.998,2.539 0.878,5.489 -0.592,8.035c-1.47,2.546 -3.966,4.126 -6.664,4.53l0.029,0.017c-2.687,0.411 -5.171,1.989 -6.636,4.527c-1.465,2.537 -1.59,5.478 -0.602,8.011l-0.027,-0.015c0.997,2.537 0.877,5.487 -0.593,8.032c-2.561,4.436 -8.233,5.956 -12.67,3.395c-4.436,-2.561 -5.956,-8.234 -3.395,-12.67c1.47,-2.545 3.964,-4.124 6.661,-4.53l-0.027,-0.015c2.687,-0.411 5.171,-1.989 6.636,-4.527c1.466,-2.538 1.59,-5.478 0.602,-8.011l0.03,0.017c-0.999,-2.539 -0.879,-5.49 0.591,-8.036c1.47,-2.546 3.966,-4.125 6.663,-4.53l-0.029,-0.017c2.688,-0.411 5.172,-1.989 6.637,-4.526c1.465,-2.538 1.589,-5.478 0.602,-8.011l0.026,0.015c-0.997,-2.538 -0.876,-5.487 0.593,-8.032c1.469,-2.545 3.965,-4.124 6.662,-4.529l-0.028,-0.017c2.687,-0.411 5.171,-1.989 6.636,-4.526c1.465,-2.538 1.59,-5.478 0.602,-8.011l0.028,0.016c-0.997,-2.538 -0.878,-5.489 0.592,-8.034c2.561,-4.437 8.234,-5.956 12.67,-3.395" style="fill:#e02926;"/><path d="M115.407,55.761l18.066,-20.149l-18.066,-20.146l12.924,0l11.701,13.568l11.702,-13.568l12.927,0l-18.136,20.146l18.136,20.149l-12.927,0l-11.702,-13.511l-11.702,13.511l-12.923,0Z" style="fill:#1d1d1b;fill-rule:nonzero;"/><path d="M164.696,27.869l0.047,-1.456l0.159,-1.386l0.255,-1.296l0.36,-1.213l0.467,-1.123l0.578,-1.031l0.685,-0.938l0.79,-0.841l0.889,-0.741l1,-0.637l1.09,-0.538l1.193,-0.437l1.282,-0.335l1.372,-0.241l1.467,-0.141l1.549,-0.048l13.208,0l1.551,0.048l1.461,0.141l1.378,0.241l1.282,0.335l1.187,0.437l1.088,0.538l0.998,0.637l0.897,0.741l0.786,0.841l0.689,0.938l0.572,1.031l0.469,1.123l0.364,1.213l0.257,1.296l0.151,1.386l0.049,1.456l0,15.613l-0.049,1.446l-0.151,1.368l-0.257,1.29l-0.364,1.195l-0.469,1.113l-0.58,1.025l-0.681,0.926l-0.794,0.835l-0.895,0.729l-0.992,0.632l-1.096,0.531l-1.185,0.433l-1.284,0.332l-1.37,0.237l-1.461,0.138l-1.551,0.05l-13.208,0l-1.549,-0.05l-1.467,-0.141l-1.372,-0.242l-1.282,-0.334l-1.193,-0.435l-1.09,-0.537l-1,-0.641l-0.889,-0.737l-0.79,-0.842l-0.685,-0.937l-0.578,-1.03l-0.467,-1.124l-0.36,-1.209l-0.255,-1.299l-0.159,-1.386l-0.047,-1.461l0,-15.488ZM191.058,46.397l0.856,-0.051l0.697,-0.146l0.558,-0.228l0.426,-0.302l0.331,-0.38l0.243,-0.481l0.15,-0.604l0.056,-0.758l0,-15.543l-0.056,-0.789l-0.158,-0.631l-0.241,-0.504l-0.331,-0.395l-0.428,-0.314l-0.558,-0.237l-0.689,-0.148l-0.856,-0.056l-13.15,0l-0.86,0.056l-0.691,0.148l-0.552,0.237l-0.432,0.314l-0.333,0.395l-0.241,0.504l-0.152,0.631l-0.055,0.789l0,15.419l0.055,0.792l0.152,0.631l0.241,0.503l0.333,0.396l0.432,0.314l0.552,0.234l0.691,0.153l0.86,0.051l13.15,0Z" style="fill:#1d1d1b;fill-rule:nonzero;"/><path d="M219.436,55.761l-9.891,0l0,-27.893l0.047,-1.456l0.152,-1.386l0.255,-1.296l0.364,-1.213l0.469,-1.123l0.572,-1.031l0.685,-0.938l0.793,-0.84l0.894,-0.742l0.994,-0.637l1.095,-0.538l1.186,-0.437l1.28,-0.335l1.374,-0.241l1.467,-0.141l1.551,-0.048l6.604,0l0,9.364l-6.571,0l-0.859,0.055l-0.696,0.148l-0.551,0.237l-0.43,0.315l-0.329,0.395l-0.241,0.504l-0.161,0.63l-0.053,0.789l0,27.858Z" style="fill:#1d1d1b;fill-rule:nonzero;"/><path d="M262.96,27.98l-0.053,0.758l-0.153,0.606l-0.24,0.479l-0.332,0.38l-0.427,0.302l-0.558,0.228l-0.695,0.149l-0.856,0.051l-16.467,0l0,-3.029l0.056,-0.789l0.152,-0.631l0.241,-0.504l0.331,-0.395l0.434,-0.314l0.551,-0.237l0.69,-0.148l0.862,-0.056l13.15,0l0.856,0.056l0.687,0.148l0.559,0.237l0.428,0.314l0.332,0.395l0.24,0.504l0.159,0.631l0.053,0.789l0,0.076ZM272.804,26.412l-0.15,-1.385l-0.257,-1.296l-0.364,-1.213l-0.469,-1.123l-0.572,-1.031l-0.689,-0.938l-0.786,-0.841l-0.897,-0.741l-1,-0.637l-1.088,-0.538l-1.185,-0.437l-1.282,-0.335l-1.372,-0.241l-1.469,-0.141l-1.549,-0.048l-13.206,0l-1.551,0.048l-1.469,0.141l-1.371,0.241l-1.283,0.335l-1.191,0.437l-1.091,0.538l-0.991,0.637l-0.897,0.741l-0.792,0.841l-0.683,0.938l-0.578,1.031l-0.469,1.123l-0.358,1.213l-0.255,1.296l-0.159,1.385l-0.047,1.457l0,15.488l0.047,1.461l0.159,1.385l0.255,1.3l0.358,1.209l0.469,1.124l0.578,1.03l0.683,0.937l0.792,0.842l0.897,0.737l0.991,0.641l1.091,0.537l1.191,0.435l1.283,0.333l1.371,0.243l1.469,0.141l1.551,0.05l23.089,0l0,-9.365l-23.062,0l-0.862,-0.052l-0.69,-0.152l-0.551,-0.234l-0.434,-0.314l-0.331,-0.396l-0.241,-0.503l-0.152,-0.631l-0.056,-0.792l0,-3.03l16.496,0l1.549,-0.044l1.463,-0.141l1.378,-0.235l1.275,-0.335l1.185,-0.43l1.095,-0.531l0.994,-0.631l0.894,-0.733l0.795,-0.831l0.68,-0.93l0.581,-1.021l0.469,-1.112l0.364,-1.2l0.257,-1.285l0.15,-1.373l0.056,-1.446l0,-0.146l-0.056,-1.457Z" style="fill:#1d1d1b;fill-rule:nonzero;"/><path d="M307.093,44.115l-0.158,0.63l-0.241,0.504l-0.329,0.395l-0.428,0.314l-0.56,0.235l-0.689,0.152l-0.854,0.051l-13.15,0l-0.862,-0.051l-0.689,-0.152l-0.56,-0.235l-0.428,-0.314l-0.329,-0.395l-0.243,-0.504l-0.156,-0.63l-0.056,-0.792l0,-15.419l0.056,-0.79l0.156,-0.63l0.243,-0.504l0.329,-0.395l0.428,-0.315l0.56,-0.236l0.689,-0.149l0.862,-0.055l16.467,0l0,18.493l-0.058,0.792ZM307.151,0l0,15.466l-16.502,0l-1.551,0.049l-1.461,0.14l-1.38,0.241l-1.274,0.335l-1.193,0.438l-1.088,0.538l-1,0.636l-0.895,0.742l-0.788,0.84l-0.687,0.939l-0.572,1.03l-0.469,1.123l-0.366,1.213l-0.254,1.296l-0.154,1.386l-0.047,1.457l0,15.488l0.047,1.461l0.154,1.385l0.254,1.299l0.366,1.21l0.469,1.123l0.572,1.031l0.687,0.937l0.794,0.841l0.889,0.738l1,0.641l1.088,0.537l1.193,0.435l1.274,0.333l1.38,0.243l1.461,0.141l1.551,0.049l13.212,0l1.545,-0.049l1.467,-0.141l1.372,-0.243l1.282,-0.333l1.193,-0.438l1.088,-0.534l1,-0.641l0.889,-0.738l0.792,-0.841l0.683,-0.937l0.58,-1.031l0.469,-1.123l0.358,-1.213l0.261,-1.296l0.153,-1.385l0.047,-1.461l0,-43.357l-9.889,0Z" style="fill:#1d1d1b;fill-rule:nonzero;"/></g></svg> \ No newline at end of file diff --git a/web/assets/index.js b/web/assets/index.js index 376e606c5..4c8b90f95 100644 --- a/web/assets/index.js +++ b/web/assets/index.js @@ -2,7 +2,7 @@ var timer; var prevIndex = 0; -var slideCount = 2; +var slideCount = 4; function modifyActive(el, add) { var element = document.getElementById(el); diff --git a/web/assets/news/images/survey/dev_os.png b/web/assets/news/images/survey/dev_os.png new file mode 100644 index 000000000..088918dc3 --- /dev/null +++ b/web/assets/news/images/survey/dev_os.png Binary files differdiff --git a/web/assets/news/images/survey/do_you_use_nim.png b/web/assets/news/images/survey/do_you_use_nim.png new file mode 100644 index 000000000..257148325 --- /dev/null +++ b/web/assets/news/images/survey/do_you_use_nim.png Binary files differdiff --git a/web/assets/news/images/survey/editors.png b/web/assets/news/images/survey/editors.png new file mode 100644 index 000000000..816ad515f --- /dev/null +++ b/web/assets/news/images/survey/editors.png Binary files differdiff --git a/web/assets/news/images/survey/nim_at_work.png b/web/assets/news/images/survey/nim_at_work.png new file mode 100644 index 000000000..f00ab1a94 --- /dev/null +++ b/web/assets/news/images/survey/nim_at_work.png Binary files differdiff --git a/web/assets/news/images/survey/nim_found.png b/web/assets/news/images/survey/nim_found.png new file mode 100644 index 000000000..a0a65b813 --- /dev/null +++ b/web/assets/news/images/survey/nim_found.png Binary files differdiff --git a/web/assets/news/images/survey/nim_time.png b/web/assets/news/images/survey/nim_time.png new file mode 100644 index 000000000..23bc4a136 --- /dev/null +++ b/web/assets/news/images/survey/nim_time.png Binary files differdiff --git a/web/assets/news/images/survey/nim_time_rust.png b/web/assets/news/images/survey/nim_time_rust.png new file mode 100644 index 000000000..9b861608a --- /dev/null +++ b/web/assets/news/images/survey/nim_time_rust.png Binary files differdiff --git a/web/assets/news/images/survey/nim_versions.png b/web/assets/news/images/survey/nim_versions.png new file mode 100644 index 000000000..ba382c93d --- /dev/null +++ b/web/assets/news/images/survey/nim_versions.png Binary files differdiff --git a/web/assets/news/images/survey/planning_to_use_at_work.png b/web/assets/news/images/survey/planning_to_use_at_work.png new file mode 100644 index 000000000..be3a50467 --- /dev/null +++ b/web/assets/news/images/survey/planning_to_use_at_work.png Binary files differdiff --git a/web/assets/news/images/survey/project_size.png b/web/assets/news/images/survey/project_size.png new file mode 100644 index 000000000..ad1359d0c --- /dev/null +++ b/web/assets/news/images/survey/project_size.png Binary files differdiff --git a/web/assets/news/images/survey/project_size_nim_rust.png b/web/assets/news/images/survey/project_size_nim_rust.png new file mode 100644 index 000000000..41e3ec8b1 --- /dev/null +++ b/web/assets/news/images/survey/project_size_nim_rust.png Binary files differdiff --git a/web/assets/news/images/survey/project_size_work.png b/web/assets/news/images/survey/project_size_work.png new file mode 100644 index 000000000..fab6e52f2 --- /dev/null +++ b/web/assets/news/images/survey/project_size_work.png Binary files differdiff --git a/web/assets/news/images/survey/reliability.png b/web/assets/news/images/survey/reliability.png new file mode 100644 index 000000000..1767e9803 --- /dev/null +++ b/web/assets/news/images/survey/reliability.png Binary files differdiff --git a/web/assets/news/images/survey/target_os.png b/web/assets/news/images/survey/target_os.png new file mode 100644 index 000000000..a36915af1 --- /dev/null +++ b/web/assets/news/images/survey/target_os.png Binary files differdiff --git a/web/assets/news/images/survey/upgrades_broke_things.png b/web/assets/news/images/survey/upgrades_broke_things.png new file mode 100644 index 000000000..28a8ee3f0 --- /dev/null +++ b/web/assets/news/images/survey/upgrades_broke_things.png Binary files differdiff --git a/web/assets/niminaction/banner.jpg b/web/assets/niminaction/banner.jpg new file mode 100644 index 000000000..b2fb3efc9 --- /dev/null +++ b/web/assets/niminaction/banner.jpg Binary files differdiff --git a/web/assets/style.css b/web/assets/style.css index 529358ec9..af32fad07 100644 --- a/web/assets/style.css +++ b/web/assets/style.css @@ -316,7 +316,7 @@ pre .end { background:url("images/tabEnd.png") no-repeat left bottom; } background:url("images/glow-line2.png") no-repeat right; } #content { padding:40px 0; line-height:150%; } - #content.page { width:680px; min-height:800px; padding-left:20px; } + #content.page { width:680px; min-height:1220px; padding-left:20px; } #content h1 { font-size:20pt; letter-spacing:1px; color:rgba(0,0,0,.75); } #content h2 { font-size:16pt; letter-spacing:1px; color:rgba(0,0,0,.7); margin-top:40px; } #content p { text-align:justify; } @@ -563,3 +563,60 @@ pre .end { background:url("images/tabEnd.png") no-repeat left bottom; } border-collapse: collapse; text-align: left; border-spacing: 0px; } + +#bountysource { + width: 100%; + height: 30px; + background-color: #19975d; +} + +#bountysource a, #bountysource a:visited, #bountysource a:hover { + color: #1a1a1a; +} + +/* Current sponsors page */ + +dt { + font-size: 20pt; + clear: both; + margin-bottom: 10pt; +} + +dd.logo { + width: 200px; + min-height: 50px; + margin-bottom: 10pt; + margin-right: 20pt; + float: left; +} + +dd.logo img { + max-height: 200px; +} + +dd.this_month { + font-size: 20pt; +} + +dt a, dt a:visited, dt a:hover { + color: #1d1d1d !important; +} + +dt.level-1 { + color: #2f2f2f !important; +} + +p.lastUpdate { + font-size: 12pt; + color: #6d6d6d; +} + +/* News articles */ + +.metadata { + font-size: 12pt; + margin-bottom: 20pt; + margin-top: -16pt; + color: #4f4f4f; +} + diff --git a/web/bountysource.nim b/web/bountysource.nim new file mode 100644 index 000000000..a4f76417e --- /dev/null +++ b/web/bountysource.nim @@ -0,0 +1,144 @@ +# Based on bountysource.cr located at https://github.com/crystal-lang/crystal-website/blob/master/scripts/bountysource.cr +import httpclient, asyncdispatch, json, strutils, os, strtabs, sequtils, future, + algorithm, times + +type + BountySource = ref object + client: AsyncHttpClient + team: string + + Sponsor = object + name, url, logo: string + amount, allTime: float + since: TimeInfo + +const + team = "nim" + apiUrl = "https://api.bountysource.com" + githubApiUrl = "https://api.github.com" + +proc newBountySource(team, token: string): BountySource = + result = BountySource( + client: newAsyncHttpClient(userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36"), + team: team + ) + + # Set up headers + result.client.headers["Accept"] = "application/vnd.bountysource+json; version=2" + result.client.headers["Authorization"] = "token " & token + result.client.headers["Referer"] = "https://salt.bountysource.com/teams/nim/admin/supporters" + result.client.headers["Origin"] = "https://salt.bountysource.com/" + +proc getSupportLevels(self: BountySource): Future[JsonNode] {.async.} = + let response = await self.client.get(apiUrl & + "/support_levels?supporters_for_team=" & self.team) + doAssert response.status.startsWith($Http200) # TODO: There should be a == + return parseJson(response.body) + +proc getSupporters(self: BountySource): Future[JsonNode] {.async.} = + let response = await self.client.get(apiUrl & + "/supporters?order=monthly&per_page=200&team_slug=" & self.team) + doAssert response.status.startsWith($Http200) + return parseJson(response.body) + +proc getGithubUser(username: string): Future[JsonNode] {.async.} = + let client = newAsyncHttpClient() + let response = await client.get(githubApiUrl & "/users/" & username) + if response.status.startsWith($Http200): + return parseJson(response.body) + else: + return nil + +proc processLevels(supportLevels: JsonNode) = + var before = supportLevels.elems.len + supportLevels.elems.keepIf( + item => item["status"].getStr == "active" and + item["owner"]["display_name"].getStr != "Anonymous" + ) + echo("Found ", before - supportLevels.elems.len, " sponsors that cancelled or didn't pay.") + echo("Found ", supportLevels.elems.len, " active sponsors!") + + supportLevels.elems.sort( + (x, y) => cmp(y["amount"].getFNum, x["amount"].getFNum) + ) + +proc getSupporter(supporters: JsonNode, displayName: string): JsonNode = + for supporter in supporters: + if supporter["display_name"].getStr == displayName: + return supporter + doAssert false + +proc quote(text: string): string = + if {' ', ','} in text: + return "\"" & text & "\"" + else: + return text + +proc getLevel(amount: float): int = + result = 0 + const levels = [250, 150, 75, 25, 10, 5, 1] + for i in levels: + if amount.int <= i: + result = i + +proc writeCsv(sponsors: seq[Sponsor]) = + var csv = "" + csv.add "logo, name, url, this_month, all_time, since, level\n" + for sponsor in sponsors: + csv.add "$#,$#,$#,$#,$#,$#,$#\n" % [ + sponsor.logo.quote, sponsor.name.quote, + sponsor.url.quote, $sponsor.amount.int, + $sponsor.allTime.int, sponsor.since.format("MMM d, yyyy").quote, + $sponsor.amount.getLevel + ] + writeFile("sponsors.new.csv", csv) + +when isMainModule: + if paramCount() == 0: + quit("You need to specify the BountySource access token on the command\n" & + "line, you can find it by going onto https://www.bountysource.com/people/25278-dom96\n" & + "and looking at your browser's network inspector tab to see the token being\n" & + "sent to api.bountysource.com") + + let token = paramStr(1) + let bountysource = newBountySource(team, token) + + echo("Getting sponsors...") + let supporters = waitFor bountysource.getSupporters() + + var supportLevels = waitFor bountysource.getSupportLevels() + processLevels(supportLevels) + + echo("Generating sponsors list... (please be patient)") + var sponsors: seq[Sponsor] = @[] + for supportLevel in supportLevels: + let name = supportLevel["owner"]["display_name"].getStr + var url = "" + let ghUser = waitFor getGithubUser(name) + if not ghUser.isNil: + if ghUser["blog"].kind != JNull: + url = ghUser["blog"].getStr + else: + url = ghUser["html_url"].getStr + + if url.len > 0 and not url.startsWith("http"): + url = "http://" & url + + let amount = supportLevel["amount"].getFNum + # Only show URL when user donated at least $5. + if amount < 5: + url = "" + + let supporter = getSupporter(supporters, supportLevel["owner"]["display_name"].getStr) + var logo = "" + if amount >= 75: + discard # TODO + + sponsors.add(Sponsor(name: name, url: url, logo: logo, amount: amount, + allTime: supporter["alltime_amount"].getFNum(), + since: parse(supporter["created_at"].getStr, "yyyy-MM-dd'T'hh:mm:ss") + )) + + echo("Generated ", sponsors.len, " sponsors") + writeCsv(sponsors) + echo("Written csv file to sponsors.new.csv") diff --git a/web/community.txt b/web/community.rst index 9ce87b546..9ce87b546 100644 --- a/web/community.txt +++ b/web/community.rst diff --git a/web/documentation.txt b/web/documentation.rst index ec33d0827..ec33d0827 100644 --- a/web/documentation.txt +++ b/web/documentation.rst diff --git a/web/download.rst b/web/download.rst new file mode 100644 index 000000000..6593a928c --- /dev/null +++ b/web/download.rst @@ -0,0 +1,98 @@ +Download the compiler +===================== + +You can download the latest version of the Nim compiler here. + +**Note:** The Nim compiler requires a C compiler to compile software. On +Windows we recommend that you use +`Mingw-w64 <http://mingw-w64.sourceforge.net/>`_. GCC is recommended on Linux +and Clang on Mac. + + +Binaries +-------- + +Unfortunately, right now we only provide binaries for Windows. You can download +an installer for both 32 bit and 64 bit versions of Windows below. + +* | 32 bit: `nim-0.14.2_x32.exe <download/nim-0.14.2_x32.exe>`_ + | SHA-256 ca2de37759006d95db98732083a6fab20151bb9819186af2fa29d41884df78c9 +* | 64 bit: `nim-0.14.2_x64.exe <download/nim-0.14.2_x64.exe>`_ + | SHA-256 1fec054d3a5f54c0a67a40db615bb9ecb1d56413b19e324244110713bd4337d1 + +These installers also include Aporia, Nimble and other useful Nim tools to get +you started with Nim development! + +Installation based on generated C code +-------------------------------------- + +This installation method is the preferred way for Linux, Mac OS X, and other Unix +like systems. Binary packages may be provided later. + + +Firstly, download this archive: + +* | `nim-0.14.2.tar.xz (4.5MB) <download/nim-0.14.2.tar.xz>`_ + | SHA-256 8f8d38d70ed57164795fc55e19de4c11488fcd31dbe42094e44a92a23e3f5e92 + +Extract the archive. Then copy the extracted files into your chosen installation +directory, ideally somewhere in your home directory. +For example: ``~/programs/nim``. + +Now open a terminal and follow these instructions: + +* ``cd`` into your installation directory, for example by executing +``cd ~/programs/nim``. +* run ``sh build.sh``. +* Add ``$your_install_dir/bin`` to your PATH. + +After restarting your terminal, you should be able to run ``nim -v`` +which should show you the version of Nim you just installed. + +There are other ways to install Nim (like using the ``install.sh`` script), +but these tend to cause more problems. + + +Bleeding edge installation from GitHub +-------------------------------------- + +`GitHub <http://github.com/nim-lang/nim>`_ is where Nim's development takes +place. You may wish to grab the latest development version of Nim, because +sometimes bug fixes and new features may not have made it to an official +release yet. In those circumstances you are better off grabbing the +current development branch. + +You will also need to do this if you would like to contribute to Nim. + +Before you download the code, open a new terminal and ``cd`` into the +directory where you would like the download to take place. + +The following commands can be used to download the current development branch +and then to build it:: + + git clone git://github.com/nim-lang/Nim.git + cd Nim + git clone --depth 1 git://github.com/nim-lang/csources + cd csources && sh build.sh + cd .. + bin/nim c koch + ./koch boot -d:release + +You should then add the ``./bin`` (make sure to expand this into an +absolute path) directory to your ``PATH``. + + +Docker Hub +---------- + +The `official Docker images <https://hub.docker.com/r/nimlang/nim/>`_ +are published Docker Hub and include the compiler and Nimble. There are images +for standalone scripts as well as Nimble packages. + +Get the latest stable image:: + + docker pull nimlang/nim + +The latest development version:: + + docker pull nimlang/nim:devel diff --git a/web/download.txt b/web/download.txt deleted file mode 100644 index cffad4ac9..000000000 --- a/web/download.txt +++ /dev/null @@ -1,56 +0,0 @@ -Download the compiler -===================== - -You can download the latest version of the Nim compiler here. - -**Note:** The Nim compiler requires a C compiler to compile software. On -Windows we recommend that you use -`Mingw-w64 <http://mingw-w64.sourceforge.net/>`_. GCC is recommended on Linux -and Clang on Mac. - - -Binaries --------- - -Unfortunately for now we only provide builds for Windows. -* 32 bit: `nim-0.13.0_x32.exe <download/nim-0.13.0_x32.exe>`_ -* 64 bit: `nim-0.13.0_x64.exe <download/nim-0.13.0_x64.exe>`_ - - -Installation based on generated C code --------------------------------------- - -This installation method is the preferred way for Linux, Mac OS X, and other Unix -like systems. Binary packages may be provided later. - - -Download this: - -* `nim-0.13.0.tar.xz (3.1MB) <download/nim-0.13.0.tar.xz>`_ - -Extract the file and follow these instructions: - -* sh build.sh -* Add ``$your_install_dir/bin`` to your PATH. - -There are other ways to install Nim (like using the ``install.sh`` script), -but these tend to cause more problems. - - -Installation from github ------------------------- - -Use the following commands to build the compiler from source. -Change the branch to suit your needs:: - - git clone -b master git://github.com/nim-lang/Nim.git - cd Nim - git clone -b master --depth 1 git://github.com/nim-lang/csources - cd csources && sh build.sh - cd .. - bin/nim c koch - ./koch boot -d:release - -The ``master`` branch always contains the latest stable version of the compiler. -If you want bleeding edge then switch to the ``devel`` branch and follow -the same instructions outlined above. diff --git a/web/index.txt b/web/index.rst index 506453423..506453423 100644 --- a/web/index.txt +++ b/web/index.rst diff --git a/web/learn.txt b/web/learn.rst index c0b583429..c0b583429 100644 --- a/web/learn.txt +++ b/web/learn.rst diff --git a/web/links.txt b/web/links.rst index e69de29bb..e69de29bb 100644 --- a/web/links.txt +++ b/web/links.rst diff --git a/web/news.rst b/web/news.rst new file mode 100644 index 000000000..95822a459 --- /dev/null +++ b/web/news.rst @@ -0,0 +1,126 @@ +==== +News +==== + +`2016-08-06 BountySource Update: The Road to v1.0 <news/2016_08_06_bountysource_update_the_road_to_v10.html>`_ +=================================== + +`2016-06-23 Launching the 2016 Nim community survey <news/2016_06_23_launching_the_2016_nim_community_survey.html>`_ +=================================== + +`2016-06-11 Version 0.14.2 released <news/2016_06_11_version_0_14_2_released.html>`_ +=================================== + +`2016-06-07 Version 0.14.0 released <news/2016_06_07_version_0_14_0_released.html>`_ +=================================== + +`2016-06-04 Meet our BountySource sponsors <news/2016_06_04_meet_our_bountysource_sponsors.html>`_ +=================================== + +`2016-01-27 Nim in Action is now available! <news/2016_01_27_nim_in_action_is_now_available.html>`_ +================================== + +`2016-01-18 Version 0.13.0 released <news/2016_01_18_version_0_13_0_released.html>`_ +================================== + +`2016-01-18 Andreas Rumpf's talk at OSCON Amsterdam <news/2016_01_18_andreas_rumpfs_talk_at_oscon_amsterdam.html>`_ +================================================== + +`2015-10-27 Version 0.12.0 released <news/2015_10_27_version_0_12_0_released.html>`_ +================================== + +`2015-10-16 First Nim conference <news/2015_10_16_first_nim_conference.html>`_ +=============================== + +`2015-05-04 Version 0.11.2 released <news/2015_05_04_version_0_11_2_released.html>`_ +================================== + +`2015-04-30 Version 0.11.0 released <news/2015_04_30_version_0_11_0_released.html>`_ +================================== + +`2014-12-29 Version 0.10.2 released <news/2014_12_29_version_0_10_2_released.html>`_ +================================== + + +`2014-10-19 Version 0.9.6 released <news/2014_10_19_version_0_9_6_released.html>`_ +================================= + + +`2014-04-21 Version 0.9.4 released <news/2014_04_21_version_0_9_4_released.html>`_ +================================= + + +`2014-02-11 Nimrod Featured in Dr. Dobb's Journal <news/2014_02_11_nimrod_featured_in_dr_dobbs_journal.html>`_ +================================================ + + +`2014-01-15 Andreas Rumpf's talk on Nimrod at Strange Loop 2013 is now online <news/2014_01_15_andreas_rumpfs_talk_on_nimrod.html>`_ +============================================================================ + + +`2013-05-20 New website design! <news/2013_05_20_new_website_design.html>`_ +============================== + + + +`2013-05-20 Version 0.9.2 released <news/2013_05_20_version_0_9_2_released.html>`_ +================================= + + + +`2012-09-23 Version 0.9.0 released <news/2012_09_23_version_0_9_0_released.html>`_ +================================= + + + +`2012-02-09 Version 0.8.14 released <news/2012_02_09_version_0_8_14_released.html>`_ +================================== + + + +`2011-07-10 Version 0.8.12 released <news/2011_07_10_version_0_8_12_released.html>`_ +================================== + + +`2010-10-20 Version 0.8.10 released <news/2010_10_20_version_0_8_10_released.html>`_ +================================== + + + +`2010-03-14 Version 0.8.8 released <news/2010_03_14_version_0_8_8_released.html>`_ +================================= + + +`2009-12-21 Version 0.8.6 released <news/2009_12_21_version_0_8_6_released.html>`_ +================================= + + +2009-10-21 Version 0.8.2 released +================================= + + +2009-09-12 Version 0.8.0 released +================================= + + +2009-06-08 Version 0.7.10 released +================================== + + +2009-05-08 Version 0.7.8 released +================================= + + +2009-04-22 Version 0.7.6 released +================================= + + +2008-11-16 Version 0.7.0 released +================================= + + +2008-08-22 Version 0.6.0 released +================================= + +Nimrod version 0.6.0 has been released! +**This is the first version of the compiler that is able to compile itself!** diff --git a/web/news.txt b/web/news.txt deleted file mode 100644 index e818ed66e..000000000 --- a/web/news.txt +++ /dev/null @@ -1,2388 +0,0 @@ -==== -News -==== - -2016-XX-XX Version 0.13.1 released -================================== - -Changes affecting backwards compatibility ------------------------------------------ - -- ``--out`` and ``--nimcache`` command line arguments are now relative to - current directory. Previously they were relative to project directory. -- The json module now stores the name/value pairs in objects internally as a - hash table of type ``fields*: Table[string, JsonNode]`` instead of a - sequence. This means that order is no longer preserved. When using the - ``table.mpairs`` iterator only the returned values can be modified, no - longer the keys. -- The deprecated Nim shebang notation ``#!`` was removed from the language. Use ``#?`` instead. -- The ``using`` statement now means something completely different. You can use the - new experimental ``this`` pragma to achieve a similar effect to what the old ``using`` statement tried to achieve. -- Typeless parameters have been removed from the language since it would - clash with ``using``. - - -Library Additions ------------------ - -- The rlocks module has been added providing reentrant lock synchronization - primitive. -- A generic "sink operator" written as ``&=`` has been added to the ``system`` and the ``net`` modules. - - -Compiler Additions ------------------- - -- Added a new ``--noCppExceptions`` switch that allows to use default exception - handling (no ``throw`` or ``try``/``catch`` generated) when compiling to C++ - code. - -Language Additions ------------------- - -- Nim now supports a ``.this`` pragma for more notational convenience. -- Nim now supports a different ``using`` statement for more convenience. -- Nim now supports ``partial`` object declarations to mitigate the problems - that arise when types are mutually dependent and yet should be kept in - different modules. - - -2016-01-27 Nim in Action is now available! -========================================== - -.. raw::html - - <a href="https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81"> - <img src="assets/niminaction/banner.jpg" alt="New in Manning Early Access Program: Nim in Action!" width="682"/> - </a> - -We are proud to announce that *Nim in Action*, a book about the Nim programming -language, is now available! - -The book is available at this URL: -`https://www.manning.com/books/nim-in-action <https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81>`_ - -The first three chapters are available for download -as an eBook through Manning's Early Access program. You can download a free -sample of the book containing the first chapter as well! - -*Nim in Action* is currently being written and is expected to be completed by -Summer 2016. If you purchase the eBook you will start receiving new chapters -as they become available. You can also purchase the printed book together with -the eBook for a slightly higher price. - -If you do read the book, even if it's just the first chapter, then please share -any comments, suggestions and questions on the -`Nim forum <http://forum.nim-lang.org/t/1978>`_ or in -Manning's own `Author Online forum! <https://forums.manning.com/forums/nim-in-action>`_ - - -2016-01-18 Version 0.13.0 released -================================== - -Once again we are proud to announce the latest release of the Nim compiler -and related tools. This release comes just 3 months after the last -release! - -A new version of Nimble which depends on this release, has also been -released. See `this <http://forum.nim-lang.org/t/1912>`_ forum thread for -more information about the Nimble release. - -This release of Nim includes over 116 bug fixes, many of which are related -to closures. The lambda lifting algorithm in the compiler has been completely -rewritten, and some changes have been made to the semantics of closures in -Nim as a result. These changes may affect backwards compatibility and are all -described in the section below. - -With this release, we are one step closer to Nim version 1.0. -The 1.0 release will be a big milestone for Nim, because after that version -is released there will be no more breaking changes made to the language -or the standard library. - -That being said, the next release will likely be Nim 0.14. It will focus on -improvements to the GC and concurrency. We will in particular be looking at -ways to add multi-core support to async await. Standard library improvements -are also on our roadmap but may not make it for Nim 0.14. - -As always you can download the latest version of Nim from the -`download <download.html>`_ page. - -Happy coding! - -Changes affecting backwards compatibility ------------------------------------------ - -- ``macros.newLit`` for ``bool`` now produces false/true symbols which - actually work with the bool datatype. -- When compiling to JS: ``Node``, ``NodeType`` and ``Document`` are no longer - defined. Use the types defined in ``dom.nim`` instead. -- The check ``x is iterator`` (used for instance in concepts) was always a - weird special case (you could not use ``x is proc``) and was removed from - the language. -- Top level routines cannot have the calling convention ``closure`` - anymore. -- The ``redis`` module has been moved out of the standard library. It can - now be installed via Nimble and is located here: - https://github.com/nim-lang/redis -- ``math.RunningStat`` and its associated procs have been moved from - the ``math`` module to a new ``stats`` module. - - -Syntax changes -~~~~~~~~~~~~~~ - -The parser now considers leading whitespace in front of operators -to determine if an operator is used in prefix or infix position. -This means that finally ``echo $foo`` is parsed as people expect, -which is as ``echo($foo)``. It used to be parsed as ``(echo) $ (foo)``. - -``echo $ foo`` continues to be parsed as ``(echo) $ (foo)``. - -This also means that ``-1`` is always parsed as prefix operator so -code like ``0..kArraySize div 2 -1`` needs to be changed to -``0..kArraySize div 2 - 1``. - -This release also adds multi-line comments to Nim. The syntax for them is: -``#[ comment here ]#``. For more details read the section of -the `manual <docs/manual.html#lexical-analysis-multiline-comments>`_. - -Iterator changes -~~~~~~~~~~~~~~~~ - -Implicit return type inference for iterators has been removed from the language. The following used to work: - -.. code-block:: nim - iterator it = - yield 7 - -This was a strange special case and has been removed. Now you need to write it like so which is consistent with procs: - -.. code-block:: nim - iterator it: auto = - yield 7 - - -Closure changes -~~~~~~~~~~~~~~~ - -The semantics of closures changed: Capturing variables that are in loops do not produce a new environment. Nim closures behave like JavaScript closures now. - -The following used to work as the environment creation used to be attached to the loop body: - -.. code-block:: nim - - proc outer = - var s: seq[proc(): int {.closure.}] = @[] - for i in 0 ..< 30: - let ii = i - s.add(proc(): int = return ii*ii) - -This behaviour has changed in 0.13.0 and now needs to be written as: - -.. code-block:: nim - - proc outer = - var s: seq[proc(): int {.closure.}] = @[] - for i in 0 ..< 30: - (proc () = - let ii = i - s.add(proc(): int = return ii*ii))() - -The reason is that environment creations are now only performed once -per proc call. This change is subtle and unfortunate, but: - -1. Affects almost no code out there. -2. Is easier to implement and we are at a point in Nim's development process where simple+stable wins over perfect-in-theory+unstable-in-practice. -3. Implies programmers are more in control of where memory is allocated which is beneficial for a systems programming language. - -Bugfixes --------- - -The list below has been generated based on the commits in Nim's git -repository. As such it lists only the issues which have been closed -via a commit, for a full list see -`this link on Github <https://github.com/nim-lang/Nim/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222015-10-27+..+2016-01-19%22+>`_. - -- Fixed "Generic arguments cannot be used in templates (raising undeclared identifier)" - (`#3498 <https://github.com/nim-lang/Nim/issues/3498>`_) -- Fixed "multimethods: Error: internal error: cgmeth.genConv" - (`#3550 <https://github.com/nim-lang/Nim/issues/3550>`_) -- Fixed "nimscript - SIGSEGV in except block" - (`#3546 <https://github.com/nim-lang/Nim/issues/3546>`_) -- Fixed "Bool literals in macros do not work." - (`#3541 <https://github.com/nim-lang/Nim/issues/3541>`_) -- Fixed "Docs: nativesocket.html - 404" - (`#3582 <https://github.com/nim-lang/Nim/issues/3582>`_) -- Fixed ""not nil" return types never trigger an error or warning" - (`#2285 <https://github.com/nim-lang/Nim/issues/2285>`_) -- Fixed "No warning or error is raised even if not nil is specified " - (`#3222 <https://github.com/nim-lang/Nim/issues/3222>`_) -- Fixed "Incorrect fsmonitor add() filter logic" - (`#3611 <https://github.com/nim-lang/Nim/issues/3611>`_) -- Fixed ""nimble install nimsuggest" failed" - (`#3622 <https://github.com/nim-lang/Nim/issues/3622>`_) -- Fixed "compile time `excl ` cause SIGSEGV" - (`#3639 <https://github.com/nim-lang/Nim/issues/3639>`_) -- Fixed "Unable to echo unsigned ints at compile-time" - (`#2514 <https://github.com/nim-lang/Nim/issues/2514>`_) -- Fixed "Nested closure iterator produces internal error" - (`#1725 <https://github.com/nim-lang/Nim/issues/1725>`_) -- Fixed "C Error on walkDirRec closure" - (`#3636 <https://github.com/nim-lang/Nim/issues/3636>`_) -- Fixed "Error in generated c code" - (`#3201 <https://github.com/nim-lang/Nim/issues/3201>`_) -- Fixed "C Compile-time error with generic proc type." - (`#2659 <https://github.com/nim-lang/Nim/issues/2659>`_) -- Fixed "ICE dereferencing array pointer" - (`#2240 <https://github.com/nim-lang/Nim/issues/2240>`_) -- Fixed "Lambda lifting crash" - (`#2007 <https://github.com/nim-lang/Nim/issues/2007>`_) -- Fixed "Can't reference outer variables from a closure in an iterator" - (`#2604 <https://github.com/nim-lang/Nim/issues/2604>`_) -- Fixed "M&S collector breaks with nested for loops." - (`#603 <https://github.com/nim-lang/Nim/issues/603>`_) -- Fixed "Regression: bad C codegen" - (`#3723 <https://github.com/nim-lang/Nim/issues/3723>`_) -- Fixed "JS backend - handle bool type in case statement" - (`#3722 <https://github.com/nim-lang/Nim/issues/3722>`_) -- Fixed "linenoise compilation with cpp" - (`#3720 <https://github.com/nim-lang/Nim/issues/3720>`_) -- Fixed "(???,???) duplicate case label" - (`#3665 <https://github.com/nim-lang/Nim/issues/3665>`_) -- Fixed "linenoise compilation with cpp" - (`#3720 <https://github.com/nim-lang/Nim/issues/3720>`_) -- Fixed "Update list of backward incompatibilities for Nim 0.12.0 in the main site" - (`#3689 <https://github.com/nim-lang/Nim/issues/3689>`_) -- Fixed "Can't compile nimble with latest devel - codegen bug" - (`#3730 <https://github.com/nim-lang/Nim/issues/3730>`_) - - -2016-01-18 Andreas Rumpf's talk at OSCON Amsterdam -================================================== - -In case you have missed it, here is Andreas' Nim: An Overview talk at -OSCON Amsterdam. - -.. raw:: html - - <iframe width="560" height="315" src="https://www.youtube.com/embed/4rJEBs_Nnaw" frameborder="0" allowfullscreen></iframe> - -2015-10-27 Version 0.12.0 released -================================== - -The Nim community of developers is proud to announce the new version of the -Nim compiler. This has been a long time coming as the last release has been -made over 5 months ago! - -This release includes some changes which affect backwards compatibility, -one major change is that now the hash table ``[]`` operators now raise a -``KeyError`` exception when the key does not exist. - -Some of the more exciting new features include: the ability to unpack tuples -in any assignment context, the introduction of `NimScript <docs/nims.html>`_, -and improvements to the type inference of lambdas. - -There are of course many many many bug fixes included with this release. -We are getting closer and closer to a 1.0 release and are hoping that only -a few 0.x releases will be necessary before we are happy to release version 1.0. - -As always you can download the latest version of Nim from the -`download <download.html>`_ page. - -For a more detailed list of changes look below. Some of the upcoming breaking -changes are also documented in this forum -`thread <http://forum.nim-lang.org/t/1708>`_. - -Changes affecting backwards compatibility ------------------------------------------ -- The regular expression modules, ``re`` and ``nre`` now depend on version - 8.36 of PCRE. If you have an older version you may see a message similar - to ``could not import: pcre_free_study`` output when you start your - program. See `this issue <https://github.com/docopt/docopt.nim/issues/13>`_ - for more information. -- ``tables.[]``, ``strtabs.[]``, ``critbits.[]`` **now raise** - the ``KeyError`` **exception when the key does not exist**! Use the - new ``getOrDefault`` instead to get the old behaviour. Compile all your - code with ``-d:nimTableGet`` to get a listing of where your code - uses ``[]``! -- The ``rawsockets`` module has been renamed to ``nativesockets`` to avoid - confusion with TCP/IP raw sockets, so ``newNativeSocket`` should be used - instead of ``newRawSocket``. -- The ``miliseconds`` property of ``times.TimeInterval`` is now ``milliseconds``. - Code accessing that property is deprecated and code using ``miliseconds`` - during object initialization or as a named parameter of ``initInterval()`` - will need to be updated. -- ``std.logging`` functions no longer do formatting and semantically treat - their arguments just like ``echo`` does. Affected functions: ``log``, - ``debug``, ``info``, ``warn``, ``error``, ``fatal``. Custom subtypes of - ``Logger`` also need to be adjusted accordingly. -- Floating point numbers can now look like ``2d`` (float64) - and ``2f`` (float32) which means imports like ``import scene/2d/sprite`` - do not work anymore. Instead quotes have to be - used: ``import "scene/2d/sprite"``. The former code never was valid Nim. -- The Windows API wrapper (``windows.nim``) is now not part of the official - distribution anymore. Instead use the ``oldwinapi`` Nimble package. -- There is now a clear distinction between ``--os:standalone`` - and ``--gc:none``. So if you use ``--os:standalone`` ensure you also use - ``--gc:none``. ``--os:standalone`` without ``--gc:none`` is now a version - that doesn't depend on any OS but includes the GC. However this version - is currently untested! -- All procedures which construct a ``Socket``/``AsyncSocket`` now need to - specify the socket domain, type and protocol. The param name - ``typ: SockType`` (in ``newSocket``/``newAsyncSocket`` procs) was also - renamed to ``sockType``. The param ``af`` in the ``connect`` procs was - removed. This affects ``asyncnet``, ``asyncdispatch``, ``net``, and - ``rawsockets``. -- ``varargs[typed]`` and ``varargs[untyped]`` have been refined and now work - as expected. However ``varargs[untyped]`` is not an alias anymore for - ``varargs[expr]``. So if your code breaks for ``varargs[untyped]``, use - ``varargs[expr]`` instead. The same applies to ``varargs[typed]`` vs - ``varargs[stmt]``. -- ``sequtils.delete`` doesn't take confusing default arguments anymore. -- ``system.free`` was an error-prone alias to ``system.dealloc`` and has - been removed. -- ``macros.high`` never worked and the manual says ``high`` cannot be - overloaded, so we removed it with no deprecation cycle. -- To use the ``parallel`` statement you now have to - use the ``--experimental`` mode. -- Toplevel procs of calling convention ``closure`` never worked reliably - and are now deprecated and will be removed from the language. Instead you - have to insert type conversions - like ``(proc (a, b: int) {.closure.})(myToplevelProc)`` if necessary. -- The modules ``libffi``, ``sdl``, ``windows``, ``zipfiles``, ``libzip``, - ``zlib``, ``zzip``, ``dialogs``, ``expat``, ``graphics``, ``libcurl``, - ``sphinx`` have been moved out of the stdlib and are Nimble packages now. -- The constant fights between 32 and 64 bit DLLs on Windows have been put to - an end: The standard distribution now ships with 32 and 64 bit versions - of all the DLLs the standard library needs. This means that the following - DLLs are now split into 32 and 64 versions: - - * ``pcre.dll``: Split into ``pcre32.dll`` and ``pcre64.dll``. - * ``pdcurses.dll``: Split into ``pdcurses32.dll`` and ``pdcurses64.dll``. - * ``sqlite3.dll``: Split into ``sqlite3_32.dll`` and ``sqlite3_64.dll``. - * ``ssleay32.dll``: Split into ``ssleay32.dll`` and ``ssleay64.dll``. - * ``libeay32.dll``: Split into ``libeay32.dll`` and ``libeay64.dll``. - - Compile with ``-d:nimOldDLLs`` to make the stdlib use the old DLL names. -- Nim VM now treats objects as ``nkObjConstr`` nodes, and not ``nkPar`` nodes - as it was previously. Macros that generate ``nkPar`` nodes when object is - expected are likely to break. Macros that expect ``nkPar`` nodes to which - objects are passed are likely to break as well. -- Base methods now need to be annotated with the ``base`` pragma. This makes - multi methods less error-prone to use with the effect system. -- Nim's parser directive ``#!`` is now ``#?`` in order to produce no conflicts - with Unix's ``#!``. -- An implicit return type for an iterator is now deprecated. Use ``auto`` if - you want more type inference. -- The type ``auto`` is now a "multi-bind" metatype, so the following compiles: - - .. code-block:: nim - proc f(x, y: auto): auto = - result = $x & y - - echo f(0, "abc") -- The ``ftpclient`` module is now deprecated in favour of the - ``asyncftpclient`` module. -- In sequtils.nim renamed ``repeat`` function to ``cycle`` (concatenating - a sequence by itself the given times), and also introduced ``repeat``, - which repeats an element the given times. -- The function ``map`` is moved to sequtils.nim. The inplace ``map`` version - is renamed to ``apply``. -- The template ``mapIt`` now doesn't require the result's type parameter. - Also the inplace ``mapIt`` is renamed to ``apply``. -- The compiler is now stricter with what is allowed as a case object - discriminator. The following code used to compile but was not supported - completely and so now fails: - -.. code-block:: nim - type - DataType* {.pure.} = enum - Char = 1, - Int8 = 2, - Int16 = 3, - Int32 = 4, - Int64 = 5, - Float32 = 6, - Float64 = 7 - - DataSeq* = object - case kind* : DataType - of DataType.Char: charSeq* : seq[char] - of DataType.Int8: int8Seq* : seq[int8] - of DataType.Int16: int16Seq* : seq[int16] - of DataType.Int32: int32Seq* : seq[int32] - of DataType.Int64: int64Seq* : seq[int64] - of DataType.Float32: float32Seq* : seq[float32] - of DataType.Float64: float64Seq* : seq[float64] - - length* : int - - - -Library Additions ------------------ - -- The nre module has been added, providing a better interface to PCRE than re. -- The ``expandSymlink`` proc has been added to the ``os`` module. -- The ``tailDir`` proc has been added to the ``os`` module. -- Define ``nimPinToCpu`` to make the ``threadpool`` use explicit thread - affinities. This can speed up or slow down the thread pool; it's up to you - to benchmark it. -- ``strutils.formatFloat`` and ``formatBiggestFloat`` do not depend on the C - locale anymore and now take an optional ``decimalSep = '.'`` parameter. -- Added ``unicode.lastRune``, ``unicode.graphemeLen``. - - -Compiler Additions ------------------- - -- The compiler now supports a new configuration system based on - `NimScript <docs/nims.html>`_. -- The compiler finally considers symbol binding rules in templates and - generics for overloaded ``[]``, ``[]=``, ``{}``, ``{}=`` operators - (issue `#2599 <https://github.com/nim-lang/Nim/issues/2599>`_). -- The compiler now supports a `bitsize pragma <docs/manual.html#pragmas-bitsize-pragma>`_ - for constructing bitfields. -- Added a new ``--reportConceptFailures`` switch for better debugging of - concept related type mismatches. This can also be used to debug - ``system.compiles`` failures. - - -Language Additions ------------------- - -- ``system.unsafeAddr`` can be used to access the address of a ``let`` - variable or parameter for C interoperability. Since technically this - makes parameters and ``let`` variables mutable, it is considered even more - unsafe than the ordinary ``addr`` builtin. -- Added ``macros.getImpl`` that can be used to access the implementation of - a routine or a constant. This allows for example for user-defined inlining - of function calls. -- Tuple unpacking finally works in a non-var/let context: ``(x, y) = f()`` - is allowed. Note that this doesn't declare ``x`` and ``y`` variables, for - this ``let (x, y) = f()`` still needs to be used. -- ``when nimvm`` can now be used for compiletime versions of some code - sections. Click `here <docs/manual.html#when-nimvm-statement>`_ for details. -- Usage of the type ``NimNode`` in a proc now implicitly annotates the proc - with ``.compileTime``. This means generics work much better for ``NimNode``. - - -Bugfixes --------- -- Fixed "Compiler internal error on iterator it(T: typedesc[Base]) called with it(Child), where Child = object of Base" - (`#2662 <https://github.com/Araq/Nim/issues/2662>`_) -- Fixed "repr() misses base object field in 2nd level derived object" - (`#2749 <https://github.com/Araq/Nim/issues/2749>`_) -- Fixed "nimsuggest doesn't work more than once on the non-main file" - (`#2694 <https://github.com/Araq/Nim/issues/2694>`_) -- Fixed "JS Codegen. Passing arguments by var in certain cases leads to invalid JS." - (`#2798 <https://github.com/Araq/Nim/issues/2798>`_) -- Fixed ""check" proc in unittest.nim prevents the propagation of changes to var parameters." - (`#964 <https://github.com/Araq/Nim/issues/964>`_) -- Fixed "Excessive letters in integer literals are not an error" - (`#2523 <https://github.com/Araq/Nim/issues/2523>`_) -- Fixed "Unicode dashes as "lisp'ish" alternative to hump and snake notation" - (`#2811 <https://github.com/Araq/Nim/issues/2811>`_) -- Fixed "Bad error message when trying to construct an object incorrectly" - (`#2584 <https://github.com/Araq/Nim/issues/2584>`_) -- Fixed "Determination of GC safety of globals is broken " - (`#2854 <https://github.com/Araq/Nim/issues/2854>`_) -- Fixed "v2 gc crashes compiler" - (`#2687 <https://github.com/Araq/Nim/issues/2687>`_) -- Fixed "Compile error using object in const array" - (`#2774 <https://github.com/Araq/Nim/issues/2774>`_) -- Fixed "httpclient async requests with method httpPOST isn't sending Content-Length header" - (`#2884 <https://github.com/Araq/Nim/issues/2884>`_) -- Fixed "Streams module not working with JS backend" - (`#2148 <https://github.com/Araq/Nim/issues/2148>`_) -- Fixed "Sign of certain short constants is wrong" - (`#1179 <https://github.com/Araq/Nim/issues/1179>`_) -- Fixed "Symlinks to directories reported as symlinks to files" - (`#1985 <https://github.com/Araq/Nim/issues/1985>`_) -- Fixed "64-bit literals broken on x86" - (`#2909 <https://github.com/Araq/Nim/issues/2909>`_) -- Fixed "import broken for certain names" - (`#2904 <https://github.com/Araq/Nim/issues/2904>`_) -- Fixed "Invalid UTF-8 strings in JavaScript" - (`#2917 <https://github.com/Araq/Nim/issues/2917>`_) -- Fixed "[JS][Codegen] Initialising object doesn't create unmentioned fields." - - (`#2617 <https://github.com/Araq/Nim/issues/2617>`_) -- Fixed "Table returned from proc computed at compile time is missing keys:" - (`#2297 <https://github.com/Araq/Nim/issues/2297>`_) -- Fixed "Clarify copyright status for some files" - (`#2949 <https://github.com/Araq/Nim/issues/2949>`_) -- Fixed "math.nim: trigonometry: radians to degrees conversion" - (`#2881 <https://github.com/Araq/Nim/issues/2881>`_) -- Fixed "xoring unsigned integers yields RangeError in certain conditions" - (`#2979 <https://github.com/Araq/Nim/issues/2979>`_) -- Fixed "Directly checking equality between procs" - (`#2985 <https://github.com/Araq/Nim/issues/2985>`_) -- Fixed "Compiler crashed, but there have to be meaningful error message" - (`#2974 <https://github.com/Araq/Nim/issues/2974>`_) -- Fixed "repr is broken" - (`#2992 <https://github.com/Araq/Nim/issues/2992>`_) -- Fixed "Ipv6 devel - add IPv6 support for asyncsockets, make AF_INET6 a default" - (`#2976 <https://github.com/Araq/Nim/issues/2976>`_) -- Fixed "Compilation broken on windows" - (`#2996 <https://github.com/Araq/Nim/issues/2996>`_) -- Fixed "'u64 literal conversion compiler error" - (`#2731 <https://github.com/Araq/Nim/issues/2731>`_) -- Fixed "Importing 'impure' libraries while using threads causes segfaults" - (`#2672 <https://github.com/Araq/Nim/issues/2672>`_) -- Fixed "Uncatched exception in async procedure on raise statement" - (`#3014 <https://github.com/Araq/Nim/issues/3014>`_) -- Fixed "nim doc2 fails in Mac OS X due to system.nim (possibly related to #1898)" - (`#3005 <https://github.com/Araq/Nim/issues/3005>`_) -- Fixed "IndexError when rebuilding Nim on iteration 2" - (`#3018 <https://github.com/Araq/Nim/issues/3018>`_) -- Fixed "Assigning large const set to variable looses some information" - (`#2880 <https://github.com/Araq/Nim/issues/2880>`_) -- Fixed "Inconsistent generics behavior" - (`#3022 <https://github.com/Araq/Nim/issues/3022>`_) -- Fixed "Compiler breaks on float64 division" - (`#3028 <https://github.com/Araq/Nim/issues/3028>`_) -- Fixed "Confusing error message comparing string to nil " - (`#2935 <https://github.com/Araq/Nim/issues/2935>`_) -- Fixed "convert 64bit number to float on 32bit" - (`#1463 <https://github.com/Araq/Nim/issues/1463>`_) -- Fixed "Type redefinition and construction will break nim check" - (`#3032 <https://github.com/Araq/Nim/issues/3032>`_) -- Fixed "XmlParser fails on very large XML files without new lines" - (`#2429 <https://github.com/Araq/Nim/issues/2429>`_) -- Fixed "Error parsing arguments with whitespaces" - (`#2874 <https://github.com/Araq/Nim/issues/2874>`_) -- Fixed "Crash when missing one arg and used a named arg" - (`#2993 <https://github.com/Araq/Nim/issues/2993>`_) -- Fixed "Wrong number of arguments in assert will break nim check" - (`#3044 <https://github.com/Araq/Nim/issues/3044>`_) -- Fixed "Wrong const definition will break nim check" - (`#3041 <https://github.com/Araq/Nim/issues/3041>`_) -- Fixed "Wrong set declaration will break nim check" - (`#3040 <https://github.com/Araq/Nim/issues/3040>`_) -- Fixed "Compiler segfault (type section)" - (`#2540 <https://github.com/Araq/Nim/issues/2540>`_) -- Fixed "Segmentation fault when compiling this code" - (`#3038 <https://github.com/Araq/Nim/issues/3038>`_) -- Fixed "Kill nim i" - (`#2633 <https://github.com/Araq/Nim/issues/2633>`_) -- Fixed "Nim check will break on wrong array declaration" - (`#3048 <https://github.com/Araq/Nim/issues/3048>`_) -- Fixed "boolVal seems to be broken" - (`#3046 <https://github.com/Araq/Nim/issues/3046>`_) -- Fixed "Nim check crashes on wrong set/array declaration inside ref object" - (`#3062 <https://github.com/Araq/Nim/issues/3062>`_) -- Fixed "Nim check crashes on incorrect generic arg definition" - (`#3051 <https://github.com/Araq/Nim/issues/3051>`_) -- Fixed "Nim check crashes on iterating nonexistent var" - (`#3053 <https://github.com/Araq/Nim/issues/3053>`_) -- Fixed "Nim check crashes on wrong param set declaration + iteration" - (`#3054 <https://github.com/Araq/Nim/issues/3054>`_) -- Fixed "Wrong sharing of static_t instantations" - (`#3112 <https://github.com/Araq/Nim/issues/3112>`_) -- Fixed "Automatically generated proc conflicts with user-defined proc when .exportc.'ed" - (`#3134 <https://github.com/Araq/Nim/issues/3134>`_) -- Fixed "getTypeInfo call crashes nim" - (`#3099 <https://github.com/Araq/Nim/issues/3099>`_) -- Fixed "Array ptr dereference" - (`#2963 <https://github.com/Araq/Nim/issues/2963>`_) -- Fixed "Internal error when `repr`-ing a type directly" - (`#3079 <https://github.com/Araq/Nim/issues/3079>`_) -- Fixed "unknown type name 'TNimType' after importing typeinfo module" - (`#2841 <https://github.com/Araq/Nim/issues/2841>`_) -- Fixed "Can export a template twice and from inside a block" - (`#1738 <https://github.com/Araq/Nim/issues/1738>`_) -- Fixed "C Codegen: C Types are defined after their usage in certain cases" - (`#2823 <https://github.com/Araq/Nim/issues/2823>`_) -- Fixed "s.high refers to the current seq instead of the old one" - (`#1832 <https://github.com/Araq/Nim/issues/1832>`_) -- Fixed "Error while unmarshaling null values" - (`#3149 <https://github.com/Araq/Nim/issues/3149>`_) -- Fixed "Inference of `static[T]` in sequences" - (`#3144 <https://github.com/Araq/Nim/issues/3144>`_) -- Fixed "Argument named "closure" to proc inside template interfere with closure pragma" - (`#3171 <https://github.com/Araq/Nim/issues/3171>`_) -- Fixed "Internal error with aliasing inside template" - (`#3158 <https://github.com/Araq/Nim/issues/3158>`_) -- Fixed "Cardinality of sets prints unexpected value" - (`#3135 <https://github.com/Araq/Nim/issues/3135>`_) -- Fixed "Nim crashes on const assignment from function returning var ref object" - (`#3103 <https://github.com/Araq/Nim/issues/3103>`_) -- Fixed "`repr` cstring" - (`#3080 <https://github.com/Araq/Nim/issues/3080>`_) -- Fixed "Nim check crashes on wrong enum declaration" - (`#3052 <https://github.com/Araq/Nim/issues/3052>`_) -- Fixed "Compiler assertion when evaluating template with static[T]" - (`#1858 <https://github.com/Araq/Nim/issues/1858>`_) -- Fixed "Erroneous overflow in iterators when compiler built with overflowChecks enabled" - (`#3140 <https://github.com/Araq/Nim/issues/3140>`_) -- Fixed "Unicode dashes as "lisp'ish" alternative to hump and snake notation" - (`#2811 <https://github.com/Araq/Nim/issues/2811>`_) -- Fixed "Calling discardable proc from a defer is an error." - (`#3185 <https://github.com/Araq/Nim/issues/3185>`_) -- Fixed "Defer statement at the end of a block produces ICE" - (`#3186 <https://github.com/Araq/Nim/issues/3186>`_) -- Fixed "Call to `createU` fails to compile" - (`#3193 <https://github.com/Araq/Nim/issues/3193>`_) -- Fixed "VM crash when accessing array's element" - (`#3192 <https://github.com/Araq/Nim/issues/3192>`_) -- Fixed "Unexpected proc invoked when different modules add procs to a type from a 3rd module" - (`#2664 <https://github.com/Araq/Nim/issues/2664>`_) -- Fixed "Nim crashes on conditional declaration inside a template" - (`#2670 <https://github.com/Araq/Nim/issues/2670>`_) -- Fixed "Iterator names conflict within different scopes" - (`#2752 <https://github.com/Araq/Nim/issues/2752>`_) -- Fixed "VM: Cannot assign int value to ref variable" - (`#1329 <https://github.com/Araq/Nim/issues/1329>`_) -- Fixed "Incorrect code generated for tagged unions with enums not starting at zero" - (`#3096 <https://github.com/Araq/Nim/issues/3096>`_) -- Fixed "Compile time procs using forward declarations are silently ignored" - (`#3066 <https://github.com/Araq/Nim/issues/3066>`_) -- Fixed "re binding error in generic" - (`#1965 <https://github.com/Araq/Nim/issues/1965>`_) -- Fixed "os.getCreationTime is incorrect/impossible on Posix systems" - (`#1058 <https://github.com/Araq/Nim/issues/1058>`_) -- Fixed "Improve error message for osproc.startProcess when command does not exist" - (`#2183 <https://github.com/Araq/Nim/issues/2183>`_) -- Fixed "gctest segfaults with --gc:markandsweep on x86_64" - (`#2305 <https://github.com/Araq/Nim/issues/2305>`_) -- Fixed "Coroutine changes break compilation on unsupported architectures" - (`#3245 <https://github.com/Araq/Nim/issues/3245>`_) -- Fixed "Bugfix: Windows 32bit TinyCC support issue fixed" - (`#3237 <https://github.com/Araq/Nim/issues/3237>`_) -- Fixed "db_mysql getValue() followed by exec() causing error" - (`#3220 <https://github.com/Araq/Nim/issues/3220>`_) -- Fixed "xmltree.newEntity creates xnCData instead of xnEntity" - (`#3282 <https://github.com/Araq/Nim/issues/3282>`_) -- Fixed "Methods and modules don't work together" - (`#2590 <https://github.com/Araq/Nim/issues/2590>`_) -- Fixed "String slicing not working in the vm" - (`#3300 <https://github.com/Araq/Nim/issues/3300>`_) -- Fixed "internal error: evalOp(mTypeOf)" - (`#3230 <https://github.com/Araq/Nim/issues/3230>`_) -- Fixed "#! source code prefix collides with Unix Shebang" - (`#2559 <https://github.com/Araq/Nim/issues/2559>`_) -- Fixed "wrong codegen for constant object" - (`#3195 <https://github.com/Araq/Nim/issues/3195>`_) -- Fixed "Doc comments inside procs with implicit returns don't work" - (`#1528 <https://github.com/Araq/Nim/issues/1528>`_) - - -2015-10-16 First Nim conference -=============================== - -.. raw::html - - <img src="assets/zeo/banner.jpg" alt="First Nim conference in Ukraine!" width="682"/> - -This Autumn you have the unique opportunity to take part in the first Nim event -held in Kyiv and to meet the creator of the Nim programming language - -Andreas Rumpf. The event is hosted by Zeo Alliance and is taking place between -14-15 November 2015 in Kyiv, Ukraine. - -During the workshop you will learn: - -- The basics of the language including its safe and unsafe subsets. -- How to use Nim to develop web applications. -- How Nim's meta programming capabilities make Nim the ultimate glue language, - excellent at interoperability with C++, JavaScript, Java and others. -- Games in Nim and the ability to rapidly prototype without sacrificing speed. - -Registration is free, but the number of places is limited. More details -can be found `here <https://nimworkshop.splashthat.com/>`_. - - - - - - - - -2015-05-04 Version 0.11.2 released -================================== - -This is just a bugfix release that fixes the most pressing regressions we -introduced with version 0.11.0. The way types are computed was -changed significantly causing all sort of problems. Sorry for the -inconvenience; we grew overconfident our large test suite would prevent these -things. - - -2015-04-30 Version 0.11.0 released -================================== - -With this release we are one step closer to reaching version 1.0 and by -extension the persistence of the Nim specification. As mentioned in the -previous release notes, starting with version 1.0, we will not be introducing -any more breaking changes to Nim. - -The *language* itself is very close to 1.0, the primary area that requires -more work is the standard library. - -Take a look at the `download <download.html>`_ page for binaries (Windows-only) -and 0.11.0 snapshots of the source code. The Windows installer now also -includes `Aporia <https://github.com/nim-lang/aporia>`_, -`Nimble <https://github.com/nim-lang/nimble>`_ and other useful tools to get -you started with Nim. - -What's left to be done -~~~~~~~~~~~~~~~~~~~~~~ - -The 1.0 release is expected by the end of this year. Rumors say it will be in -summer 2015. What's left: - -* Bug fixes, bug fixes, bug fixes, in particular: - - The remaining bugs of the lambda lifting pass that is responsible to enable - closures and closure iterators need to be fixed. - - ``concept`` needs to be refined, a nice name for the feature is not enough. - - Destructors need to be refined. - - ``static[T]`` needs to be fixed. - - Finish the implementation of the 'parallel' statement. -* ``immediate`` templates and macros will be deprecated as these will soon be - completely unnecessary, instead the ``typed`` or ``untyped`` metatypes can - be used. -* More of the standard library should be moved to Nimble packages and what's - left should use the features we have for concurrency and parallelism. - - - -Changes affecting backwards compatibility ------------------------------------------ - -- Parameter names are finally properly ``gensym``'ed. This can break - templates though that used to rely on the fact that they are not. - (Bug #1915.) This means this doesn't compile anymore: - -.. code-block:: nim - - template doIt(body: stmt) {.immediate.} = - # this used to inject the 'str' parameter: - proc res(str: string) = - body - - doIt: - echo str # Error: undeclared identifier: 'str' -.. - - This used to inject the ``str`` parameter into the scope of the body. - Declare the ``doIt`` template as ``immediate, dirty`` to get the old - behaviour. -- Tuple field names are not ignored anymore, this caused too many problems - in practice so now the behaviour is as it was for version 0.9.6: If field - names exist for the tuple type, they are checked. -- ``logging.level`` and ``logging.handlers`` are no longer exported. - ``addHandler``, ``getHandlers``, ``setLogFilter`` and ``getLogFilter`` - should be used instead. -- ``nim idetools`` has been replaced by a separate - tool `nimsuggest <0.11.0/nimsuggest.html>`_. -- *arrow like* operators are not right associative anymore and are required - to end with either ``->``, ``~>`` or - ``=>``, not just ``>``. Examples of operators still considered arrow like: - ``->``, ``==>``, ``+=>``. On the other hand, the following operators are now - considered regular operators again: ``|>``, ``-+>``, etc. -- Typeless parameters are now only allowed in templates and macros. The old - way turned out to be too error-prone. -- The 'addr' and 'type' operators are now parsed as unary function - application. This means ``type(x).name`` is now parsed as ``(type(x)).name`` - and not as ``type((x).name)``. Note that this also affects the AST - structure; for immediate macro parameters ``nkCall('addr', 'x')`` is - produced instead of ``nkAddr('x')``. -- ``concept`` is now a keyword and is used instead of ``generic``. -- The ``inc``, ``dec``, ``+=``, ``-=`` builtins now produce OverflowError - exceptions. This means code like the following: - -.. code-block:: nim - var x = low(T) - while x <= high(T): - echo x - inc x - -Needs to be replaced by something like this: - -.. code-block:: nim - var x = low(T).int - while x <= high(T).int: - echo x.T - inc x - -- **Negative indexing for slicing does not work anymore!** Instead - of ``a[0.. -1]`` you can - use ``a[0.. ^1]``. This also works with accessing a single - element ``a[^1]``. Note that we cannot detect this reliably as it is - determined at **runtime** whether negative indexing is used! - ``a[0.. -1]`` now produces the empty string/sequence. -- The compiler now warns about code like ``foo +=1`` which uses inconsistent - spacing around binary operators. Later versions of the language will parse - these as unary operators instead so that ``echo $foo`` finally can do what - people expect it to do. -- ``system.untyped`` and ``system.typed`` have been introduced as aliases - for ``expr`` and ``stmt``. The new names capture the semantics much better - and most likely ``expr`` and ``stmt`` will be deprecated in favor of the - new names. -- The ``split`` method in module ``re`` has changed. It now handles the case - of matches having a length of 0, and empty strings being yielded from the - iterator. A notable change might be that a pattern being matched at the - beginning and end of a string, will result in an empty string being produced - at the start and the end of the iterator. -- The compiler and nimsuggest now count columns starting with 1, not 0 for - consistency with the rest of the world. - - -Language Additions ------------------- - -- For empty ``case object`` branches ``discard`` can finally be used instead - of ``nil``. -- Automatic dereferencing is now done for the first argument of a routine - call if overloading resolution produces no match otherwise. This feature - has to be enabled with - the `experimental <0.11.0/manual.html#pragmas-experimental-pragma>`_ pragma. -- Objects that do not use inheritance nor ``case`` can be put into ``const`` - sections. This means that finally this is possible and produces rather - nice code: - -.. code-block:: nim - import tables - - const - foo = {"ah": "finally", "this": "is", "possible.": "nice!"}.toTable() - - -- Ordinary parameters can follow after a varargs parameter. This means the - following is finally accepted by the compiler: - -.. code-block:: nim - template takesBlock(a, b: int, x: varargs[expr]; blck: stmt) = - blck - echo a, b - - takesBlock 1, 2, "some", 0.90, "random stuff": - echo "yay" - -- Overloading by 'var T' is now finally possible: - -.. code-block:: nim - proc varOrConst(x: var int) = echo "var" - proc varOrConst(x: int) = echo "const" - - var x: int - varOrConst(x) # "var" - varOrConst(45) # "const" - -- Array and seq indexing can now use the builtin ``^`` operator to access - things from backwards: ``a[^1]`` is like Python's ``a[-1]``. -- A first version of the specification and implementation of the overloading - of the assignment operator has arrived! -- ``system.len`` for strings and sequences now returns 0 for nil. - -- A single underscore can now be used to discard values when unpacking tuples: - -.. code-block:: nim - let (path, _, _) = os.splitFile("path/file.ext") - - -- ``marshal.$$`` and ``marshal.to`` can be executed at compile-time. -- Interoperability with C++ improved tremendously; C++'s templates and - operators can be wrapped directly. See - `this <0.11.0/nimc.html#additional-features-importcpp-pragma>`_ - for more information. -- ``macros.getType`` can be used to query an AST's type at compile-time. This - enables more powerful macros, for instance *currying* can now be done with - a macro. - - -Library additions ------------------ - -- ``reversed`` proc added to the ``unicode`` module. -- Added multipart param to httpclient's ``post`` and ``postContent`` together - with a ``newMultipartData`` proc. -- Added `%*` operator for JSON. -- The compiler is now available as Nimble package for c2nim. -- Added ``..^`` and ``..<`` templates to system so that the rather annoying - space between ``.. <`` and ``.. ^`` is not necessary anymore. -- Added ``system.xlen`` for strings and sequences to get back the old ``len`` - operation that doesn't check for ``nil`` for efficiency. -- Added sexp.nim to parse and generate sexp. - - -Bugfixes --------- - -- Fixed internal compiler error when using ``char()`` in an echo call - (`#1788 <https://github.com/Araq/Nim/issues/1788>`_). -- Fixed Windows cross-compilation on Linux. -- Overload resolution now works for types distinguished only by a - ``static[int]`` param - (`#1056 <https://github.com/Araq/Nim/issues/1056>`_). -- Other fixes relating to generic types and static params. -- Fixed some compiler crashes with unnamed tuples - (`#1774 <https://github.com/Araq/Nim/issues/1774>`_). -- Fixed ``channels.tryRecv`` blocking - (`#1816 <https://github.com/Araq/Nim/issues/1816>`_). -- Fixed generic instantiation errors with ``typedesc`` - (`#419 <https://github.com/Araq/Nim/issues/419>`_). -- Fixed generic regression where the compiler no longer detected constant - expressions properly (`#544 <https://github.com/Araq/Nim/issues/544>`_). -- Fixed internal error with generic proc using ``static[T]`` in a specific - way (`#1049 <https://github.com/Araq/Nim/issues/1049>`_). -- More fixes relating to generics (`#1820 <https://github.com/Araq/Nim/issues/1820>`_, - `#1050 <https://github.com/Araq/Nim/issues/1050>`_, - `#1859 <https://github.com/Araq/Nim/issues/1859>`_, - `#1858 <https://github.com/Araq/Nim/issues/1858>`_). -- Fixed httpclient to properly encode queries. -- Many fixes to the ``uri`` module. -- Async sockets are now closed on error. -- Fixes to httpclient's handling of multipart data. -- Fixed GC segfaults with asynchronous sockets - (`#1796 <https://github.com/Araq/Nim/issues/1796>`_). -- Added more versions to openssl's DLL version list - (`076f993 <https://github.com/Araq/Nim/commit/076f993>`_). -- Fixed shallow copy in iterators being broken - (`#1803 <https://github.com/Araq/Nim/issues/1803>`_). -- ``nil`` can now be inserted into tables with the ``db_sqlite`` module - (`#1866 <https://github.com/Araq/Nim/issues/1866>`_). -- Fixed "Incorrect assembler generated" - (`#1907 <https://github.com/Araq/Nim/issues/1907>`_) -- Fixed "Expression templates that define macros are unusable in some contexts" - (`#1903 <https://github.com/Araq/Nim/issues/1903>`_) -- Fixed "a second level generic subclass causes the compiler to crash" - (`#1919 <https://github.com/Araq/Nim/issues/1919>`_) -- Fixed "nim 0.10.2 generates invalid AsyncHttpClient C code for MSVC " - (`#1901 <https://github.com/Araq/Nim/issues/1901>`_) -- Fixed "1 shl n produces wrong C code" - (`#1928 <https://github.com/Araq/Nim/issues/1928>`_) -- Fixed "Internal error on tuple yield" - (`#1838 <https://github.com/Araq/Nim/issues/1838>`_) -- Fixed "ICE with template" - (`#1915 <https://github.com/Araq/Nim/issues/1915>`_) -- Fixed "include the tool directory in the installer as it is required by koch" - (`#1947 <https://github.com/Araq/Nim/issues/1947>`_) -- Fixed "Can't compile if file location contains spaces on Windows" - (`#1955 <https://github.com/Araq/Nim/issues/1955>`_) -- Fixed "List comprehension macro only supports infix checks as guards" - (`#1920 <https://github.com/Araq/Nim/issues/1920>`_) -- Fixed "wrong field names of compatible tuples in generic types" - (`#1910 <https://github.com/Araq/Nim/issues/1910>`_) -- Fixed "Macros within templates no longer work as expected" - (`#1944 <https://github.com/Araq/Nim/issues/1944>`_) -- Fixed "Compiling for Standalone AVR broken in 0.10.2" - (`#1964 <https://github.com/Araq/Nim/issues/1964>`_) -- Fixed "Compiling for Standalone AVR broken in 0.10.2" - (`#1964 <https://github.com/Araq/Nim/issues/1964>`_) -- Fixed "Code generation for mitems with tuple elements" - (`#1833 <https://github.com/Araq/Nim/issues/1833>`_) -- Fixed "httpclient.HttpMethod should not be an enum" - (`#1962 <https://github.com/Araq/Nim/issues/1962>`_) -- Fixed "terminal / eraseScreen() throws an OverflowError" - (`#1906 <https://github.com/Araq/Nim/issues/1906>`_) -- Fixed "setControlCHook(nil) disables registered quit procs" - (`#1546 <https://github.com/Araq/Nim/issues/1546>`_) -- Fixed "Unexpected idetools behaviour" - (`#325 <https://github.com/Araq/Nim/issues/325>`_) -- Fixed "Unused lifted lambda does not compile" - (`#1642 <https://github.com/Araq/Nim/issues/1642>`_) -- Fixed "'low' and 'high' don't work with cstring asguments" - (`#2030 <https://github.com/Araq/Nim/issues/2030>`_) -- Fixed "Converting to int does not round in JS backend" - (`#1959 <https://github.com/Araq/Nim/issues/1959>`_) -- Fixed "Internal error genRecordField 2 when adding region to pointer." - (`#2039 <https://github.com/Araq/Nim/issues/2039>`_) -- Fixed "Macros fail to compile when compiled with --os:standalone" - (`#2041 <https://github.com/Araq/Nim/issues/2041>`_) -- Fixed "Reading from {.compileTime.} variables can cause code generation to fail" - (`#2022 <https://github.com/Araq/Nim/issues/2022>`_) -- Fixed "Passing overloaded symbols to templates fails inside generic procedures" - (`#1988 <https://github.com/Araq/Nim/issues/1988>`_) -- Fixed "Compiling iterator with object assignment in release mode causes "var not init"" - (`#2023 <https://github.com/Araq/Nim/issues/2023>`_) -- Fixed "calling a large number of macros doing some computation fails" - (`#1989 <https://github.com/Araq/Nim/issues/1989>`_) -- Fixed "Can't get Koch to install nim under Windows" - (`#2061 <https://github.com/Araq/Nim/issues/2061>`_) -- Fixed "Template with two stmt parameters segfaults compiler" - (`#2057 <https://github.com/Araq/Nim/issues/2057>`_) -- Fixed "`noSideEffect` not affected by `echo`" - (`#2011 <https://github.com/Araq/Nim/issues/2011>`_) -- Fixed "Compiling with the cpp backend ignores --passc" - (`#1601 <https://github.com/Araq/Nim/issues/1601>`_) -- Fixed "Put untyped procedure parameters behind the experimental pragma" - (`#1956 <https://github.com/Araq/Nim/issues/1956>`_) -- Fixed "generic regression" - (`#2073 <https://github.com/Araq/Nim/issues/2073>`_) -- Fixed "generic regression" - (`#2073 <https://github.com/Araq/Nim/issues/2073>`_) -- Fixed "Regression in template lookup with generics" - (`#2004 <https://github.com/Araq/Nim/issues/2004>`_) -- Fixed "GC's growObj is wrong for edge cases" - (`#2070 <https://github.com/Araq/Nim/issues/2070>`_) -- Fixed "Compiler internal error when creating an array out of a typeclass" - (`#1131 <https://github.com/Araq/Nim/issues/1131>`_) -- Fixed "GC's growObj is wrong for edge cases" - (`#2070 <https://github.com/Araq/Nim/issues/2070>`_) -- Fixed "Invalid Objective-C code generated when calling class method" - (`#2068 <https://github.com/Araq/Nim/issues/2068>`_) -- Fixed "walkDirRec Error" - (`#2116 <https://github.com/Araq/Nim/issues/2116>`_) -- Fixed "Typo in code causes compiler SIGSEGV in evalAtCompileTime" - (`#2113 <https://github.com/Araq/Nim/issues/2113>`_) -- Fixed "Regression on exportc" - (`#2118 <https://github.com/Araq/Nim/issues/2118>`_) -- Fixed "Error message" - (`#2102 <https://github.com/Araq/Nim/issues/2102>`_) -- Fixed "hint[path] = off not working in nim.cfg" - (`#2103 <https://github.com/Araq/Nim/issues/2103>`_) -- Fixed "compiler crashes when getting a tuple from a sequence of generic tuples" - (`#2121 <https://github.com/Araq/Nim/issues/2121>`_) -- Fixed "nim check hangs with when" - (`#2123 <https://github.com/Araq/Nim/issues/2123>`_) -- Fixed "static[T] param in nested type resolve/caching issue" - (`#2125 <https://github.com/Araq/Nim/issues/2125>`_) -- Fixed "repr should display ``\0``" - (`#2124 <https://github.com/Araq/Nim/issues/2124>`_) -- Fixed "'nim check' never ends in case of recursive dependency " - (`#2051 <https://github.com/Araq/Nim/issues/2051>`_) -- Fixed "From macros: Error: unhandled exception: sons is not accessible" - (`#2167 <https://github.com/Araq/Nim/issues/2167>`_) -- Fixed "`fieldPairs` doesn't work inside templates" - (`#1902 <https://github.com/Araq/Nim/issues/1902>`_) -- Fixed "fields iterator misbehavior on break statement" - (`#2134 <https://github.com/Araq/Nim/issues/2134>`_) -- Fixed "Fix for compiler not building anymore since #c3244ef1ff" - (`#2193 <https://github.com/Araq/Nim/issues/2193>`_) -- Fixed "JSON parser fails in cpp output mode" - (`#2199 <https://github.com/Araq/Nim/issues/2199>`_) -- Fixed "macros.getType mishandles void return" - (`#2211 <https://github.com/Araq/Nim/issues/2211>`_) -- Fixed "Regression involving templates instantiated within generics" - (`#2215 <https://github.com/Araq/Nim/issues/2215>`_) -- Fixed ""Error: invalid type" for 'not nil' on generic type." - (`#2216 <https://github.com/Araq/Nim/issues/2216>`_) -- Fixed "--threads:on breaks async" - (`#2074 <https://github.com/Araq/Nim/issues/2074>`_) -- Fixed "Type mismatch not always caught, can generate bad code for C backend." - (`#2169 <https://github.com/Araq/Nim/issues/2169>`_) -- Fixed "Failed C compilation when storing proc to own type in object" - (`#2233 <https://github.com/Araq/Nim/issues/2233>`_) -- Fixed "Unknown line/column number in constant declaration type conversion error" - (`#2252 <https://github.com/Araq/Nim/issues/2252>`_) -- Fixed "Adding {.compile.} fails if nimcache already exists." - (`#2247 <https://github.com/Araq/Nim/issues/2247>`_) -- Fixed "Two different type names generated for a single type (C backend)" - (`#2250 <https://github.com/Araq/Nim/issues/2250>`_) -- Fixed "Ambigous call when it should not be" - (`#2229 <https://github.com/Araq/Nim/issues/2229>`_) -- Fixed "Make sure we can load root urls" - (`#2227 <https://github.com/Araq/Nim/issues/2227>`_) -- Fixed "Failure to slice a string with an int subrange type" - (`#794 <https://github.com/Araq/Nim/issues/794>`_) -- Fixed "documentation error" - (`#2205 <https://github.com/Araq/Nim/issues/2205>`_) -- Fixed "Code growth when using `const`" - (`#1940 <https://github.com/Araq/Nim/issues/1940>`_) -- Fixed "Instances of generic types confuse overload resolution" - (`#2220 <https://github.com/Araq/Nim/issues/2220>`_) -- Fixed "Compiler error when initializing sdl2's EventType" - (`#2316 <https://github.com/Araq/Nim/issues/2316>`_) -- Fixed "Parallel disjoint checking can't handle `<`, `items`, or arrays" - (`#2287 <https://github.com/Araq/Nim/issues/2287>`_) -- Fixed "Strings aren't copied in parallel loop" - (`#2286 <https://github.com/Araq/Nim/issues/2286>`_) -- Fixed "JavaScript compiler crash with tables" - (`#2298 <https://github.com/Araq/Nim/issues/2298>`_) -- Fixed "Range checker too restrictive" - (`#1845 <https://github.com/Araq/Nim/issues/1845>`_) -- Fixed "Failure to slice a string with an int subrange type" - (`#794 <https://github.com/Araq/Nim/issues/794>`_) -- Fixed "Remind user when compiling in debug mode" - (`#1868 <https://github.com/Araq/Nim/issues/1868>`_) -- Fixed "Compiler user guide has jumbled options/commands." - (`#1819 <https://github.com/Araq/Nim/issues/1819>`_) -- Fixed "using `method`: 1 in a objects constructor fails when compiling" - (`#1791 <https://github.com/Araq/Nim/issues/1791>`_) - - -2014-12-29 Version 0.10.2 released -================================== - -This release marks the completion of a very important change to the project: -the official renaming from Nimrod to Nim. Version 0.10.2 contains many language -changes, some of which may break your existing code. For your convenience, we -added a new tool called `nimfix <nimfix.html>`_ that will help you convert your -existing projects so that it works with the latest version of the compiler. - -Progress towards version 1.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Although Nim is still pre-1.0, we were able to keep the number of breaking -changes to a minimum so far. Starting with version 1.0, we will not introduce -any breaking changes between major release versions. -One of Nim's goals is to ensure that the compiler is as efficient as possible. -Take a look at the -`latest benchmarks <https://github.com/logicchains/LPATHBench/blob/master/writeup.md>`_, -which show that Nim is consistently near -the top and already nearly as fast as C and C++. Recent developments, such as -the new ``asyncdispatch`` module will allow you to write efficient web server -applications using non-blocking code. Nim now also has a built-in thread pool -for lightweight threading through the use of ``spawn``. - -The unpopular "T" and "P" prefixes on types have been deprecated. Nim also -became more expressive by weakening the distinction between statements and -expressions. We also added a new and searchable forum, a new website, and our -documentation generator ``docgen`` has seen major improvements. Many thanks to -Nick Greenfield for the much more beautiful documentation! - - - -What's left to be done -~~~~~~~~~~~~~~~~~~~~~~ - -The 1.0 release is actually very close. Apart from bug fixes, there are -two major features missing or incomplete: - -* ``static[T]`` needs to be defined precisely and the bugs in the - implementation need to be fixed. -* Overloading of the assignment operator is required for some generic - containers and needs to be implemented. - -This means that fancy matrix libraries will finally start to work, which used -to be a major point of pain in the language. - - -Nimble and other Nim tools -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Outside of the language and the compiler itself many Nim tools have seen -considerable improvements. - -Babel the Nim package manager has been renamed to Nimble. Nimble's purpose -is the installation of packages containing libraries and/or applications -written in Nim. -Even though Nimble is still very young it already is very -functional. It can install packages by name, it does so by accessing a -packages repository which is hosted on a GitHub repo. Packages can also be -installed via a Git repo URL or Mercurial repo URL. The package repository -is searchable through Nimble. Anyone is free to add their own packages to -the package repository by forking the -`nim-lang/packages <https://github.com/nim-lang/packages>`_ repo and creating -a pull request. Nimble is fully cross-platform and should be fully functional -on all major operating systems. -It is of course completely written in Nim. - -Changelog -~~~~~~~~~ - -Changes affecting backwards compatibility ------------------------------------------ - -- **The language has been renamed from Nimrod to Nim.** The name of the - compiler changed from ``nimrod`` to ``nim`` too. -- ``system.fileHandle`` has been renamed to ``system.getFileHandle`` to - prevent name conflicts with the new type ``FileHandle``. -- Comments are now not part of the AST anymore, as such you cannot use them - in place of ``discard``. -- Large parts of the stdlib got rid of the T/P type prefixes. Instead most - types now simply start with an uppercased letter. The - so called "partial case sensitivity" rule is now active allowing for code - like ``var foo: Foo`` in more contexts. -- String case (or any non-ordinal case) statements - without 'else' are deprecated. -- Recursive tuple types are not allowed anymore. Use ``object`` instead. -- The PEGS module returns ``nil`` instead of ``""`` when an optional capture - fails to match. -- The re module returns ``nil`` instead of ``""`` when an optional capture - fails to match. -- The "symmetric set difference" operator (``-+-``) never worked and has been - removed. -- ``defer`` is a keyword now. -- ``func`` is a keyword now. -- The ``using`` language feature now needs to be activated via the new - ``{.experimental.}`` pragma that enables experimental language features. -- Destructors are now officially *experimental*. -- Standalone ``except`` and ``finally`` statements are deprecated now. - The standalone ``finally`` can be replaced with ``defer``, - standalone ``except`` requires an explicit ``try``. -- Operators ending in ``>`` are considered as "arrow like" and have their - own priority level and are right associative. This means that - the ``=>`` and ``->`` operators from the `future <future.html>`_ module - work better. -- Field names in tuples are now ignored for type comparisons. This allows - for greater interoperability between different modules. -- Statement lists are not converted to an implicit ``do`` block anymore. This - means the confusing ``nnkDo`` nodes when working with macros are gone for - good. - - -Language Additions ------------------- - -- The new concurrency model has been implemented including ``locks`` sections, - lock levels and object field ``guards``. -- The ``parallel`` statement has been implemented. -- ``deepCopy`` has been added to the language. -- The builtin ``procCall`` can be used to get ``super``-like functionality - for multi methods. -- There is a new pragma ``{.experimental.}`` that enables experimental - language features per module, or you can enable these features on a global - level with the ``--experimental`` command line option. - - -Compiler Additions ------------------- - -- The compiler now supports *mixed* Objective C / C++ / C code generation: - The modules that use ``importCpp`` or ``importObjc`` are compiled to C++ - or Objective C code, any other module is compiled to C code. This - improves interoperability. -- There is a new ``parallel`` statement for safe fork&join parallel computing. -- ``guard`` and ``lock`` pragmas have been implemented to support safer - concurrent programming. -- The following procs are now available at compile-time:: - - math.sqrt, math.ln, math.log10, math.log2, math.exp, math.round, - math.arccos, math.arcsin, math.arctan, math.arctan2, math.cos, - math.cosh, math.hypot, math.sinh, math.sin, math.tan, math.tanh, - math.pow, math.trunc, math.floor, math.ceil, math.fmod, - os.getEnv, os.existsEnv, os.dirExists, os.fileExists, - system.writeFile - -- Two backticks now produce a single backtick within an ``emit`` or ``asm`` - statement. -- There is a new tool, `nimfix <nimfix.html>`_ to help you in updating your - code from Nimrod to Nim. -- The compiler's output has been prettified. - -Library Additions ------------------ - -- Added module ``fenv`` to control the handling of floating-point rounding and - exceptions (overflow, division by zero, etc.). -- ``system.setupForeignThreadGc`` can be used for better interaction with - foreign libraries that create threads and run a Nim callback from these - foreign threads. -- List comprehensions have been implemented as a macro in the ``future`` - module. -- The new Async module (``asyncnet``) now supports SSL. -- The ``smtp`` module now has an async implementation. -- Added module ``asyncfile`` which implements asynchronous file reading - and writing. -- ``osproc.kill`` has been added. -- ``asyncnet`` and ``asynchttpserver`` now support ``SO_REUSEADDR``. - -Bugfixes --------- - -- ``nil`` and ``NULL`` are now preserved between Nim and databases in the - ``db_*`` modules. -- Fixed issue with OS module in non-unicode mode on Windows. -- Fixed issue with ``x.low`` - (`#1366 <https://github.com/Araq/Nim/issues/1366>`_). -- Fixed tuple unpacking issue inside closure iterators - (`#1067 <https://github.com/Araq/Nim/issues/1067>`_). -- Fixed ENDB compilation issues. -- Many ``asynchttpserver`` fixes. -- Macros can now keep global state across macro calls - (`#903 <https://github.com/Araq/Nim/issues/903>`_). -- ``osproc`` fixes on Windows. -- ``osproc.terminate`` fixed. -- Improvements to exception handling in async procedures. - (`#1487 <https://github.com/Araq/Nim/issues/1487>`_). -- ``try`` now works at compile-time. -- Fixes ``T = ref T`` to be an illegal recursive type. -- Self imports are now disallowed. -- Improved effect inference. -- Fixes for the ``math`` module on Windows. -- User defined pragmas will now work for generics that have - been instantiated in different modules. -- Fixed queue exhaustion bug. -- Many, many more. - -2014-12-09 New website design! -============================== - -A brand new website including an improved forum is now live. -All thanks go to Philip Witte and -Dominik Picheta, Philip Witte for the design of the website (together with -the logo) as well as the HTML and CSS code for his template, and Dominik Picheta -for integrating Philip's design with Nim's forum. We're sure you will -agree that Philip's design is beautiful. - - - -2014-10-19 Version 0.9.6 released -================================= - -**Note: 0.9.6 is the last release of Nimrod. The language is being renamed to -Nim. Nim slightly breaks compatibility.** - -This is a maintenance release. The upcoming 0.10.0 release has -the new features and exciting developments. - - -Changes affecting backwards compatibility ------------------------------------------ - -- ``spawn`` now uses an elaborate self-adapting thread pool and as such - has been moved into its own module. So to use it, you now have to import - ``threadpool``. -- The symbol binding rules in generics changed: ``bar`` in ``foo.bar`` is - now considered for implicit early binding. -- ``c2nim`` moved into its own repository and is now a Babel package. -- ``pas2nim`` moved into its own repository and is now a Babel package. -- ``system.$`` for floating point types now produces a human friendly string - representation. -- ``uri.TUrl`` as well as the ``parseurl`` module are now deprecated in favour - of the new ``TUri`` type in the ``uri`` module. -- The ``destructor`` pragma has been deprecated. Use the ``override`` pragma - instead. The destructor's name has to be ``destroy`` now. -- ``lambda`` is not a keyword anymore. -- **system.defined has been split into system.defined and system.declared**. - You have to use ``--symbol`` to declare new conditional symbols that can be - set via ``--define``. -- ``--threadanalysis:on`` is now the default. To make your program compile - you can disable it but this is only a temporary solution as this option - will disappear soon! - - -Compiler improvements ---------------------- - -- Multi method dispatching performance has been improved by a factor of 10x for - pathological cases. - - -Language Additions ------------------- - -- This version introduces the ``deprecated`` pragma statement that is used - to handle the upcoming massive amount of symbol renames. -- ``spawn`` can now wrap proc that has a return value. It then returns a data - flow variable of the wrapped return type. - - -Library Additions ------------------ - -- Added module ``cpuinfo``. -- Added module ``threadpool``. -- ``sequtils.distnct`` has been renamed to ``sequtils.deduplicate``. -- Added ``algorithm.reversed`` -- Added ``uri.combine`` and ``uri.parseUri``. -- Some sockets procedures now support a ``SafeDisconn`` flag which causes - them to handle disconnection errors and not raise them. - - -2014-04-21 Version 0.9.4 released -================================= - -The Nimrod development community is proud to announce the release of version -0.9.4 of the Nimrod compiler and tools. **Note: This release has to be -considered beta quality! Lots of new features have been implemented but -unfortunately some do not fulfill our quality standards yet.** - -Prebuilt binaries and instructions for building from source are available -on the `download page <download.html>`_. - -This release includes about -`1400 changes <https://github.com/Araq/Nimrod/compare/v0.9.2...v0.9.4>`_ -in total including various bug -fixes, new languages features and standard library additions and improvements. -This release brings with it support for user-defined type classes, a brand -new VM for executing Nimrod code at compile-time and new symbol binding -rules for clean templates. - -It also introduces support for the brand new -`Babel package manager <https://github.com/nimrod-code/babel>`_ which -has itself seen its first release recently. Many of the wrappers that were -present in the standard library have been moved to separate repositories -and should now be installed using Babel. - -Apart from that a new **experimental** Asynchronous IO API has been added via -the ``asyncdispatch`` and ``asyncnet`` modules. The ``net`` and ``rawsockets`` -modules have also been added and they will likely replace the sockets -module in the next release. The Asynchronous IO API has been designed to -take advantage of Linux's epoll and Windows' IOCP APIs, support for BSD's -kqueue has not been implemented yet but will be in the future. -The Asynchronous IO API provides both -a callback interface and an interface which allows you to write code as you -would if you were writing synchronous code. The latter is done through -the use of an ``await`` macro which behaves similar to C#'s await. The -following is a very simple chat server demonstrating Nimrod's new async -capabilities. - -.. code-block::nim - import asyncnet, asyncdispatch - - var clients: seq[PAsyncSocket] = @[] - - proc processClient(client: PAsyncSocket) {.async.} = - while true: - let line = await client.recvLine() - for c in clients: - await c.send(line & "\c\L") - - proc serve() {.async.} = - var server = newAsyncSocket() - server.bindAddr(TPort(12345)) - server.listen() - - while true: - let client = await server.accept() - clients.add client - - processClient(client) - - serve() - runForever() - - -Note that this feature has been implemented with Nimrod's macro system and so -``await`` and ``async`` are no keywords. - -Syntactic sugar for anonymous procedures has also been introduced. It too has -been implemented as a macro. The following shows some simple usage of the new -syntax: - -.. code-block::nim - import future - - var s = @[1, 2, 3, 4, 5] - echo(s.map((x: int) => x * 5)) - -A list of changes follows, for a comprehensive list of changes take a look -`here <https://github.com/Araq/Nimrod/compare/v0.9.2...v0.9.4>`_. - -Library Additions ------------------ - -- Added ``macros.genSym`` builtin for AST generation. -- Added ``macros.newLit`` procs for easier AST generation. -- Added module ``logging``. -- Added module ``asyncdispatch``. -- Added module ``asyncnet``. -- Added module ``net``. -- Added module ``rawsockets``. -- Added module ``selectors``. -- Added module ``asynchttpserver``. -- Added support for the new asynchronous IO in the ``httpclient`` module. -- Added a Python-inspired ``future`` module that features upcoming additions - to the ``system`` module. - - -Changes affecting backwards compatibility ------------------------------------------ - -- The scoping rules for the ``if`` statement changed for better interaction - with the new syntactic construct ``(;)``. -- ``OSError`` family of procedures has been deprecated. Procedures with the same - name but which take different parameters have been introduced. These procs now - require an error code to be passed to them. This error code can be retrieved - using the new ``OSLastError`` proc. -- ``os.parentDir`` now returns "" if there is no parent dir. -- In CGI scripts stacktraces are shown to the user only - if ``cgi.setStackTraceStdout`` is used. -- The symbol binding rules for clean templates changed: ``bind`` for any - symbol that's not a parameter is now the default. ``mixin`` can be used - to require instantiation scope for a symbol. -- ``quoteIfContainsWhite`` now escapes argument in such way that it can be safely - passed to shell, instead of just adding double quotes. -- ``macros.dumpTree`` and ``macros.dumpLisp`` have been made ``immediate``, - ``dumpTreeImm`` and ``dumpLispImm`` are now deprecated. -- The ``nil`` statement has been deprecated, use an empty ``discard`` instead. -- ``sockets.select`` now prunes sockets that are **not** ready from the list - of sockets given to it. -- The ``noStackFrame`` pragma has been renamed to ``asmNoStackFrame`` to - ensure you only use it when you know what you're doing. -- Many of the wrappers that were present in the standard library have been - moved to separate repositories and should now be installed using Babel. - - -Compiler Additions ------------------- - -- The compiler can now warn about "uninitialized" variables. (There are no - real uninitialized variables in Nimrod as they are initialized to binary - zero). Activate via ``{.warning[Uninit]:on.}``. -- The compiler now enforces the ``not nil`` constraint. -- The compiler now supports a ``codegenDecl`` pragma for even more control - over the generated code. -- The compiler now supports a ``computedGoto`` pragma to support very fast - dispatching for interpreters and the like. -- The old evaluation engine has been replaced by a proper register based - virtual machine. This fixes numerous bugs for ``nimrod i`` and for macro - evaluation. -- ``--gc:none`` produces warnings when code uses the GC. -- A ``union`` pragma for better C interoperability is now supported. -- A ``packed`` pragma to control the memory packing/alignment of fields in - an object. -- Arrays can be annotated to be ``unchecked`` for easier low level - manipulations of memory. -- Support for the new Babel package manager. - - -Language Additions ------------------- - -- Arrays can now be declared with a single integer literal ``N`` instead of a - range; the range is then ``0..N-1``. -- Added ``requiresInit`` pragma to enforce explicit initialization. -- Exported templates are allowed to access hidden fields. -- The ``using statement`` enables you to more easily author domain-specific - languages and libraries providing OOP-like syntactic sugar. -- Added the possibility to override various dot operators in order to handle - calls to missing procs and reads from undeclared fields at compile-time. -- The overload resolution now supports ``static[T]`` params that must be - evaluable at compile-time. -- Support for user-defined type classes has been added. -- The *command syntax* is supported in a lot more contexts. -- Anonymous iterators are now supported and iterators can capture variables - of an outer proc. -- The experimental ``strongSpaces`` parsing mode has been implemented. -- You can annotate pointer types with regions for increased type safety. -- Added support for the builtin ``spawn`` for easy thread pool usage. - - -Tools improvements ------------------- - -- c2nim can deal with a subset of C++. Use the ``--cpp`` command line option - to activate. - - -2014-02-11 Nimrod Featured in Dr. Dobb's Journal -================================================ - -Nimrod has been `featured<http://www.drdobbs.com/open-source/nimrod-a-new-systems-programming-languag/240165321>`_ -as the cover story in the February 2014 issue of Dr. Dobb's Journal. - - -2014-01-15 Andreas Rumpf's talk on Nimrod at Strange Loop 2013 is now online -============================================================================ - -Andreas Rumpf presented *Nimrod: A New Approach to Metaprogramming* at -`Strange Loop 2013<https://thestrangeloop.com/sessions/nimrod-a-new-approach-to-meta-programming>`_. -The `video and slides<http://www.infoq.com/presentations/nimrod>`_ -of the talk are now available. - - -2013-05-20 New website design! -============================== - -A brand new website is now live. All thanks go to Philip Witte and -Dominik Picheta, Philip Witte for the design of the website (together with -the logo) as well as the HTML and CSS code for his template, and Dominik Picheta -for integrating Philip's design with the ``nimweb`` utility. We're sure you will -agree that Philip's design is beautiful. - - -2013-05-20 Version 0.9.2 released -================================= - -We are pleased to announce that version 0.9.2 of the Nimrod compiler has been -released. This release has attracted by far the most contributions in comparison -to any other release. - -This release brings with it many new features and bug fixes, a list of which -can be seen later. One of the major new features is the effect system together -with exception tracking which allows for checked exceptions and more, -for further details check out the `manual <manual.html#effect-system>`_. -Another major new feature is the introduction of statement list expressions, -more details on these can be found `here <manual.html#statement-list-expression>`_. -The ability to exclude symbols from modules has also been -implemented, this feature can be used like so: ``import module except symbol``. - -Thanks to all `contributors <https://github.com/Araq/Nimrod/contributors>`_! - -Bugfixes --------- - -- The old GC never collected cycles correctly. Fixed but it can cause - performance regressions. However you can deactivate the cycle collector - with ``GC_disableMarkAndSweep`` and run it explicitly at an appropriate time - or not at all. There is also a new GC you can activate - with ``--gc:markAndSweep`` which does not have this problem but is slower in - general and has no realtime guarantees. -- ``cast`` for floating point types now does the bitcast as specified in the - manual. This breaks code that erroneously uses ``cast`` to convert different - floating point values. -- SCGI module's performance has been improved greatly, it will no longer block - on many concurrent requests. -- In total fixed over 70 github issues and merged over 60 pull requests. - - -Library Additions ------------------ - -- There is a new experimental mark&sweep GC which can be faster (or much - slower) than the default GC. Enable with ``--gc:markAndSweep``. -- Added ``system.onRaise`` to support a condition system. -- Added ``system.locals`` that provides access to a proc's locals. -- Added ``macros.quote`` for AST quasi-quoting. -- Added ``system.unsafeNew`` to support hacky variable length objects. -- ``system.fields`` and ``system.fieldPairs`` support ``object`` too; they - used to only support tuples. -- Added ``system.CurrentSourcePath`` returning the full file-system path of - the current source file. -- The ``macros`` module now contains lots of useful helpers for building up - abstract syntax trees. - - -Changes affecting backwards compatibility ------------------------------------------ - -- ``shared`` is a keyword now. -- Deprecated ``sockets.recvLine`` and ``asyncio.recvLine``, added - ``readLine`` instead. -- The way indentation is handled in the parser changed significantly. However, - this affects very little (if any) real world code. -- The expression/statement unification has been implemented. Again this - only affects edge cases and no known real world code. -- Changed the async interface of the ``scgi`` module. -- WideStrings are now garbage collected like other string types. - - -Compiler Additions ------------------- - -- The ``doc2`` command does not generate output for the whole project anymore. - Use the new ``--project`` switch to enable this behaviour. -- The compiler can now warn about shadowed local variables. However, this needs - to be turned on explicitly via ``--warning[ShadowIdent]:on``. -- The compiler now supports almost every pragma in a ``push`` pragma. -- Generic converters have been implemented. -- Added a **highly experimental** ``noforward`` pragma enabling a special - compilation mode that largely eliminates the need for forward declarations. - -Language Additions ------------------- - -- ``case expressions`` are now supported. -- Table constructors now mimic more closely the syntax of the ``case`` - statement. -- Nimrod can now infer the return type of a proc from its body. -- Added a ``mixin`` declaration to affect symbol binding rules in generics. -- Exception tracking has been added and the ``doc2`` command annotates possible - exceptions for you. -- User defined effects ("tags") tracking has been added and the ``doc2`` - command annotates possible tags for you. -- Types can be annotated with the new syntax ``not nil`` to explicitly state - that ``nil`` is not allowed. However currently the compiler performs no - advanced static checking for this; for now it's merely for documentation - purposes. -- An ``export`` statement has been added to the language: It can be used for - symbol forwarding so client modules don't have to import a module's - dependencies explicitly. -- Overloading based on ASTs has been implemented. -- Generics are now supported for multi methods. -- Objects can be initialized via an *object constructor expression*. -- There is a new syntactic construct ``(;)`` unifying expressions and - statements. -- You can now use ``from module import nil`` if you want to import the module - but want to enforce fully qualified access to every symbol in ``module``. - - -Notes for the future --------------------- - -- The scope rules of ``if`` statements will change in 0.9.4. This affects the - ``=~`` pegs/re templates. -- The ``sockets`` module will become a low-level wrapper of OS-specific socket - functions. All the high-level features of the current ``sockets`` module - will be moved to a ``network`` module. - - -2012-09-23 Version 0.9.0 released -================================= - -Summary -------- - -* Unsigned integers have been added. -* The integer type promotion rules changed. -* The template and macro system evolved. -* Closures have been implemented. -* Term rewriting macros have been implemented. -* First steps to unify expressions and statements have been taken. -* Symbol lookup rules in generics have become stricter to catch more errors. - - -Bugfixes --------- - -- Fixed a bug where the compiler would "optimize away" valid constant parts of - a string concatenation. -- Fixed a bug concerning implicit type conversions in ``case`` statements. -- Fixed a serious code generation bug that caused ``algorithm.sort`` to - produce segmentation faults. -- Fixed ambiguity in recvLine which meant that receiving ``\r\L`` was - indistinguishable from disconnections. -- Many more bugfixes, too many to list them all. - - -Library Additions ------------------ - -- Added the (already existing) module ``htmlgen`` to the documentation. -- Added the (already existing) module ``cookies`` to the documentation. -- Added ``system.shallow`` that can be used to speed up string and sequence - assignments. -- Added ``system.eval`` that can execute an anonymous block of code at - compile time as if was a macro. -- Added ``system.staticExec`` and ``system.gorge`` for compile-time execution - of external programs. -- Added ``system.staticRead`` as a synonym for ``system.slurp``. -- Added ``macros.emit`` that can emit an arbitrary computed string as nimrod - code during compilation. -- Added ``strutils.parseEnum``. -- Added ``json.%`` constructor operator. -- The stdlib can now be avoided to a point where C code generation for 16bit - micro controllers is feasible. -- Added module ``oids``. -- Added module ``endians``. -- Added a new OpenGL wrapper that supports OpenGL up to version 4.2. -- Added a wrapper for ``libsvm``. -- Added a wrapper for ``mongodb``. -- Added ``terminal.isatty``. -- Added an overload for ``system.items`` that can be used to iterate over the - values of an enum. -- Added ``system.TInteger`` and ``system.TNumber`` type classes matching - any of the corresponding types available in Nimrod. -- Added ``system.clamp`` to limit a value within an interval ``[a, b]``. -- Added ``strutils.continuesWith``. -- Added ``system.getStackTrace``. -- Added ``system.||`` for parallel ``for`` loop support. -- The GC supports (soft) realtime systems via ``GC_setMaxPause`` - and ``GC_step`` procs. -- The sockets module now supports ssl through the OpenSSL library, ``recvLine`` - is now much more efficient thanks to the newly implemented sockets buffering. -- The httpclient module now supports ssl/tls. -- Added ``times.format`` as well as many other utility functions - for managing time. -- Added ``system.@`` for converting an ``openarray`` to a ``seq`` (it used to - only support fixed length arrays). -- Added ``system.compiles`` which can be used to check whether a type supports - some operation. -- Added ``strutils.format``, ``subexes.format`` which use the - new ``varargs`` type. -- Added module ``fsmonitor``. - -Changes affecting backwards compatibility ------------------------------------------ - -- On Windows filenames and paths are supposed to be in UTF-8. - The ``system``, ``os``, ``osproc`` and ``memfiles`` modules use the wide - string versions of the WinAPI. Use the ``-d:useWinAnsi`` switch to revert - back to the old behaviour which uses the Ansi string versions. -- ``static``, ``do``, ``interface`` and ``mixin`` are now keywords. -- Templates now participate in overloading resolution which can break code that - uses templates in subtle ways. Use the new ``immediate`` pragma for templates - to get a template of old behaviour. -- There is now a proper distinction in the type system between ``expr`` and - ``PNimrodNode`` which unfortunately breaks the old macro system. -- ``pegs.@`` has been renamed to ``pegs.!*`` and ``pegs.@@`` has been renamed - to ``pegs.!*\`` as ``@`` operators now have different precedence. -- The type ``proc`` (without any params or return type) is now considered a - type class matching all proc types. Use ``proc ()`` to get the old meaning - denoting a proc expecing no arguments and returing no value. -- Deprecated ``system.GC_setStrategy``. -- ``re.findAll`` and ``pegs.findAll`` don't return *captures* anymore but - matching *substrings*. -- RTTI and thus the ``marshall`` module don't contain the proper field names - of tuples anymore. This had to be changed as the old behaviour never - produced consistent results. -- Deprecated the ``ssl`` module. -- Deprecated ``nimrod pretty`` as it never worked good enough and has some - inherent problems. -- The integer promotion rules changed; the compiler is now less picky in some - situations and more picky in other situations: In particular implicit - conversions from ``int`` to ``int32`` are now forbidden. -- ``system.byte`` is now an alias for ``uint8``; it used to be an alias - to ``int8``. -- ``bind`` expressions in templates are not properly supported anymore. Use - the declarative ``bind`` statement instead. -- The default calling convention for a procedural **type** is now ``closure``, - for procs it remains ``nimcall`` (which is compatible to ``closure``). - Activate the warning ``ImplicitClosure`` to make the compiler list the - occurrences of proc types which are affected. -- The Nimrod type system now distinguishes ``openarray`` from ``varargs``. -- Templates are now ``hygienic``. Use the ``dirty`` pragma to get the old - behaviour. -- Objects that have no ancestor are now implicitly ``final``. Use - the ``inheritable`` pragma to introduce new object roots apart - from ``TObject``. -- Macros now receive parameters like templates do; use the ``callsite`` builtin - to gain access to the invocation AST. -- Symbol lookup rules in generics have become stricter to catch more errors. - - -Compiler Additions ------------------- - -- Win64 is now an officially supported target. -- The Nimrod compiler works on BSD again, but has some issues - as ``os.getAppFilename`` and ``os.getAppDir`` cannot work reliably on BSD. -- The compiler can detect and evaluate calls that can be evaluated at compile - time for optimization purposes with the ``--implicitStatic`` command line - option or pragma. -- The compiler now generates marker procs that the GC can use instead of RTTI. - This speeds up the GC quite a bit. -- The compiler now includes a new advanced documentation generator - via the ``doc2`` command. This new generator uses all of the semantic passes - of the compiler and can thus generate documentation for symbols hiding in - macros. -- The compiler now supports the ``dynlib`` pragma for variables. -- The compiler now supports ``bycopy`` and ``byref`` pragmas that affect how - objects/tuples are passed. -- The embedded profiler became a stack trace profiler and has been documented. - - -Language Additions ------------------- - -- Added explicit ``static`` sections for enforced compile time evaluation. -- Added an alternative notation for lambdas with ``do``. -- ``addr`` is now treated like a prefix operator syntactically. -- Added ``global`` pragma that can be used to introduce new global variables - from within procs. -- ``when`` expressions are now allowed just like ``if`` expressions. -- The precedence for operators starting with ``@`` is different now - allowing for *sigil-like* operators. -- Stand-alone ``finally`` and ``except`` blocks are now supported. -- Macros and templates can now be invoked as pragmas. -- The apostrophe in type suffixes for numerical literals is now optional. -- Unsigned integer types have been added. -- The integer promotion rules changed. -- Nimrod now tracks proper intervals for ``range`` over some built-in operators. -- In parameter lists a semicolon instead of a comma can be used to improve - readability: ``proc divmod(a, b: int; resA, resB: var int)``. -- A semicolon can now be used to have multiple simple statements on a single - line: ``inc i; inc j``. -- ``bind`` supports overloaded symbols and operators. -- A ``distinct`` type can now borrow from generic procs. -- Added the pragmas ``gensym``, ``inject`` and ``dirty`` for hygiene - in templates. -- Comments can be continued with a backslash continuation character so that - comment pieces don't have to align on the same column. -- Enums can be annotated with ``pure`` so that their field names do not pollute - the current scope. -- A proc body can consist of an expression that has a type. This is rewritten - to ``result = expression`` then. -- Term rewriting macros (see `trmacros <http://nimrod-code.org/trmacros.html>`_) - have been implemented but are still in alpha. - - -2012-02-09 Version 0.8.14 released -================================== - -Version 0.8.14 has been released! - -Bugfixes --------- - -- Fixed a serious memory corruption concerning message passing. -- Fixed a serious bug concerning different instantiations of a generic proc. -- Fixed a newly introduced bug where a wrong ``EIO`` exception was raised for - the end of file for text files that do not end with a newline. -- Bugfix c2nim, c2pas: the ``--out`` option has never worked properly. -- Bugfix: forwarding of generic procs never worked. -- Some more bugfixes for macros and compile-time evaluation. -- The GC now takes into account interior pointers on the stack which may be - introduced by aggressive C optimizers. -- Nimrod's native allocator/GC now works on PowerPC. -- Lots of other bugfixes: Too many to list them all. - - -Changes affecting backwards compatibility ------------------------------------------ - -- Removed deprecated ``os.AppendFileExt``, ``os.executeShellCommand``, - ``os.iterOverEnvironment``, ``os.pcDirectory``, ``os.pcLinkToDirectory``, - ``os.SplitPath``, ``os.extractDir``, ``os.SplitFilename``, - ``os.extractFileTrunk``, ``os.extractFileExt``, ``osproc.executeProcess``, - ``osproc.executeCommand``. -- Removed deprecated ``parseopt.init``, ``parseopt.getRestOfCommandLine``. -- Moved ``strutils.validEmailAddress`` to ``matchers.validEmailAddress``. -- The pointer dereference operator ``^`` has been removed, so that ``^`` - can now be a user-defined operator. -- ``implies`` is no keyword anymore. -- The ``is`` operator is now the ``of`` operator. -- The ``is`` operator is now used to check type equivalence in generic code. -- The ``pure`` pragma for procs has been renamed to ``noStackFrame``. -- The threading API has been completely redesigned. -- The ``unidecode`` module is now thread-safe and its interface has changed. -- The ``bind`` expression is deprecated, use a ``bind`` declaration instead. -- ``system.raiseHook`` is now split into ``system.localRaiseHook`` and - ``system.globalRaiseHook`` to distinguish between thread local and global - raise hooks. -- Changed exception handling/error reporting for ``os.removeFile`` and - ``os.removeDir``. -- The algorithm for searching and loading configuration files has been changed. -- Operators now have diffent precedence rules: Assignment-like operators - (like ``*=``) are now special-cased. -- The fields in ``TStream`` have been renamed to have an ``Impl`` suffix - because they should not be used directly anymore. - Wrapper procs have been created that should be used instead. -- ``export`` is now a keyword. -- ``assert`` is now implemented in pure Nimrod as a template; it's easy - to implement your own assertion templates with ``system.astToStr``. - - -Language Additions ------------------- - -- Added new ``is`` and ``of`` operators. -- The built-in type ``void`` can be used to denote the absence of any type. - This is useful in generic code. -- Return types may be of the type ``var T`` to return an l-value. -- The error pragma can now be used to mark symbols whose *usage* should trigger - a compile-time error. -- There is a new ``discardable`` pragma that can be used to mark a routine - so that its result can be discarded implicitly. -- Added a new ``noinit`` pragma to prevent automatic initialization to zero - of variables. -- Constants can now have the type ``seq``. -- There is a new user-definable syntactic construct ``a{i, ...}`` - that has no semantics yet for built-in types and so can be overloaded to your - heart's content. -- ``bind`` (used for symbol binding in templates and generics) is now a - declarative statement. -- Nimrod now supports single assignment variables via the ``let`` statement. -- Iterators named ``items`` and ``pairs`` are implicitly invoked when - an explicit iterator is missing. -- The slice assignment ``a[i..j] = b`` where ``a`` is a sequence or string - now supports *splicing*. - - -Compiler Additions ------------------- - -- The compiler can generate C++ code for easier interfacing with C++. -- The compiler can generate Objective C code for easier interfacing with - Objective C. -- The new pragmas ``importcpp`` and ``importobjc`` make interfacing with C++ - and Objective C somewhat easier. -- Added a new pragma ``incompleteStruct`` to deal with incomplete C struct - definitions. -- Added a ``--nimcache:PATH`` configuration option for control over the output - directory for generated code. -- The ``--genScript`` option now produces different compilation scripts - which do not contain absolute paths. -- Added ``--cincludes:dir``, ``--clibdir:lib`` configuration options for - modifying the C compiler's header/library search path in cross-platform way. -- Added ``--clib:lib`` configuration option for specifying additional - C libraries to be linked. -- Added ``--mainmodule:file`` configuration options for specifying the main - project file. This is intended to be used in project configuration files to - allow commands like ``nimrod c`` or ``nimrod check`` to be executed anywhere - within the project's directory structure. -- Added a ``--app:staticlib`` option for creating static libraries. -- Added a ``--tlsEmulation:on|off`` switch for control over thread local - storage emulation. -- The compiler and standard library now support a *taint mode*. Input strings - are declared with the ``TaintedString`` string type. If the taint - mode is turned on it is a distinct string type which helps to detect input - validation errors. -- The compiler now supports the compilation cache via ``--symbolFiles:on``. - This potentially speeds up compilations by an order of magnitude, but is - still highly experimental! -- Added ``--import:file`` and ``--include:file`` configuration options - for specifying modules that will be automatically imported/incluced. -- ``nimrod i`` can now optionally be given a module to execute. -- The compiler now performs a simple alias analysis to generate better code. -- The compiler and ENDB now support *watchpoints*. -- The compiler now supports proper compile time expressions of type ``bool`` - for ``on|off`` switches in pragmas. In order to not break existing code, - ``on`` and ``off`` are now aliases for ``true`` and ``false`` and declared - in the system module. -- The compiler finally supports **closures**. This is a preliminary - implementation, which does not yet support nestings deeper than 1 level - and still has many known bugs. - - -Library Additions ------------------ - -- Added ``system.allocShared``, ``system.allocShared0``, - ``system.deallocShared``, ``system.reallocShared``. -- Slicing as implemented by the system module now supports *splicing*. -- Added explicit channels for thread communication. -- Added ``matchers`` module for email address etc. matching. -- Added ``strutils.unindent``, ``strutils.countLines``, - ``strutils.replaceWord``. -- Added ``system.slurp`` for easy resource embedding. -- Added ``system.running`` for threads. -- Added ``system.programResult``. -- Added ``xmltree.innerText``. -- Added ``os.isAbsolute``, ``os.dynLibFormat``, ``os.isRootDir``, - ``os.parentDirs``. -- Added ``parseutils.interpolatedFragments``. -- Added ``macros.treeRepr``, ``macros.lispRepr``, ``macros.dumpTree``, - ``macros.dumpLisp``, ``macros.parseExpr``, ``macros.parseStmt``, - ``macros.getAst``. -- Added ``locks`` core module for more flexible locking support. -- Added ``irc`` module. -- Added ``ftpclient`` module. -- Added ``memfiles`` module. -- Added ``subexes`` module. -- Added ``critbits`` module. -- Added ``asyncio`` module. -- Added ``actors`` module. -- Added ``algorithm`` module for generic ``sort``, ``reverse`` etc. operations. -- Added ``osproc.startCmd``, ``osproc.execCmdEx``. -- The ``osproc`` module now uses ``posix_spawn`` instead of ``fork`` - and ``exec`` on Posix systems. Define the symbol ``useFork`` to revert to - the old implementation. -- Added ``intsets.assign``. -- Added ``system.astToStr`` and ``system.rand``, ``system.doAssert``. -- Added ``system.pairs`` for built-in types like arrays and strings. - - -2011-07-10 Version 0.8.12 released -================================== - -Bugfixes --------- - -- Bugfix: ``httpclient`` correct passes the path starting with ``/``. -- Bugfixes for the ``htmlparser`` module. -- Bugfix: ``pegs.find`` did not respect ``start`` parameter. -- Bugfix: ``dialogs.ChooseFilesToOpen`` did not work if only one file is - selected. -- Bugfix: niminst: ``nimrod`` is not default dir for *every* project. -- Bugfix: Multiple yield statements in iterators did not cause local vars to be - copied. -- Bugfix: The compiler does not emit very inaccurate floating point literals - anymore. -- Bugfix: Subclasses are taken into account for ``try except`` matching. -- Bugfix: Generics and macros are more stable. There are still known bugs left - though. -- Bugfix: Generated type information for tuples was sometimes wrong, causing - random crashes. -- Lots of other bugfixes: Too many to list them all. - - -Changes affecting backwards compatibility ------------------------------------------ - -- Operators starting with ``^`` are now right-associative and have the highest - priority. -- Deprecated ``os.getApplicationFilename``: Use ``os.getAppFilename`` instead. -- Deprecated ``os.getApplicationDir``: Use ``os.getAppDir`` instead. -- Deprecated ``system.copy``: Use ``substr`` or string slicing instead. -- Changed and documented how generalized string literals work: The syntax - ``module.re"abc"`` is now supported. -- Changed the behaviour of ``strutils.%``, ``ropes.%`` - if both notations ``$#`` and ``$i`` are involved. -- The ``pegs`` and ``re`` modules distinguish between ``replace`` - and ``replacef`` operations. -- The pointer dereference operation ``p^`` is deprecated and might become - ``^p`` in later versions or be dropped entirely since it is rarely used. - Use the new notation ``p[]`` in the rare cases where you need to - dereference a pointer explicitly. -- ``system.readFile`` does not return ``nil`` anymore but raises an ``EIO`` - exception instead. -- Unsound co-/contravariance for procvars has been removed. - - -Language Additions ------------------- - -- Source code filters are now documented. -- Added the ``linearScanEnd``, ``unroll``, ``shallow`` pragmas. -- Added ``emit`` pragma for direct code generator control. -- Case statement branches support constant sets for programming convenience. -- Tuple unpacking is not enforced in ``for`` loops anymore. -- The compiler now supports array, sequence and string slicing. -- A field in an ``enum`` may be given an explicit string representation. - This yields more maintainable code than using a constant - ``array[TMyEnum, string]`` mapping. -- Indices in array literals may be explicitly given, enhancing readability: - ``[enumValueA: "a", enumValueB: "b"]``. -- Added thread support via the ``threads`` core module and - the ``--threads:on`` command line switch. -- The built-in iterators ``system.fields`` and ``system.fieldPairs`` can be - used to iterate over any field of a tuple. With this mechanism operations - like ``==`` and ``hash`` are lifted to tuples. -- The slice ``..`` is now a first-class operator, allowing code like: - ``x in 1000..100_000``. - - -Compiler Additions ------------------- - -- The compiler supports IDEs via the new group of ``idetools`` command line - options. -- The *interactive mode* (REPL) has been improved and documented for the - first time. -- The compiler now might use hashing for string case statements depending - on the number of string literals in the case statement. - - -Library Additions ------------------ - -- Added ``lists`` module which contains generic linked lists. -- Added ``sets`` module which contains generic hash sets. -- Added ``tables`` module which contains generic hash tables. -- Added ``queues`` module which contains generic sequence based queues. -- Added ``intsets`` module which contains a specialized int set data type. -- Added ``scgi`` module. -- Added ``smtp`` module. -- Added ``encodings`` module. -- Added ``re.findAll``, ``pegs.findAll``. -- Added ``os.findExe``. -- Added ``parseutils.parseUntil`` and ``parseutils.parseWhile``. -- Added ``strutils.align``, ``strutils.tokenize``, ``strutils.wordWrap``. -- Pegs support a *captured search loop operator* ``{@}``. -- Pegs support new built-ins: ``\letter``, ``\upper``, ``\lower``, - ``\title``, ``\white``. -- Pegs support the new built-in ``\skip`` operation. -- Pegs support the ``$`` and ``^`` anchors. -- Additional operations were added to the ``complex`` module. -- Added ``strutils.formatFloat``, ``strutils.formatBiggestFloat``. -- Added unary ``<`` for nice looking excluding upper bounds in ranges. -- Added ``math.floor``. -- Added ``system.reset`` and a version of ``system.open`` that - returns a ``TFile`` and raises an exception in case of an error. -- Added a wrapper for ``redis``. -- Added a wrapper for ``0mq`` via the ``zmq`` module. -- Added a wrapper for ``sphinx``. -- Added ``system.newStringOfCap``. -- Added ``system.raiseHook`` and ``system.outOfMemHook``. -- Added ``system.writeFile``. -- Added ``system.shallowCopy``. -- ``system.echo`` is guaranteed to be thread-safe. -- Added ``prelude`` include file for scripting convenience. -- Added ``typeinfo`` core module for access to runtime type information. -- Added ``marshal`` module for JSON serialization. - - -2010-10-20 Version 0.8.10 released -================================== - -Bugfixes --------- - -- Bugfix: Command line parsing on Windows and ``os.parseCmdLine`` now adheres - to the same parsing rules as Microsoft's C/C++ startup code. -- Bugfix: Passing a ``ref`` pointer to the untyped ``pointer`` type is invalid. -- Bugfix: Updated ``keyval`` example. -- Bugfix: ``system.splitChunk`` still contained code for debug output. -- Bugfix: ``dialogs.ChooseFileToSave`` uses ``STOCK_SAVE`` instead of - ``STOCK_OPEN`` for the GTK backend. -- Bugfix: Various bugs concerning exception handling fixed. -- Bugfix: ``low(somestring)`` crashed the compiler. -- Bugfix: ``strutils.endsWith`` lacked range checking. -- Bugfix: Better detection for AMD64 on Mac OS X. - - -Changes affecting backwards compatibility ------------------------------------------ - -- Reversed parameter order for ``os.copyFile`` and ``os.moveFile``!!! -- Procs not marked as ``procvar`` cannot only be passed to a procvar anymore, - unless they are used in the same module. -- Deprecated ``times.getStartMilsecs``: Use ``epochTime`` or ``cpuTime`` - instead. -- Removed ``system.OpenFile``. -- Removed ``system.CloseFile``. -- Removed ``strutils.replaceStr``. -- Removed ``strutils.deleteStr``. -- Removed ``strutils.splitLinesSeq``. -- Removed ``strutils.splitSeq``. -- Removed ``strutils.toString``. -- If a DLL cannot be loaded (via the ``dynlib`` pragma) ``EInvalidLibrary`` - is not raised anymore. Instead ``system.quit()`` is called. This is because - raising an exception requires heap allocations. However the memory manager - might be contained in the DLL that failed to load. -- The ``re`` module (and the ``pcre`` wrapper) now depend on the pcre dll. - - -Additions ---------- - -- The ``{.compile: "file.c".}`` pragma uses a CRC check to see if the file - needs to be recompiled. -- Added ``system.reopen``. -- Added ``system.getCurrentException``. -- Added ``system.appType``. -- Added ``system.compileOption``. -- Added ``times.epochTime`` and ``times.cpuTime``. -- Implemented explicit type arguments for generics. -- Implemented ``{.size: sizeof(cint).}`` pragma for enum types. This is useful - for interfacing with C. -- Implemented ``{.pragma.}`` pragma for user defined pragmas. -- Implemented ``{.extern.}`` pragma for better control of name mangling. -- The ``importc`` and ``exportc`` pragmas support format strings: - ``proc p{.exportc: "nim_$1".}`` exports ``p`` as ``nim_p``. This is useful - for user defined pragmas. -- The standard library can be built as a DLL. Generating DLLs has been - improved. -- Added ``expat`` module. -- Added ``json`` module. -- Added support for a *Tiny C* backend. Currently this only works on Linux. - You need to bootstrap with ``-d:tinyc`` to enable Tiny C support. Nimrod - can then execute code directly via ``nimrod run myfile``. - - -2010-03-14 Version 0.8.8 released -================================= - -Bugfixes --------- -- The Posix version of ``os.copyFile`` has better error handling. -- Fixed bug #502670 (underscores in identifiers). -- Fixed a bug in the ``parsexml`` module concerning the parsing of - ``<tag attr="value" />``. -- Fixed a bug in the ``parsexml`` module concerning the parsing of - enities like ``<XX``. -- ``system.write(f: TFile, s: string)`` now works even if ``s`` contains binary - zeros. -- Fixed a bug in ``os.setFilePermissions`` for Windows. -- An overloadable symbol can now have the same name as an imported module. -- Fixed a serious bug in ``strutils.cmpIgnoreCase``. -- Fixed ``unicode.toUTF8``. -- The compiler now rejects ``'\n'`` (use ``"\n"`` instead). -- ``times.getStartMilsecs()`` now works on Mac OS X. -- Fixed a bug in ``pegs.match`` concerning start offsets. -- Lots of other little bugfixes. - - -Additions ---------- -- Added ``system.cstringArrayToSeq``. -- Added ``system.lines(f: TFile)`` iterator. -- Added ``system.delete``, ``system.del`` and ``system.insert`` for sequences. -- Added ``system./`` for int. -- Exported ``system.newException`` template. -- Added ``cgi.decodeData(data: string): tuple[key, value: string]``. -- Added ``strutils.insertSep``. -- Added ``math.trunc``. -- Added ``ropes`` module. -- Added ``sockets`` module. -- Added ``browsers`` module. -- Added ``httpserver`` module. -- Added ``httpclient`` module. -- Added ``parseutils`` module. -- Added ``unidecode`` module. -- Added ``xmldom`` module. -- Added ``xmldomparser`` module. -- Added ``xmltree`` module. -- Added ``xmlparser`` module. -- Added ``htmlparser`` module. -- Added ``re`` module. -- Added ``graphics`` module. -- Added ``colors`` module. -- Many wrappers now do not contain redundant name prefixes (like ``GTK_``, - ``lua``). The old wrappers are still available in ``lib/oldwrappers``. - You can change your configuration file to use these. -- Triple quoted strings allow for ``"`` in more contexts. -- ``""`` within raw string literals stands for a single quotation mark. -- Arguments to ``openArray`` parameters can be left out. -- More extensive subscript operator overloading. (To be documented.) -- The documentation generator supports the ``.. raw:: html`` directive. -- The Pegs module supports back references via the notation ``$capture_index``. - - -Changes affecting backwards compatibility ------------------------------------------ - -- Overloading of the subscript operator only works if the type does not provide - a built-in one. -- The search order for libraries which is affected by the ``path`` option - has been reversed, so that the project's path is searched before - the standard library's path. -- The compiler does not include a Pascal parser for bootstrapping purposes any - more. Instead there is a ``pas2nim`` tool that contains the old functionality. -- The procs ``os.copyFile`` and ``os.moveFile`` have been deprecated - temporarily, so that the compiler warns about their usage. Use them with - named arguments only, because the parameter order will change the next - version! -- ``atomic`` and ``let`` are now keywords. -- The ``\w`` character class for pegs now includes the digits ``'0'..'9'``. -- Many wrappers now do not contain redundant name prefixes (like ``GTK_``, - ``lua``) anymore. -- Arguments to ``openArray`` parameters can be left out. - - -2009-12-21 Version 0.8.6 released -================================= - -The version jump from 0.8.2 to 0.8.6 acknowledges the fact that all development -of the compiler is now done in Nimrod. - - -Bugfixes --------- -- The pragmas ``hint[X]:off`` and ``warning[X]:off`` now work. -- Method call syntax for iterators works again (``for x in lines.split()``). -- Fixed a typo in ``removeDir`` for POSIX that lead to an infinite recursion. -- The compiler now checks that module filenames are valid identifiers. -- Empty patterns for the ``dynlib`` pragma are now possible. -- ``os.parseCmdLine`` returned wrong results for trailing whitespace. -- Inconsequent tuple usage (using the same tuple with and without named fields) - does not crash the code generator anymore. -- A better error message is provided when the loading of a proc within a - dynamic lib fails. - - -Additions ---------- -- Added ``system.contains`` for open arrays. -- The PEG module now supports the *search loop operator* ``@``. -- Grammar/parser: ``SAD|IND`` is allowed before any kind of closing bracket. - This allows for more flexible source code formating. -- The compiler now uses a *bind* table for symbol lookup within a ``bind`` - context. (See `<manual.html#templates>`_ for details.) -- ``discard """my long comment"""`` is now optimized away. -- New ``--floatChecks: on|off`` switches and pragmas for better debugging - of floating point operations. (See - `<manual.html#pre-defined-floating-point-types>`_ for details.) -- The manual has been improved. (Many thanks to Philippe Lhoste!) - - -Changes affecting backwards compatibility ------------------------------------------ -- The compiler does not skip the linking step anymore even if no file - has changed. -- ``os.splitFile(".xyz")`` now returns ``("", ".xyz", "")`` instead of - ``("", "", ".xyz")``. So filenames starting with a dot are handled - differently. -- ``strutils.split(s: string, seps: set[char])`` never yields the empty string - anymore. This behaviour is probably more appropriate for whitespace splitting. -- The compiler now stops after the ``--version`` command line switch. -- Removed support for enum inheritance in the parser; enum inheritance has - never been documented anyway. -- The ``msg`` field of ``system.E_base`` has now the type ``string``, instead - of ``cstring``. This improves memory safety. - - -2009-10-21 Version 0.8.2 released -================================= - - -2009-09-12 Version 0.8.0 released -================================= - - -2009-06-08 Version 0.7.10 released -================================== - - -2009-05-08 Version 0.7.8 released -================================= - - -2009-04-22 Version 0.7.6 released -================================= - - -2008-11-16 Version 0.7.0 released -================================= - - -2008-08-22 Version 0.6.0 released -================================= - -Nimrod version 0.6.0 has been released! -**This is the first version of the compiler that is able to compile itself!** diff --git a/web/news/2009_12_21_version_0_8_6_released.rst b/web/news/2009_12_21_version_0_8_6_released.rst new file mode 100644 index 000000000..019168a44 --- /dev/null +++ b/web/news/2009_12_21_version_0_8_6_released.rst @@ -0,0 +1,54 @@ +2009-12-21 Version 0.8.6 released +================================= + +.. container:: metadata + + Posted by Andreas Rumpf on 21/12/2009 + +The version jump from 0.8.2 to 0.8.6 acknowledges the fact that all development +of the compiler is now done in Nimrod. + + +Bugfixes +-------- +- The pragmas ``hint[X]:off`` and ``warning[X]:off`` now work. +- Method call syntax for iterators works again (``for x in lines.split()``). +- Fixed a typo in ``removeDir`` for POSIX that lead to an infinite recursion. +- The compiler now checks that module filenames are valid identifiers. +- Empty patterns for the ``dynlib`` pragma are now possible. +- ``os.parseCmdLine`` returned wrong results for trailing whitespace. +- Inconsequent tuple usage (using the same tuple with and without named fields) + does not crash the code generator anymore. +- A better error message is provided when the loading of a proc within a + dynamic lib fails. + + +Additions +--------- +- Added ``system.contains`` for open arrays. +- The PEG module now supports the *search loop operator* ``@``. +- Grammar/parser: ``SAD|IND`` is allowed before any kind of closing bracket. + This allows for more flexible source code formating. +- The compiler now uses a *bind* table for symbol lookup within a ``bind`` + context. (See `<manual.html#templates>`_ for details.) +- ``discard """my long comment"""`` is now optimized away. +- New ``--floatChecks: on|off`` switches and pragmas for better debugging + of floating point operations. (See + `<manual.html#pre-defined-floating-point-types>`_ for details.) +- The manual has been improved. (Many thanks to Philippe Lhoste!) + + +Changes affecting backwards compatibility +----------------------------------------- +- The compiler does not skip the linking step anymore even if no file + has changed. +- ``os.splitFile(".xyz")`` now returns ``("", ".xyz", "")`` instead of + ``("", "", ".xyz")``. So filenames starting with a dot are handled + differently. +- ``strutils.split(s: string, seps: set[char])`` never yields the empty string + anymore. This behaviour is probably more appropriate for whitespace splitting. +- The compiler now stops after the ``--version`` command line switch. +- Removed support for enum inheritance in the parser; enum inheritance has + never been documented anyway. +- The ``msg`` field of ``system.E_base`` has now the type ``string``, instead + of ``cstring``. This improves memory safety. diff --git a/web/news/2010_03_14_version_0_8_8_released.rst b/web/news/2010_03_14_version_0_8_8_released.rst new file mode 100644 index 000000000..2df476814 --- /dev/null +++ b/web/news/2010_03_14_version_0_8_8_released.rst @@ -0,0 +1,82 @@ +Version 0.8.8 released +====================== + +.. container:: metadata + + Posted by Andreas Rumpf on 14/03/2010 + +Bugfixes +-------- +- The Posix version of ``os.copyFile`` has better error handling. +- Fixed bug #502670 (underscores in identifiers). +- Fixed a bug in the ``parsexml`` module concerning the parsing of + ``<tag attr="value" />``. +- Fixed a bug in the ``parsexml`` module concerning the parsing of + enities like ``<XX``. +- ``system.write(f: TFile, s: string)`` now works even if ``s`` contains binary + zeros. +- Fixed a bug in ``os.setFilePermissions`` for Windows. +- An overloadable symbol can now have the same name as an imported module. +- Fixed a serious bug in ``strutils.cmpIgnoreCase``. +- Fixed ``unicode.toUTF8``. +- The compiler now rejects ``'\n'`` (use ``"\n"`` instead). +- ``times.getStartMilsecs()`` now works on Mac OS X. +- Fixed a bug in ``pegs.match`` concerning start offsets. +- Lots of other little bugfixes. + + +Additions +--------- +- Added ``system.cstringArrayToSeq``. +- Added ``system.lines(f: TFile)`` iterator. +- Added ``system.delete``, ``system.del`` and ``system.insert`` for sequences. +- Added ``system./`` for int. +- Exported ``system.newException`` template. +- Added ``cgi.decodeData(data: string): tuple[key, value: string]``. +- Added ``strutils.insertSep``. +- Added ``math.trunc``. +- Added ``ropes`` module. +- Added ``sockets`` module. +- Added ``browsers`` module. +- Added ``httpserver`` module. +- Added ``httpclient`` module. +- Added ``parseutils`` module. +- Added ``unidecode`` module. +- Added ``xmldom`` module. +- Added ``xmldomparser`` module. +- Added ``xmltree`` module. +- Added ``xmlparser`` module. +- Added ``htmlparser`` module. +- Added ``re`` module. +- Added ``graphics`` module. +- Added ``colors`` module. +- Many wrappers now do not contain redundant name prefixes (like ``GTK_``, + ``lua``). The old wrappers are still available in ``lib/oldwrappers``. + You can change your configuration file to use these. +- Triple quoted strings allow for ``"`` in more contexts. +- ``""`` within raw string literals stands for a single quotation mark. +- Arguments to ``openArray`` parameters can be left out. +- More extensive subscript operator overloading. (To be documented.) +- The documentation generator supports the ``.. raw:: html`` directive. +- The Pegs module supports back references via the notation ``$capture_index``. + + +Changes affecting backwards compatibility +----------------------------------------- + +- Overloading of the subscript operator only works if the type does not provide + a built-in one. +- The search order for libraries which is affected by the ``path`` option + has been reversed, so that the project's path is searched before + the standard library's path. +- The compiler does not include a Pascal parser for bootstrapping purposes any + more. Instead there is a ``pas2nim`` tool that contains the old functionality. +- The procs ``os.copyFile`` and ``os.moveFile`` have been deprecated + temporarily, so that the compiler warns about their usage. Use them with + named arguments only, because the parameter order will change the next + version! +- ``atomic`` and ``let`` are now keywords. +- The ``\w`` character class for pegs now includes the digits ``'0'..'9'``. +- Many wrappers now do not contain redundant name prefixes (like ``GTK_``, + ``lua``) anymore. +- Arguments to ``openArray`` parameters can be left out. diff --git a/web/news/2010_10_20_version_0_8_10_released.rst b/web/news/2010_10_20_version_0_8_10_released.rst new file mode 100644 index 000000000..f72c0076c --- /dev/null +++ b/web/news/2010_10_20_version_0_8_10_released.rst @@ -0,0 +1,70 @@ +Version 0.8.10 released +======================= + +.. container:: metadata + + Posted by Andreas Rumpf on 20/10/2010 + +Bugfixes +-------- + +- Bugfix: Command line parsing on Windows and ``os.parseCmdLine`` now adheres + to the same parsing rules as Microsoft's C/C++ startup code. +- Bugfix: Passing a ``ref`` pointer to the untyped ``pointer`` type is invalid. +- Bugfix: Updated ``keyval`` example. +- Bugfix: ``system.splitChunk`` still contained code for debug output. +- Bugfix: ``dialogs.ChooseFileToSave`` uses ``STOCK_SAVE`` instead of + ``STOCK_OPEN`` for the GTK backend. +- Bugfix: Various bugs concerning exception handling fixed. +- Bugfix: ``low(somestring)`` crashed the compiler. +- Bugfix: ``strutils.endsWith`` lacked range checking. +- Bugfix: Better detection for AMD64 on Mac OS X. + + +Changes affecting backwards compatibility +----------------------------------------- + +- Reversed parameter order for ``os.copyFile`` and ``os.moveFile``!!! +- Procs not marked as ``procvar`` cannot only be passed to a procvar anymore, + unless they are used in the same module. +- Deprecated ``times.getStartMilsecs``: Use ``epochTime`` or ``cpuTime`` + instead. +- Removed ``system.OpenFile``. +- Removed ``system.CloseFile``. +- Removed ``strutils.replaceStr``. +- Removed ``strutils.deleteStr``. +- Removed ``strutils.splitLinesSeq``. +- Removed ``strutils.splitSeq``. +- Removed ``strutils.toString``. +- If a DLL cannot be loaded (via the ``dynlib`` pragma) ``EInvalidLibrary`` + is not raised anymore. Instead ``system.quit()`` is called. This is because + raising an exception requires heap allocations. However the memory manager + might be contained in the DLL that failed to load. +- The ``re`` module (and the ``pcre`` wrapper) now depend on the pcre dll. + + +Additions +--------- + +- The ``{.compile: "file.c".}`` pragma uses a CRC check to see if the file + needs to be recompiled. +- Added ``system.reopen``. +- Added ``system.getCurrentException``. +- Added ``system.appType``. +- Added ``system.compileOption``. +- Added ``times.epochTime`` and ``times.cpuTime``. +- Implemented explicit type arguments for generics. +- Implemented ``{.size: sizeof(cint).}`` pragma for enum types. This is useful + for interfacing with C. +- Implemented ``{.pragma.}`` pragma for user defined pragmas. +- Implemented ``{.extern.}`` pragma for better control of name mangling. +- The ``importc`` and ``exportc`` pragmas support format strings: + ``proc p{.exportc: "nim_$1".}`` exports ``p`` as ``nim_p``. This is useful + for user defined pragmas. +- The standard library can be built as a DLL. Generating DLLs has been + improved. +- Added ``expat`` module. +- Added ``json`` module. +- Added support for a *Tiny C* backend. Currently this only works on Linux. + You need to bootstrap with ``-d:tinyc`` to enable Tiny C support. Nimrod + can then execute code directly via ``nimrod run myfile``. diff --git a/web/news/2011_07_10_version_0_8_12_released.rst b/web/news/2011_07_10_version_0_8_12_released.rst new file mode 100644 index 000000000..5f154b2db --- /dev/null +++ b/web/news/2011_07_10_version_0_8_12_released.rst @@ -0,0 +1,122 @@ +Version 0.8.12 released +================================== + +.. container:: metadata + + Posted by Andreas Rumpf on 10/07/2011 + +Bugfixes +-------- + +- Bugfix: ``httpclient`` correct passes the path starting with ``/``. +- Bugfixes for the ``htmlparser`` module. +- Bugfix: ``pegs.find`` did not respect ``start`` parameter. +- Bugfix: ``dialogs.ChooseFilesToOpen`` did not work if only one file is + selected. +- Bugfix: niminst: ``nimrod`` is not default dir for *every* project. +- Bugfix: Multiple yield statements in iterators did not cause local vars to be + copied. +- Bugfix: The compiler does not emit very inaccurate floating point literals + anymore. +- Bugfix: Subclasses are taken into account for ``try except`` matching. +- Bugfix: Generics and macros are more stable. There are still known bugs left + though. +- Bugfix: Generated type information for tuples was sometimes wrong, causing + random crashes. +- Lots of other bugfixes: Too many to list them all. + + +Changes affecting backwards compatibility +----------------------------------------- + +- Operators starting with ``^`` are now right-associative and have the highest + priority. +- Deprecated ``os.getApplicationFilename``: Use ``os.getAppFilename`` instead. +- Deprecated ``os.getApplicationDir``: Use ``os.getAppDir`` instead. +- Deprecated ``system.copy``: Use ``substr`` or string slicing instead. +- Changed and documented how generalized string literals work: The syntax + ``module.re"abc"`` is now supported. +- Changed the behaviour of ``strutils.%``, ``ropes.%`` + if both notations ``$#`` and ``$i`` are involved. +- The ``pegs`` and ``re`` modules distinguish between ``replace`` + and ``replacef`` operations. +- The pointer dereference operation ``p^`` is deprecated and might become + ``^p`` in later versions or be dropped entirely since it is rarely used. + Use the new notation ``p[]`` in the rare cases where you need to + dereference a pointer explicitly. +- ``system.readFile`` does not return ``nil`` anymore but raises an ``EIO`` + exception instead. +- Unsound co-/contravariance for procvars has been removed. + + +Language Additions +------------------ + +- Source code filters are now documented. +- Added the ``linearScanEnd``, ``unroll``, ``shallow`` pragmas. +- Added ``emit`` pragma for direct code generator control. +- Case statement branches support constant sets for programming convenience. +- Tuple unpacking is not enforced in ``for`` loops anymore. +- The compiler now supports array, sequence and string slicing. +- A field in an ``enum`` may be given an explicit string representation. + This yields more maintainable code than using a constant + ``array[TMyEnum, string]`` mapping. +- Indices in array literals may be explicitly given, enhancing readability: + ``[enumValueA: "a", enumValueB: "b"]``. +- Added thread support via the ``threads`` core module and + the ``--threads:on`` command line switch. +- The built-in iterators ``system.fields`` and ``system.fieldPairs`` can be + used to iterate over any field of a tuple. With this mechanism operations + like ``==`` and ``hash`` are lifted to tuples. +- The slice ``..`` is now a first-class operator, allowing code like: + ``x in 1000..100_000``. + + +Compiler Additions +------------------ + +- The compiler supports IDEs via the new group of ``idetools`` command line + options. +- The *interactive mode* (REPL) has been improved and documented for the + first time. +- The compiler now might use hashing for string case statements depending + on the number of string literals in the case statement. + + +Library Additions +----------------- + +- Added ``lists`` module which contains generic linked lists. +- Added ``sets`` module which contains generic hash sets. +- Added ``tables`` module which contains generic hash tables. +- Added ``queues`` module which contains generic sequence based queues. +- Added ``intsets`` module which contains a specialized int set data type. +- Added ``scgi`` module. +- Added ``smtp`` module. +- Added ``encodings`` module. +- Added ``re.findAll``, ``pegs.findAll``. +- Added ``os.findExe``. +- Added ``parseutils.parseUntil`` and ``parseutils.parseWhile``. +- Added ``strutils.align``, ``strutils.tokenize``, ``strutils.wordWrap``. +- Pegs support a *captured search loop operator* ``{@}``. +- Pegs support new built-ins: ``\letter``, ``\upper``, ``\lower``, + ``\title``, ``\white``. +- Pegs support the new built-in ``\skip`` operation. +- Pegs support the ``$`` and ``^`` anchors. +- Additional operations were added to the ``complex`` module. +- Added ``strutils.formatFloat``, ``strutils.formatBiggestFloat``. +- Added unary ``<`` for nice looking excluding upper bounds in ranges. +- Added ``math.floor``. +- Added ``system.reset`` and a version of ``system.open`` that + returns a ``TFile`` and raises an exception in case of an error. +- Added a wrapper for ``redis``. +- Added a wrapper for ``0mq`` via the ``zmq`` module. +- Added a wrapper for ``sphinx``. +- Added ``system.newStringOfCap``. +- Added ``system.raiseHook`` and ``system.outOfMemHook``. +- Added ``system.writeFile``. +- Added ``system.shallowCopy``. +- ``system.echo`` is guaranteed to be thread-safe. +- Added ``prelude`` include file for scripting convenience. +- Added ``typeinfo`` core module for access to runtime type information. +- Added ``marshal`` module for JSON serialization. diff --git a/web/news/2012_02_09_version_0_8_14_released.rst b/web/news/2012_02_09_version_0_8_14_released.rst new file mode 100644 index 000000000..4050c8b93 --- /dev/null +++ b/web/news/2012_02_09_version_0_8_14_released.rst @@ -0,0 +1,168 @@ +2012-02-09 Version 0.8.14 released +================================== + +.. container:: metadata + + Posted by Andreas Rumpf on 09/02/2012 + +Version 0.8.14 has been released! + +Bugfixes +-------- + +- Fixed a serious memory corruption concerning message passing. +- Fixed a serious bug concerning different instantiations of a generic proc. +- Fixed a newly introduced bug where a wrong ``EIO`` exception was raised for + the end of file for text files that do not end with a newline. +- Bugfix c2nim, c2pas: the ``--out`` option has never worked properly. +- Bugfix: forwarding of generic procs never worked. +- Some more bugfixes for macros and compile-time evaluation. +- The GC now takes into account interior pointers on the stack which may be + introduced by aggressive C optimizers. +- Nimrod's native allocator/GC now works on PowerPC. +- Lots of other bugfixes: Too many to list them all. + + +Changes affecting backwards compatibility +----------------------------------------- + +- Removed deprecated ``os.AppendFileExt``, ``os.executeShellCommand``, + ``os.iterOverEnvironment``, ``os.pcDirectory``, ``os.pcLinkToDirectory``, + ``os.SplitPath``, ``os.extractDir``, ``os.SplitFilename``, + ``os.extractFileTrunk``, ``os.extractFileExt``, ``osproc.executeProcess``, + ``osproc.executeCommand``. +- Removed deprecated ``parseopt.init``, ``parseopt.getRestOfCommandLine``. +- Moved ``strutils.validEmailAddress`` to ``matchers.validEmailAddress``. +- The pointer dereference operator ``^`` has been removed, so that ``^`` + can now be a user-defined operator. +- ``implies`` is no keyword anymore. +- The ``is`` operator is now the ``of`` operator. +- The ``is`` operator is now used to check type equivalence in generic code. +- The ``pure`` pragma for procs has been renamed to ``noStackFrame``. +- The threading API has been completely redesigned. +- The ``unidecode`` module is now thread-safe and its interface has changed. +- The ``bind`` expression is deprecated, use a ``bind`` declaration instead. +- ``system.raiseHook`` is now split into ``system.localRaiseHook`` and + ``system.globalRaiseHook`` to distinguish between thread local and global + raise hooks. +- Changed exception handling/error reporting for ``os.removeFile`` and + ``os.removeDir``. +- The algorithm for searching and loading configuration files has been changed. +- Operators now have diffent precedence rules: Assignment-like operators + (like ``*=``) are now special-cased. +- The fields in ``TStream`` have been renamed to have an ``Impl`` suffix + because they should not be used directly anymore. + Wrapper procs have been created that should be used instead. +- ``export`` is now a keyword. +- ``assert`` is now implemented in pure Nimrod as a template; it's easy + to implement your own assertion templates with ``system.astToStr``. + + +Language Additions +------------------ + +- Added new ``is`` and ``of`` operators. +- The built-in type ``void`` can be used to denote the absence of any type. + This is useful in generic code. +- Return types may be of the type ``var T`` to return an l-value. +- The error pragma can now be used to mark symbols whose *usage* should trigger + a compile-time error. +- There is a new ``discardable`` pragma that can be used to mark a routine + so that its result can be discarded implicitly. +- Added a new ``noinit`` pragma to prevent automatic initialization to zero + of variables. +- Constants can now have the type ``seq``. +- There is a new user-definable syntactic construct ``a{i, ...}`` + that has no semantics yet for built-in types and so can be overloaded to your + heart's content. +- ``bind`` (used for symbol binding in templates and generics) is now a + declarative statement. +- Nimrod now supports single assignment variables via the ``let`` statement. +- Iterators named ``items`` and ``pairs`` are implicitly invoked when + an explicit iterator is missing. +- The slice assignment ``a[i..j] = b`` where ``a`` is a sequence or string + now supports *splicing*. + + +Compiler Additions +------------------ + +- The compiler can generate C++ code for easier interfacing with C++. +- The compiler can generate Objective C code for easier interfacing with + Objective C. +- The new pragmas ``importcpp`` and ``importobjc`` make interfacing with C++ + and Objective C somewhat easier. +- Added a new pragma ``incompleteStruct`` to deal with incomplete C struct + definitions. +- Added a ``--nimcache:PATH`` configuration option for control over the output + directory for generated code. +- The ``--genScript`` option now produces different compilation scripts + which do not contain absolute paths. +- Added ``--cincludes:dir``, ``--clibdir:lib`` configuration options for + modifying the C compiler's header/library search path in cross-platform way. +- Added ``--clib:lib`` configuration option for specifying additional + C libraries to be linked. +- Added ``--mainmodule:file`` configuration options for specifying the main + project file. This is intended to be used in project configuration files to + allow commands like ``nimrod c`` or ``nimrod check`` to be executed anywhere + within the project's directory structure. +- Added a ``--app:staticlib`` option for creating static libraries. +- Added a ``--tlsEmulation:on|off`` switch for control over thread local + storage emulation. +- The compiler and standard library now support a *taint mode*. Input strings + are declared with the ``TaintedString`` string type. If the taint + mode is turned on it is a distinct string type which helps to detect input + validation errors. +- The compiler now supports the compilation cache via ``--symbolFiles:on``. + This potentially speeds up compilations by an order of magnitude, but is + still highly experimental! +- Added ``--import:file`` and ``--include:file`` configuration options + for specifying modules that will be automatically imported/incluced. +- ``nimrod i`` can now optionally be given a module to execute. +- The compiler now performs a simple alias analysis to generate better code. +- The compiler and ENDB now support *watchpoints*. +- The compiler now supports proper compile time expressions of type ``bool`` + for ``on|off`` switches in pragmas. In order to not break existing code, + ``on`` and ``off`` are now aliases for ``true`` and ``false`` and declared + in the system module. +- The compiler finally supports **closures**. This is a preliminary + implementation, which does not yet support nestings deeper than 1 level + and still has many known bugs. + + +Library Additions +----------------- + +- Added ``system.allocShared``, ``system.allocShared0``, + ``system.deallocShared``, ``system.reallocShared``. +- Slicing as implemented by the system module now supports *splicing*. +- Added explicit channels for thread communication. +- Added ``matchers`` module for email address etc. matching. +- Added ``strutils.unindent``, ``strutils.countLines``, + ``strutils.replaceWord``. +- Added ``system.slurp`` for easy resource embedding. +- Added ``system.running`` for threads. +- Added ``system.programResult``. +- Added ``xmltree.innerText``. +- Added ``os.isAbsolute``, ``os.dynLibFormat``, ``os.isRootDir``, + ``os.parentDirs``. +- Added ``parseutils.interpolatedFragments``. +- Added ``macros.treeRepr``, ``macros.lispRepr``, ``macros.dumpTree``, + ``macros.dumpLisp``, ``macros.parseExpr``, ``macros.parseStmt``, + ``macros.getAst``. +- Added ``locks`` core module for more flexible locking support. +- Added ``irc`` module. +- Added ``ftpclient`` module. +- Added ``memfiles`` module. +- Added ``subexes`` module. +- Added ``critbits`` module. +- Added ``asyncio`` module. +- Added ``actors`` module. +- Added ``algorithm`` module for generic ``sort``, ``reverse`` etc. operations. +- Added ``osproc.startCmd``, ``osproc.execCmdEx``. +- The ``osproc`` module now uses ``posix_spawn`` instead of ``fork`` + and ``exec`` on Posix systems. Define the symbol ``useFork`` to revert to + the old implementation. +- Added ``intsets.assign``. +- Added ``system.astToStr`` and ``system.rand``, ``system.doAssert``. +- Added ``system.pairs`` for built-in types like arrays and strings. diff --git a/web/news/2012_09_23_version_0_9_0_released.rst b/web/news/2012_09_23_version_0_9_0_released.rst new file mode 100644 index 000000000..5635ca94c --- /dev/null +++ b/web/news/2012_09_23_version_0_9_0_released.rst @@ -0,0 +1,182 @@ +Version 0.9.0 released +====================== + +.. container:: metadata + + Posted by Andreas Rumpf on 23/09/2012 + +Summary +------- + +* Unsigned integers have been added. +* The integer type promotion rules changed. +* The template and macro system evolved. +* Closures have been implemented. +* Term rewriting macros have been implemented. +* First steps to unify expressions and statements have been taken. +* Symbol lookup rules in generics have become stricter to catch more errors. + + +Bugfixes +-------- + +- Fixed a bug where the compiler would "optimize away" valid constant parts of + a string concatenation. +- Fixed a bug concerning implicit type conversions in ``case`` statements. +- Fixed a serious code generation bug that caused ``algorithm.sort`` to + produce segmentation faults. +- Fixed ambiguity in recvLine which meant that receiving ``\r\L`` was + indistinguishable from disconnections. +- Many more bugfixes, too many to list them all. + + +Library Additions +----------------- + +- Added the (already existing) module ``htmlgen`` to the documentation. +- Added the (already existing) module ``cookies`` to the documentation. +- Added ``system.shallow`` that can be used to speed up string and sequence + assignments. +- Added ``system.eval`` that can execute an anonymous block of code at + compile time as if was a macro. +- Added ``system.staticExec`` and ``system.gorge`` for compile-time execution + of external programs. +- Added ``system.staticRead`` as a synonym for ``system.slurp``. +- Added ``macros.emit`` that can emit an arbitrary computed string as nimrod + code during compilation. +- Added ``strutils.parseEnum``. +- Added ``json.%`` constructor operator. +- The stdlib can now be avoided to a point where C code generation for 16bit + micro controllers is feasible. +- Added module ``oids``. +- Added module ``endians``. +- Added a new OpenGL wrapper that supports OpenGL up to version 4.2. +- Added a wrapper for ``libsvm``. +- Added a wrapper for ``mongodb``. +- Added ``terminal.isatty``. +- Added an overload for ``system.items`` that can be used to iterate over the + values of an enum. +- Added ``system.TInteger`` and ``system.TNumber`` type classes matching + any of the corresponding types available in Nimrod. +- Added ``system.clamp`` to limit a value within an interval ``[a, b]``. +- Added ``strutils.continuesWith``. +- Added ``system.getStackTrace``. +- Added ``system.||`` for parallel ``for`` loop support. +- The GC supports (soft) realtime systems via ``GC_setMaxPause`` + and ``GC_step`` procs. +- The sockets module now supports ssl through the OpenSSL library, ``recvLine`` + is now much more efficient thanks to the newly implemented sockets buffering. +- The httpclient module now supports ssl/tls. +- Added ``times.format`` as well as many other utility functions + for managing time. +- Added ``system.@`` for converting an ``openarray`` to a ``seq`` (it used to + only support fixed length arrays). +- Added ``system.compiles`` which can be used to check whether a type supports + some operation. +- Added ``strutils.format``, ``subexes.format`` which use the + new ``varargs`` type. +- Added module ``fsmonitor``. + +Changes affecting backwards compatibility +----------------------------------------- + +- On Windows filenames and paths are supposed to be in UTF-8. + The ``system``, ``os``, ``osproc`` and ``memfiles`` modules use the wide + string versions of the WinAPI. Use the ``-d:useWinAnsi`` switch to revert + back to the old behaviour which uses the Ansi string versions. +- ``static``, ``do``, ``interface`` and ``mixin`` are now keywords. +- Templates now participate in overloading resolution which can break code that + uses templates in subtle ways. Use the new ``immediate`` pragma for templates + to get a template of old behaviour. +- There is now a proper distinction in the type system between ``expr`` and + ``PNimrodNode`` which unfortunately breaks the old macro system. +- ``pegs.@`` has been renamed to ``pegs.!*`` and ``pegs.@@`` has been renamed + to ``pegs.!*\`` as ``@`` operators now have different precedence. +- The type ``proc`` (without any params or return type) is now considered a + type class matching all proc types. Use ``proc ()`` to get the old meaning + denoting a proc expecing no arguments and returing no value. +- Deprecated ``system.GC_setStrategy``. +- ``re.findAll`` and ``pegs.findAll`` don't return *captures* anymore but + matching *substrings*. +- RTTI and thus the ``marshall`` module don't contain the proper field names + of tuples anymore. This had to be changed as the old behaviour never + produced consistent results. +- Deprecated the ``ssl`` module. +- Deprecated ``nimrod pretty`` as it never worked good enough and has some + inherent problems. +- The integer promotion rules changed; the compiler is now less picky in some + situations and more picky in other situations: In particular implicit + conversions from ``int`` to ``int32`` are now forbidden. +- ``system.byte`` is now an alias for ``uint8``; it used to be an alias + to ``int8``. +- ``bind`` expressions in templates are not properly supported anymore. Use + the declarative ``bind`` statement instead. +- The default calling convention for a procedural **type** is now ``closure``, + for procs it remains ``nimcall`` (which is compatible to ``closure``). + Activate the warning ``ImplicitClosure`` to make the compiler list the + occurrences of proc types which are affected. +- The Nimrod type system now distinguishes ``openarray`` from ``varargs``. +- Templates are now ``hygienic``. Use the ``dirty`` pragma to get the old + behaviour. +- Objects that have no ancestor are now implicitly ``final``. Use + the ``inheritable`` pragma to introduce new object roots apart + from ``TObject``. +- Macros now receive parameters like templates do; use the ``callsite`` builtin + to gain access to the invocation AST. +- Symbol lookup rules in generics have become stricter to catch more errors. + + +Compiler Additions +------------------ + +- Win64 is now an officially supported target. +- The Nimrod compiler works on BSD again, but has some issues + as ``os.getAppFilename`` and ``os.getAppDir`` cannot work reliably on BSD. +- The compiler can detect and evaluate calls that can be evaluated at compile + time for optimization purposes with the ``--implicitStatic`` command line + option or pragma. +- The compiler now generates marker procs that the GC can use instead of RTTI. + This speeds up the GC quite a bit. +- The compiler now includes a new advanced documentation generator + via the ``doc2`` command. This new generator uses all of the semantic passes + of the compiler and can thus generate documentation for symbols hiding in + macros. +- The compiler now supports the ``dynlib`` pragma for variables. +- The compiler now supports ``bycopy`` and ``byref`` pragmas that affect how + objects/tuples are passed. +- The embedded profiler became a stack trace profiler and has been documented. + + +Language Additions +------------------ + +- Added explicit ``static`` sections for enforced compile time evaluation. +- Added an alternative notation for lambdas with ``do``. +- ``addr`` is now treated like a prefix operator syntactically. +- Added ``global`` pragma that can be used to introduce new global variables + from within procs. +- ``when`` expressions are now allowed just like ``if`` expressions. +- The precedence for operators starting with ``@`` is different now + allowing for *sigil-like* operators. +- Stand-alone ``finally`` and ``except`` blocks are now supported. +- Macros and templates can now be invoked as pragmas. +- The apostrophe in type suffixes for numerical literals is now optional. +- Unsigned integer types have been added. +- The integer promotion rules changed. +- Nimrod now tracks proper intervals for ``range`` over some built-in operators. +- In parameter lists a semicolon instead of a comma can be used to improve + readability: ``proc divmod(a, b: int; resA, resB: var int)``. +- A semicolon can now be used to have multiple simple statements on a single + line: ``inc i; inc j``. +- ``bind`` supports overloaded symbols and operators. +- A ``distinct`` type can now borrow from generic procs. +- Added the pragmas ``gensym``, ``inject`` and ``dirty`` for hygiene + in templates. +- Comments can be continued with a backslash continuation character so that + comment pieces don't have to align on the same column. +- Enums can be annotated with ``pure`` so that their field names do not pollute + the current scope. +- A proc body can consist of an expression that has a type. This is rewritten + to ``result = expression`` then. +- Term rewriting macros (see `trmacros <http://nimrod-code.org/trmacros.html>`_) + have been implemented but are still in alpha. diff --git a/web/news/2013_05_20_new_website_design.rst b/web/news/2013_05_20_new_website_design.rst new file mode 100644 index 000000000..b36cc99dd --- /dev/null +++ b/web/news/2013_05_20_new_website_design.rst @@ -0,0 +1,13 @@ +New website design! +=================== + +.. container:: metadata + + Posted by Andreas Rumpf on 09/12/2014 + +A brand new website including an improved forum is now live. +All thanks go to Philip Witte and +Dominik Picheta, Philip Witte for the design of the website (together with +the logo) as well as the HTML and CSS code for his template, and Dominik Picheta +for integrating Philip's design with Nim's forum. We're sure you will +agree that Philip's design is beautiful. diff --git a/web/news/2013_05_20_version_0_9_2_released.rst b/web/news/2013_05_20_version_0_9_2_released.rst new file mode 100644 index 000000000..89352c06c --- /dev/null +++ b/web/news/2013_05_20_version_0_9_2_released.rst @@ -0,0 +1,118 @@ +Version 0.9.2 released +====================== + +.. container:: metadata + + Posted by Dominik Picheta on 20/05/2013 + +We are pleased to announce that version 0.9.2 of the Nimrod compiler has been +released. This release has attracted by far the most contributions in comparison +to any other release. + +This release brings with it many new features and bug fixes, a list of which +can be seen later. One of the major new features is the effect system together +with exception tracking which allows for checked exceptions and more, +for further details check out the `manual <manual.html#effect-system>`_. +Another major new feature is the introduction of statement list expressions, +more details on these can be found `here <manual.html#statement-list-expression>`_. +The ability to exclude symbols from modules has also been +implemented, this feature can be used like so: ``import module except symbol``. + +Thanks to all `contributors <https://github.com/Araq/Nimrod/contributors>`_! + +Bugfixes +-------- + +- The old GC never collected cycles correctly. Fixed but it can cause + performance regressions. However you can deactivate the cycle collector + with ``GC_disableMarkAndSweep`` and run it explicitly at an appropriate time + or not at all. There is also a new GC you can activate + with ``--gc:markAndSweep`` which does not have this problem but is slower in + general and has no realtime guarantees. +- ``cast`` for floating point types now does the bitcast as specified in the + manual. This breaks code that erroneously uses ``cast`` to convert different + floating point values. +- SCGI module's performance has been improved greatly, it will no longer block + on many concurrent requests. +- In total fixed over 70 github issues and merged over 60 pull requests. + + +Library Additions +----------------- + +- There is a new experimental mark&sweep GC which can be faster (or much + slower) than the default GC. Enable with ``--gc:markAndSweep``. +- Added ``system.onRaise`` to support a condition system. +- Added ``system.locals`` that provides access to a proc's locals. +- Added ``macros.quote`` for AST quasi-quoting. +- Added ``system.unsafeNew`` to support hacky variable length objects. +- ``system.fields`` and ``system.fieldPairs`` support ``object`` too; they + used to only support tuples. +- Added ``system.CurrentSourcePath`` returning the full file-system path of + the current source file. +- The ``macros`` module now contains lots of useful helpers for building up + abstract syntax trees. + + +Changes affecting backwards compatibility +----------------------------------------- + +- ``shared`` is a keyword now. +- Deprecated ``sockets.recvLine`` and ``asyncio.recvLine``, added + ``readLine`` instead. +- The way indentation is handled in the parser changed significantly. However, + this affects very little (if any) real world code. +- The expression/statement unification has been implemented. Again this + only affects edge cases and no known real world code. +- Changed the async interface of the ``scgi`` module. +- WideStrings are now garbage collected like other string types. + + +Compiler Additions +------------------ + +- The ``doc2`` command does not generate output for the whole project anymore. + Use the new ``--project`` switch to enable this behaviour. +- The compiler can now warn about shadowed local variables. However, this needs + to be turned on explicitly via ``--warning[ShadowIdent]:on``. +- The compiler now supports almost every pragma in a ``push`` pragma. +- Generic converters have been implemented. +- Added a **highly experimental** ``noforward`` pragma enabling a special + compilation mode that largely eliminates the need for forward declarations. + +Language Additions +------------------ + +- ``case expressions`` are now supported. +- Table constructors now mimic more closely the syntax of the ``case`` + statement. +- Nimrod can now infer the return type of a proc from its body. +- Added a ``mixin`` declaration to affect symbol binding rules in generics. +- Exception tracking has been added and the ``doc2`` command annotates possible + exceptions for you. +- User defined effects ("tags") tracking has been added and the ``doc2`` + command annotates possible tags for you. +- Types can be annotated with the new syntax ``not nil`` to explicitly state + that ``nil`` is not allowed. However currently the compiler performs no + advanced static checking for this; for now it's merely for documentation + purposes. +- An ``export`` statement has been added to the language: It can be used for + symbol forwarding so client modules don't have to import a module's + dependencies explicitly. +- Overloading based on ASTs has been implemented. +- Generics are now supported for multi methods. +- Objects can be initialized via an *object constructor expression*. +- There is a new syntactic construct ``(;)`` unifying expressions and + statements. +- You can now use ``from module import nil`` if you want to import the module + but want to enforce fully qualified access to every symbol in ``module``. + + +Notes for the future +-------------------- + +- The scope rules of ``if`` statements will change in 0.9.4. This affects the + ``=~`` pegs/re templates. +- The ``sockets`` module will become a low-level wrapper of OS-specific socket + functions. All the high-level features of the current ``sockets`` module + will be moved to a ``network`` module. diff --git a/web/news/2014_01_15_andreas_rumpfs_talk_on_nimrod.rst b/web/news/2014_01_15_andreas_rumpfs_talk_on_nimrod.rst new file mode 100644 index 000000000..00cc5e101 --- /dev/null +++ b/web/news/2014_01_15_andreas_rumpfs_talk_on_nimrod.rst @@ -0,0 +1,11 @@ +2014-01-15 Andreas Rumpf's talk on Nimrod at Strange Loop 2013 is now online +============================================================================ + +.. container:: metadata + + Posted by Dominik Picheta on 12/01/2014 + +Andreas Rumpf presented *Nimrod: A New Approach to Metaprogramming* at +`Strange Loop 2013<https://thestrangeloop.com/sessions/nimrod-a-new-approach-to-meta-programming>`_. +The `video and slides<http://www.infoq.com/presentations/nimrod>`_ +of the talk are now available. diff --git a/web/news/2014_02_11_nimrod_featured_in_dr_dobbs_journal.rst b/web/news/2014_02_11_nimrod_featured_in_dr_dobbs_journal.rst new file mode 100644 index 000000000..b48ccf31f --- /dev/null +++ b/web/news/2014_02_11_nimrod_featured_in_dr_dobbs_journal.rst @@ -0,0 +1,9 @@ +Nimrod Featured in Dr. Dobb's Journal +===================================== + +.. container:: metadata + + Posted by Dominik Picheta on 11/02/2014 + +Nimrod has been `featured<http://www.drdobbs.com/open-source/nimrod-a-new-systems-programming-languag/240165321>`_ +as the cover story in the February 2014 issue of Dr. Dobb's Journal. diff --git a/web/news/2014_04_21_version_0_9_4_released.rst b/web/news/2014_04_21_version_0_9_4_released.rst new file mode 100644 index 000000000..2714c5c78 --- /dev/null +++ b/web/news/2014_04_21_version_0_9_4_released.rst @@ -0,0 +1,179 @@ +2014-04-21 Version 0.9.4 released +================================= + +.. container:: metadata + + Posted by Dominik Picheta on 21/04/2014 + +The Nimrod development community is proud to announce the release of version +0.9.4 of the Nimrod compiler and tools. **Note: This release has to be +considered beta quality! Lots of new features have been implemented but +unfortunately some do not fulfill our quality standards yet.** + +Prebuilt binaries and instructions for building from source are available +on the `download page <download.html>`_. + +This release includes about +`1400 changes <https://github.com/Araq/Nimrod/compare/v0.9.2...v0.9.4>`_ +in total including various bug +fixes, new languages features and standard library additions and improvements. +This release brings with it support for user-defined type classes, a brand +new VM for executing Nimrod code at compile-time and new symbol binding +rules for clean templates. + +It also introduces support for the brand new +`Babel package manager <https://github.com/nimrod-code/babel>`_ which +has itself seen its first release recently. Many of the wrappers that were +present in the standard library have been moved to separate repositories +and should now be installed using Babel. + +Apart from that a new **experimental** Asynchronous IO API has been added via +the ``asyncdispatch`` and ``asyncnet`` modules. The ``net`` and ``rawsockets`` +modules have also been added and they will likely replace the sockets +module in the next release. The Asynchronous IO API has been designed to +take advantage of Linux's epoll and Windows' IOCP APIs, support for BSD's +kqueue has not been implemented yet but will be in the future. +The Asynchronous IO API provides both +a callback interface and an interface which allows you to write code as you +would if you were writing synchronous code. The latter is done through +the use of an ``await`` macro which behaves similar to C#'s await. The +following is a very simple chat server demonstrating Nimrod's new async +capabilities. + +.. code-block::nim + import asyncnet, asyncdispatch + + var clients: seq[PAsyncSocket] = @[] + + proc processClient(client: PAsyncSocket) {.async.} = + while true: + let line = await client.recvLine() + for c in clients: + await c.send(line & "\c\L") + + proc serve() {.async.} = + var server = newAsyncSocket() + server.bindAddr(TPort(12345)) + server.listen() + + while true: + let client = await server.accept() + clients.add client + + processClient(client) + + serve() + runForever() + + +Note that this feature has been implemented with Nimrod's macro system and so +``await`` and ``async`` are no keywords. + +Syntactic sugar for anonymous procedures has also been introduced. It too has +been implemented as a macro. The following shows some simple usage of the new +syntax: + +.. code-block::nim + import future + + var s = @[1, 2, 3, 4, 5] + echo(s.map((x: int) => x * 5)) + +A list of changes follows, for a comprehensive list of changes take a look +`here <https://github.com/Araq/Nimrod/compare/v0.9.2...v0.9.4>`_. + +Library Additions +----------------- + +- Added ``macros.genSym`` builtin for AST generation. +- Added ``macros.newLit`` procs for easier AST generation. +- Added module ``logging``. +- Added module ``asyncdispatch``. +- Added module ``asyncnet``. +- Added module ``net``. +- Added module ``rawsockets``. +- Added module ``selectors``. +- Added module ``asynchttpserver``. +- Added support for the new asynchronous IO in the ``httpclient`` module. +- Added a Python-inspired ``future`` module that features upcoming additions + to the ``system`` module. + + +Changes affecting backwards compatibility +----------------------------------------- + +- The scoping rules for the ``if`` statement changed for better interaction + with the new syntactic construct ``(;)``. +- ``OSError`` family of procedures has been deprecated. Procedures with the same + name but which take different parameters have been introduced. These procs now + require an error code to be passed to them. This error code can be retrieved + using the new ``OSLastError`` proc. +- ``os.parentDir`` now returns "" if there is no parent dir. +- In CGI scripts stacktraces are shown to the user only + if ``cgi.setStackTraceStdout`` is used. +- The symbol binding rules for clean templates changed: ``bind`` for any + symbol that's not a parameter is now the default. ``mixin`` can be used + to require instantiation scope for a symbol. +- ``quoteIfContainsWhite`` now escapes argument in such way that it can be safely + passed to shell, instead of just adding double quotes. +- ``macros.dumpTree`` and ``macros.dumpLisp`` have been made ``immediate``, + ``dumpTreeImm`` and ``dumpLispImm`` are now deprecated. +- The ``nil`` statement has been deprecated, use an empty ``discard`` instead. +- ``sockets.select`` now prunes sockets that are **not** ready from the list + of sockets given to it. +- The ``noStackFrame`` pragma has been renamed to ``asmNoStackFrame`` to + ensure you only use it when you know what you're doing. +- Many of the wrappers that were present in the standard library have been + moved to separate repositories and should now be installed using Babel. + + +Compiler Additions +------------------ + +- The compiler can now warn about "uninitialized" variables. (There are no + real uninitialized variables in Nimrod as they are initialized to binary + zero). Activate via ``{.warning[Uninit]:on.}``. +- The compiler now enforces the ``not nil`` constraint. +- The compiler now supports a ``codegenDecl`` pragma for even more control + over the generated code. +- The compiler now supports a ``computedGoto`` pragma to support very fast + dispatching for interpreters and the like. +- The old evaluation engine has been replaced by a proper register based + virtual machine. This fixes numerous bugs for ``nimrod i`` and for macro + evaluation. +- ``--gc:none`` produces warnings when code uses the GC. +- A ``union`` pragma for better C interoperability is now supported. +- A ``packed`` pragma to control the memory packing/alignment of fields in + an object. +- Arrays can be annotated to be ``unchecked`` for easier low level + manipulations of memory. +- Support for the new Babel package manager. + + +Language Additions +------------------ + +- Arrays can now be declared with a single integer literal ``N`` instead of a + range; the range is then ``0..N-1``. +- Added ``requiresInit`` pragma to enforce explicit initialization. +- Exported templates are allowed to access hidden fields. +- The ``using statement`` enables you to more easily author domain-specific + languages and libraries providing OOP-like syntactic sugar. +- Added the possibility to override various dot operators in order to handle + calls to missing procs and reads from undeclared fields at compile-time. +- The overload resolution now supports ``static[T]`` params that must be + evaluable at compile-time. +- Support for user-defined type classes has been added. +- The *command syntax* is supported in a lot more contexts. +- Anonymous iterators are now supported and iterators can capture variables + of an outer proc. +- The experimental ``strongSpaces`` parsing mode has been implemented. +- You can annotate pointer types with regions for increased type safety. +- Added support for the builtin ``spawn`` for easy thread pool usage. + + +Tools improvements +------------------ + +- c2nim can deal with a subset of C++. Use the ``--cpp`` command line option + to activate. diff --git a/web/news/2014_10_19_version_0_9_6_released.rst b/web/news/2014_10_19_version_0_9_6_released.rst new file mode 100644 index 000000000..7a148aaa5 --- /dev/null +++ b/web/news/2014_10_19_version_0_9_6_released.rst @@ -0,0 +1,65 @@ +Version 0.9.6 released +================================= + +.. container:: metadata + + Posted by Andreas Rumpf on 19/10/2014 + +**Note: 0.9.6 is the last release of Nimrod. The language is being renamed to +Nim. Nim slightly breaks compatibility.** + +This is a maintenance release. The upcoming 0.10.0 release has +the new features and exciting developments. + + +Changes affecting backwards compatibility +----------------------------------------- + +- ``spawn`` now uses an elaborate self-adapting thread pool and as such + has been moved into its own module. So to use it, you now have to import + ``threadpool``. +- The symbol binding rules in generics changed: ``bar`` in ``foo.bar`` is + now considered for implicit early binding. +- ``c2nim`` moved into its own repository and is now a Babel package. +- ``pas2nim`` moved into its own repository and is now a Babel package. +- ``system.$`` for floating point types now produces a human friendly string + representation. +- ``uri.TUrl`` as well as the ``parseurl`` module are now deprecated in favour + of the new ``TUri`` type in the ``uri`` module. +- The ``destructor`` pragma has been deprecated. Use the ``override`` pragma + instead. The destructor's name has to be ``destroy`` now. +- ``lambda`` is not a keyword anymore. +- **system.defined has been split into system.defined and system.declared**. + You have to use ``--symbol`` to declare new conditional symbols that can be + set via ``--define``. +- ``--threadanalysis:on`` is now the default. To make your program compile + you can disable it but this is only a temporary solution as this option + will disappear soon! + + +Compiler improvements +--------------------- + +- Multi method dispatching performance has been improved by a factor of 10x for + pathological cases. + + +Language Additions +------------------ + +- This version introduces the ``deprecated`` pragma statement that is used + to handle the upcoming massive amount of symbol renames. +- ``spawn`` can now wrap proc that has a return value. It then returns a data + flow variable of the wrapped return type. + + +Library Additions +----------------- + +- Added module ``cpuinfo``. +- Added module ``threadpool``. +- ``sequtils.distnct`` has been renamed to ``sequtils.deduplicate``. +- Added ``algorithm.reversed`` +- Added ``uri.combine`` and ``uri.parseUri``. +- Some sockets procedures now support a ``SafeDisconn`` flag which causes + them to handle disconnection errors and not raise them. diff --git a/web/news/2014_12_29_version_0_10_2_released.rst b/web/news/2014_12_29_version_0_10_2_released.rst new file mode 100644 index 000000000..ad8afa3bf --- /dev/null +++ b/web/news/2014_12_29_version_0_10_2_released.rst @@ -0,0 +1,198 @@ +Version 0.10.2 released +======================= + +.. container:: metadata + + Posted by Dominik Picheta on 29/12/2014 + +This release marks the completion of a very important change to the project: +the official renaming from Nimrod to Nim. Version 0.10.2 contains many language +changes, some of which may break your existing code. For your convenience, we +added a new tool called `nimfix <nimfix.html>`_ that will help you convert your +existing projects so that it works with the latest version of the compiler. + +Progress towards version 1.0 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Although Nim is still pre-1.0, we were able to keep the number of breaking +changes to a minimum so far. Starting with version 1.0, we will not introduce +any breaking changes between major release versions. +One of Nim's goals is to ensure that the compiler is as efficient as possible. +Take a look at the +`latest benchmarks <https://github.com/logicchains/LPATHBench/blob/master/writeup.md>`_, +which show that Nim is consistently near +the top and already nearly as fast as C and C++. Recent developments, such as +the new ``asyncdispatch`` module will allow you to write efficient web server +applications using non-blocking code. Nim now also has a built-in thread pool +for lightweight threading through the use of ``spawn``. + +The unpopular "T" and "P" prefixes on types have been deprecated. Nim also +became more expressive by weakening the distinction between statements and +expressions. We also added a new and searchable forum, a new website, and our +documentation generator ``docgen`` has seen major improvements. Many thanks to +Nick Greenfield for the much more beautiful documentation! + + + +What's left to be done +~~~~~~~~~~~~~~~~~~~~~~ + +The 1.0 release is actually very close. Apart from bug fixes, there are +two major features missing or incomplete: + +* ``static[T]`` needs to be defined precisely and the bugs in the + implementation need to be fixed. +* Overloading of the assignment operator is required for some generic + containers and needs to be implemented. + +This means that fancy matrix libraries will finally start to work, which used +to be a major point of pain in the language. + + +Nimble and other Nim tools +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Outside of the language and the compiler itself many Nim tools have seen +considerable improvements. + +Babel the Nim package manager has been renamed to Nimble. Nimble's purpose +is the installation of packages containing libraries and/or applications +written in Nim. +Even though Nimble is still very young it already is very +functional. It can install packages by name, it does so by accessing a +packages repository which is hosted on a GitHub repo. Packages can also be +installed via a Git repo URL or Mercurial repo URL. The package repository +is searchable through Nimble. Anyone is free to add their own packages to +the package repository by forking the +`nim-lang/packages <https://github.com/nim-lang/packages>`_ repo and creating +a pull request. Nimble is fully cross-platform and should be fully functional +on all major operating systems. +It is of course completely written in Nim. + +Changelog +~~~~~~~~~ + +Changes affecting backwards compatibility +----------------------------------------- + +- **The language has been renamed from Nimrod to Nim.** The name of the + compiler changed from ``nimrod`` to ``nim`` too. +- ``system.fileHandle`` has been renamed to ``system.getFileHandle`` to + prevent name conflicts with the new type ``FileHandle``. +- Comments are now not part of the AST anymore, as such you cannot use them + in place of ``discard``. +- Large parts of the stdlib got rid of the T/P type prefixes. Instead most + types now simply start with an uppercased letter. The + so called "partial case sensitivity" rule is now active allowing for code + like ``var foo: Foo`` in more contexts. +- String case (or any non-ordinal case) statements + without 'else' are deprecated. +- Recursive tuple types are not allowed anymore. Use ``object`` instead. +- The PEGS module returns ``nil`` instead of ``""`` when an optional capture + fails to match. +- The re module returns ``nil`` instead of ``""`` when an optional capture + fails to match. +- The "symmetric set difference" operator (``-+-``) never worked and has been + removed. +- ``defer`` is a keyword now. +- ``func`` is a keyword now. +- The ``using`` language feature now needs to be activated via the new + ``{.experimental.}`` pragma that enables experimental language features. +- Destructors are now officially *experimental*. +- Standalone ``except`` and ``finally`` statements are deprecated now. + The standalone ``finally`` can be replaced with ``defer``, + standalone ``except`` requires an explicit ``try``. +- Operators ending in ``>`` are considered as "arrow like" and have their + own priority level and are right associative. This means that + the ``=>`` and ``->`` operators from the `future <future.html>`_ module + work better. +- Field names in tuples are now ignored for type comparisons. This allows + for greater interoperability between different modules. +- Statement lists are not converted to an implicit ``do`` block anymore. This + means the confusing ``nnkDo`` nodes when working with macros are gone for + good. + + +Language Additions +------------------ + +- The new concurrency model has been implemented including ``locks`` sections, + lock levels and object field ``guards``. +- The ``parallel`` statement has been implemented. +- ``deepCopy`` has been added to the language. +- The builtin ``procCall`` can be used to get ``super``-like functionality + for multi methods. +- There is a new pragma ``{.experimental.}`` that enables experimental + language features per module, or you can enable these features on a global + level with the ``--experimental`` command line option. + + +Compiler Additions +------------------ + +- The compiler now supports *mixed* Objective C / C++ / C code generation: + The modules that use ``importCpp`` or ``importObjc`` are compiled to C++ + or Objective C code, any other module is compiled to C code. This + improves interoperability. +- There is a new ``parallel`` statement for safe fork&join parallel computing. +- ``guard`` and ``lock`` pragmas have been implemented to support safer + concurrent programming. +- The following procs are now available at compile-time:: + + math.sqrt, math.ln, math.log10, math.log2, math.exp, math.round, + math.arccos, math.arcsin, math.arctan, math.arctan2, math.cos, + math.cosh, math.hypot, math.sinh, math.sin, math.tan, math.tanh, + math.pow, math.trunc, math.floor, math.ceil, math.fmod, + os.getEnv, os.existsEnv, os.dirExists, os.fileExists, + system.writeFile + +- Two backticks now produce a single backtick within an ``emit`` or ``asm`` + statement. +- There is a new tool, `nimfix <nimfix.html>`_ to help you in updating your + code from Nimrod to Nim. +- The compiler's output has been prettified. + +Library Additions +----------------- + +- Added module ``fenv`` to control the handling of floating-point rounding and + exceptions (overflow, division by zero, etc.). +- ``system.setupForeignThreadGc`` can be used for better interaction with + foreign libraries that create threads and run a Nim callback from these + foreign threads. +- List comprehensions have been implemented as a macro in the ``future`` + module. +- The new Async module (``asyncnet``) now supports SSL. +- The ``smtp`` module now has an async implementation. +- Added module ``asyncfile`` which implements asynchronous file reading + and writing. +- ``osproc.kill`` has been added. +- ``asyncnet`` and ``asynchttpserver`` now support ``SO_REUSEADDR``. + +Bugfixes +-------- + +- ``nil`` and ``NULL`` are now preserved between Nim and databases in the + ``db_*`` modules. +- Fixed issue with OS module in non-unicode mode on Windows. +- Fixed issue with ``x.low`` + (`#1366 <https://github.com/Araq/Nim/issues/1366>`_). +- Fixed tuple unpacking issue inside closure iterators + (`#1067 <https://github.com/Araq/Nim/issues/1067>`_). +- Fixed ENDB compilation issues. +- Many ``asynchttpserver`` fixes. +- Macros can now keep global state across macro calls + (`#903 <https://github.com/Araq/Nim/issues/903>`_). +- ``osproc`` fixes on Windows. +- ``osproc.terminate`` fixed. +- Improvements to exception handling in async procedures. + (`#1487 <https://github.com/Araq/Nim/issues/1487>`_). +- ``try`` now works at compile-time. +- Fixes ``T = ref T`` to be an illegal recursive type. +- Self imports are now disallowed. +- Improved effect inference. +- Fixes for the ``math`` module on Windows. +- User defined pragmas will now work for generics that have + been instantiated in different modules. +- Fixed queue exhaustion bug. +- Many, many more. diff --git a/web/news/2015_04_30_version_0_11_0_released.rst b/web/news/2015_04_30_version_0_11_0_released.rst new file mode 100644 index 000000000..a8a58f2ae --- /dev/null +++ b/web/news/2015_04_30_version_0_11_0_released.rst @@ -0,0 +1,396 @@ +Version 0.11.0 released +======================= + +.. container:: metadata + + Posted by Dominik Picheta on 30/04/2015 + +With this release we are one step closer to reaching version 1.0 and by +extension the persistence of the Nim specification. As mentioned in the +previous release notes, starting with version 1.0, we will not be introducing +any more breaking changes to Nim. + +The *language* itself is very close to 1.0, the primary area that requires +more work is the standard library. + +Take a look at the `download <download.html>`_ page for binaries (Windows-only) +and 0.11.0 snapshots of the source code. The Windows installer now also +includes `Aporia <https://github.com/nim-lang/aporia>`_, +`Nimble <https://github.com/nim-lang/nimble>`_ and other useful tools to get +you started with Nim. + +What's left to be done +~~~~~~~~~~~~~~~~~~~~~~ + +The 1.0 release is expected by the end of this year. Rumors say it will be in +summer 2015. What's left: + +* Bug fixes, bug fixes, bug fixes, in particular: + - The remaining bugs of the lambda lifting pass that is responsible to enable + closures and closure iterators need to be fixed. + - ``concept`` needs to be refined, a nice name for the feature is not enough. + - Destructors need to be refined. + - ``static[T]`` needs to be fixed. + - Finish the implementation of the 'parallel' statement. +* ``immediate`` templates and macros will be deprecated as these will soon be + completely unnecessary, instead the ``typed`` or ``untyped`` metatypes can + be used. +* More of the standard library should be moved to Nimble packages and what's + left should use the features we have for concurrency and parallelism. + + + +Changes affecting backwards compatibility +----------------------------------------- + +- Parameter names are finally properly ``gensym``'ed. This can break + templates though that used to rely on the fact that they are not. + (Bug #1915.) This means this doesn't compile anymore: + +.. code-block:: nim + + template doIt(body: stmt) {.immediate.} = + # this used to inject the 'str' parameter: + proc res(str: string) = + body + + doIt: + echo str # Error: undeclared identifier: 'str' +.. + + This used to inject the ``str`` parameter into the scope of the body. + Declare the ``doIt`` template as ``immediate, dirty`` to get the old + behaviour. +- Tuple field names are not ignored anymore, this caused too many problems + in practice so now the behaviour is as it was for version 0.9.6: If field + names exist for the tuple type, they are checked. +- ``logging.level`` and ``logging.handlers`` are no longer exported. + ``addHandler``, ``getHandlers``, ``setLogFilter`` and ``getLogFilter`` + should be used instead. +- ``nim idetools`` has been replaced by a separate + tool `nimsuggest <0.11.0/nimsuggest.html>`_. +- *arrow like* operators are not right associative anymore and are required + to end with either ``->``, ``~>`` or + ``=>``, not just ``>``. Examples of operators still considered arrow like: + ``->``, ``==>``, ``+=>``. On the other hand, the following operators are now + considered regular operators again: ``|>``, ``-+>``, etc. +- Typeless parameters are now only allowed in templates and macros. The old + way turned out to be too error-prone. +- The 'addr' and 'type' operators are now parsed as unary function + application. This means ``type(x).name`` is now parsed as ``(type(x)).name`` + and not as ``type((x).name)``. Note that this also affects the AST + structure; for immediate macro parameters ``nkCall('addr', 'x')`` is + produced instead of ``nkAddr('x')``. +- ``concept`` is now a keyword and is used instead of ``generic``. +- The ``inc``, ``dec``, ``+=``, ``-=`` builtins now produce OverflowError + exceptions. This means code like the following: + +.. code-block:: nim + var x = low(T) + while x <= high(T): + echo x + inc x + +Needs to be replaced by something like this: + +.. code-block:: nim + var x = low(T).int + while x <= high(T).int: + echo x.T + inc x + +- **Negative indexing for slicing does not work anymore!** Instead + of ``a[0.. -1]`` you can + use ``a[0.. ^1]``. This also works with accessing a single + element ``a[^1]``. Note that we cannot detect this reliably as it is + determined at **runtime** whether negative indexing is used! + ``a[0.. -1]`` now produces the empty string/sequence. +- The compiler now warns about code like ``foo +=1`` which uses inconsistent + spacing around binary operators. Later versions of the language will parse + these as unary operators instead so that ``echo $foo`` finally can do what + people expect it to do. +- ``system.untyped`` and ``system.typed`` have been introduced as aliases + for ``expr`` and ``stmt``. The new names capture the semantics much better + and most likely ``expr`` and ``stmt`` will be deprecated in favor of the + new names. +- The ``split`` method in module ``re`` has changed. It now handles the case + of matches having a length of 0, and empty strings being yielded from the + iterator. A notable change might be that a pattern being matched at the + beginning and end of a string, will result in an empty string being produced + at the start and the end of the iterator. +- The compiler and nimsuggest now count columns starting with 1, not 0 for + consistency with the rest of the world. + + +Language Additions +------------------ + +- For empty ``case object`` branches ``discard`` can finally be used instead + of ``nil``. +- Automatic dereferencing is now done for the first argument of a routine + call if overloading resolution produces no match otherwise. This feature + has to be enabled with + the `experimental <0.11.0/manual.html#pragmas-experimental-pragma>`_ pragma. +- Objects that do not use inheritance nor ``case`` can be put into ``const`` + sections. This means that finally this is possible and produces rather + nice code: + +.. code-block:: nim + import tables + + const + foo = {"ah": "finally", "this": "is", "possible.": "nice!"}.toTable() + + +- Ordinary parameters can follow after a varargs parameter. This means the + following is finally accepted by the compiler: + +.. code-block:: nim + template takesBlock(a, b: int, x: varargs[expr]; blck: stmt) = + blck + echo a, b + + takesBlock 1, 2, "some", 0.90, "random stuff": + echo "yay" + +- Overloading by 'var T' is now finally possible: + +.. code-block:: nim + proc varOrConst(x: var int) = echo "var" + proc varOrConst(x: int) = echo "const" + + var x: int + varOrConst(x) # "var" + varOrConst(45) # "const" + +- Array and seq indexing can now use the builtin ``^`` operator to access + things from backwards: ``a[^1]`` is like Python's ``a[-1]``. +- A first version of the specification and implementation of the overloading + of the assignment operator has arrived! +- ``system.len`` for strings and sequences now returns 0 for nil. + +- A single underscore can now be used to discard values when unpacking tuples: + +.. code-block:: nim + let (path, _, _) = os.splitFile("path/file.ext") + + +- ``marshal.$$`` and ``marshal.to`` can be executed at compile-time. +- Interoperability with C++ improved tremendously; C++'s templates and + operators can be wrapped directly. See + `this <0.11.0/nimc.html#additional-features-importcpp-pragma>`_ + for more information. +- ``macros.getType`` can be used to query an AST's type at compile-time. This + enables more powerful macros, for instance *currying* can now be done with + a macro. + + +Library additions +----------------- + +- ``reversed`` proc added to the ``unicode`` module. +- Added multipart param to httpclient's ``post`` and ``postContent`` together + with a ``newMultipartData`` proc. +- Added `%*` operator for JSON. +- The compiler is now available as Nimble package for c2nim. +- Added ``..^`` and ``..<`` templates to system so that the rather annoying + space between ``.. <`` and ``.. ^`` is not necessary anymore. +- Added ``system.xlen`` for strings and sequences to get back the old ``len`` + operation that doesn't check for ``nil`` for efficiency. +- Added sexp.nim to parse and generate sexp. + + +Bugfixes +-------- + +- Fixed internal compiler error when using ``char()`` in an echo call + (`#1788 <https://github.com/Araq/Nim/issues/1788>`_). +- Fixed Windows cross-compilation on Linux. +- Overload resolution now works for types distinguished only by a + ``static[int]`` param + (`#1056 <https://github.com/Araq/Nim/issues/1056>`_). +- Other fixes relating to generic types and static params. +- Fixed some compiler crashes with unnamed tuples + (`#1774 <https://github.com/Araq/Nim/issues/1774>`_). +- Fixed ``channels.tryRecv`` blocking + (`#1816 <https://github.com/Araq/Nim/issues/1816>`_). +- Fixed generic instantiation errors with ``typedesc`` + (`#419 <https://github.com/Araq/Nim/issues/419>`_). +- Fixed generic regression where the compiler no longer detected constant + expressions properly (`#544 <https://github.com/Araq/Nim/issues/544>`_). +- Fixed internal error with generic proc using ``static[T]`` in a specific + way (`#1049 <https://github.com/Araq/Nim/issues/1049>`_). +- More fixes relating to generics (`#1820 <https://github.com/Araq/Nim/issues/1820>`_, + `#1050 <https://github.com/Araq/Nim/issues/1050>`_, + `#1859 <https://github.com/Araq/Nim/issues/1859>`_, + `#1858 <https://github.com/Araq/Nim/issues/1858>`_). +- Fixed httpclient to properly encode queries. +- Many fixes to the ``uri`` module. +- Async sockets are now closed on error. +- Fixes to httpclient's handling of multipart data. +- Fixed GC segfaults with asynchronous sockets + (`#1796 <https://github.com/Araq/Nim/issues/1796>`_). +- Added more versions to openssl's DLL version list + (`076f993 <https://github.com/Araq/Nim/commit/076f993>`_). +- Fixed shallow copy in iterators being broken + (`#1803 <https://github.com/Araq/Nim/issues/1803>`_). +- ``nil`` can now be inserted into tables with the ``db_sqlite`` module + (`#1866 <https://github.com/Araq/Nim/issues/1866>`_). +- Fixed "Incorrect assembler generated" + (`#1907 <https://github.com/Araq/Nim/issues/1907>`_) +- Fixed "Expression templates that define macros are unusable in some contexts" + (`#1903 <https://github.com/Araq/Nim/issues/1903>`_) +- Fixed "a second level generic subclass causes the compiler to crash" + (`#1919 <https://github.com/Araq/Nim/issues/1919>`_) +- Fixed "nim 0.10.2 generates invalid AsyncHttpClient C code for MSVC " + (`#1901 <https://github.com/Araq/Nim/issues/1901>`_) +- Fixed "1 shl n produces wrong C code" + (`#1928 <https://github.com/Araq/Nim/issues/1928>`_) +- Fixed "Internal error on tuple yield" + (`#1838 <https://github.com/Araq/Nim/issues/1838>`_) +- Fixed "ICE with template" + (`#1915 <https://github.com/Araq/Nim/issues/1915>`_) +- Fixed "include the tool directory in the installer as it is required by koch" + (`#1947 <https://github.com/Araq/Nim/issues/1947>`_) +- Fixed "Can't compile if file location contains spaces on Windows" + (`#1955 <https://github.com/Araq/Nim/issues/1955>`_) +- Fixed "List comprehension macro only supports infix checks as guards" + (`#1920 <https://github.com/Araq/Nim/issues/1920>`_) +- Fixed "wrong field names of compatible tuples in generic types" + (`#1910 <https://github.com/Araq/Nim/issues/1910>`_) +- Fixed "Macros within templates no longer work as expected" + (`#1944 <https://github.com/Araq/Nim/issues/1944>`_) +- Fixed "Compiling for Standalone AVR broken in 0.10.2" + (`#1964 <https://github.com/Araq/Nim/issues/1964>`_) +- Fixed "Compiling for Standalone AVR broken in 0.10.2" + (`#1964 <https://github.com/Araq/Nim/issues/1964>`_) +- Fixed "Code generation for mitems with tuple elements" + (`#1833 <https://github.com/Araq/Nim/issues/1833>`_) +- Fixed "httpclient.HttpMethod should not be an enum" + (`#1962 <https://github.com/Araq/Nim/issues/1962>`_) +- Fixed "terminal / eraseScreen() throws an OverflowError" + (`#1906 <https://github.com/Araq/Nim/issues/1906>`_) +- Fixed "setControlCHook(nil) disables registered quit procs" + (`#1546 <https://github.com/Araq/Nim/issues/1546>`_) +- Fixed "Unexpected idetools behaviour" + (`#325 <https://github.com/Araq/Nim/issues/325>`_) +- Fixed "Unused lifted lambda does not compile" + (`#1642 <https://github.com/Araq/Nim/issues/1642>`_) +- Fixed "'low' and 'high' don't work with cstring asguments" + (`#2030 <https://github.com/Araq/Nim/issues/2030>`_) +- Fixed "Converting to int does not round in JS backend" + (`#1959 <https://github.com/Araq/Nim/issues/1959>`_) +- Fixed "Internal error genRecordField 2 when adding region to pointer." + (`#2039 <https://github.com/Araq/Nim/issues/2039>`_) +- Fixed "Macros fail to compile when compiled with --os:standalone" + (`#2041 <https://github.com/Araq/Nim/issues/2041>`_) +- Fixed "Reading from {.compileTime.} variables can cause code generation to fail" + (`#2022 <https://github.com/Araq/Nim/issues/2022>`_) +- Fixed "Passing overloaded symbols to templates fails inside generic procedures" + (`#1988 <https://github.com/Araq/Nim/issues/1988>`_) +- Fixed "Compiling iterator with object assignment in release mode causes "var not init"" + (`#2023 <https://github.com/Araq/Nim/issues/2023>`_) +- Fixed "calling a large number of macros doing some computation fails" + (`#1989 <https://github.com/Araq/Nim/issues/1989>`_) +- Fixed "Can't get Koch to install nim under Windows" + (`#2061 <https://github.com/Araq/Nim/issues/2061>`_) +- Fixed "Template with two stmt parameters segfaults compiler" + (`#2057 <https://github.com/Araq/Nim/issues/2057>`_) +- Fixed "`noSideEffect` not affected by `echo`" + (`#2011 <https://github.com/Araq/Nim/issues/2011>`_) +- Fixed "Compiling with the cpp backend ignores --passc" + (`#1601 <https://github.com/Araq/Nim/issues/1601>`_) +- Fixed "Put untyped procedure parameters behind the experimental pragma" + (`#1956 <https://github.com/Araq/Nim/issues/1956>`_) +- Fixed "generic regression" + (`#2073 <https://github.com/Araq/Nim/issues/2073>`_) +- Fixed "generic regression" + (`#2073 <https://github.com/Araq/Nim/issues/2073>`_) +- Fixed "Regression in template lookup with generics" + (`#2004 <https://github.com/Araq/Nim/issues/2004>`_) +- Fixed "GC's growObj is wrong for edge cases" + (`#2070 <https://github.com/Araq/Nim/issues/2070>`_) +- Fixed "Compiler internal error when creating an array out of a typeclass" + (`#1131 <https://github.com/Araq/Nim/issues/1131>`_) +- Fixed "GC's growObj is wrong for edge cases" + (`#2070 <https://github.com/Araq/Nim/issues/2070>`_) +- Fixed "Invalid Objective-C code generated when calling class method" + (`#2068 <https://github.com/Araq/Nim/issues/2068>`_) +- Fixed "walkDirRec Error" + (`#2116 <https://github.com/Araq/Nim/issues/2116>`_) +- Fixed "Typo in code causes compiler SIGSEGV in evalAtCompileTime" + (`#2113 <https://github.com/Araq/Nim/issues/2113>`_) +- Fixed "Regression on exportc" + (`#2118 <https://github.com/Araq/Nim/issues/2118>`_) +- Fixed "Error message" + (`#2102 <https://github.com/Araq/Nim/issues/2102>`_) +- Fixed "hint[path] = off not working in nim.cfg" + (`#2103 <https://github.com/Araq/Nim/issues/2103>`_) +- Fixed "compiler crashes when getting a tuple from a sequence of generic tuples" + (`#2121 <https://github.com/Araq/Nim/issues/2121>`_) +- Fixed "nim check hangs with when" + (`#2123 <https://github.com/Araq/Nim/issues/2123>`_) +- Fixed "static[T] param in nested type resolve/caching issue" + (`#2125 <https://github.com/Araq/Nim/issues/2125>`_) +- Fixed "repr should display ``\0``" + (`#2124 <https://github.com/Araq/Nim/issues/2124>`_) +- Fixed "'nim check' never ends in case of recursive dependency " + (`#2051 <https://github.com/Araq/Nim/issues/2051>`_) +- Fixed "From macros: Error: unhandled exception: sons is not accessible" + (`#2167 <https://github.com/Araq/Nim/issues/2167>`_) +- Fixed "`fieldPairs` doesn't work inside templates" + (`#1902 <https://github.com/Araq/Nim/issues/1902>`_) +- Fixed "fields iterator misbehavior on break statement" + (`#2134 <https://github.com/Araq/Nim/issues/2134>`_) +- Fixed "Fix for compiler not building anymore since #c3244ef1ff" + (`#2193 <https://github.com/Araq/Nim/issues/2193>`_) +- Fixed "JSON parser fails in cpp output mode" + (`#2199 <https://github.com/Araq/Nim/issues/2199>`_) +- Fixed "macros.getType mishandles void return" + (`#2211 <https://github.com/Araq/Nim/issues/2211>`_) +- Fixed "Regression involving templates instantiated within generics" + (`#2215 <https://github.com/Araq/Nim/issues/2215>`_) +- Fixed ""Error: invalid type" for 'not nil' on generic type." + (`#2216 <https://github.com/Araq/Nim/issues/2216>`_) +- Fixed "--threads:on breaks async" + (`#2074 <https://github.com/Araq/Nim/issues/2074>`_) +- Fixed "Type mismatch not always caught, can generate bad code for C backend." + (`#2169 <https://github.com/Araq/Nim/issues/2169>`_) +- Fixed "Failed C compilation when storing proc to own type in object" + (`#2233 <https://github.com/Araq/Nim/issues/2233>`_) +- Fixed "Unknown line/column number in constant declaration type conversion error" + (`#2252 <https://github.com/Araq/Nim/issues/2252>`_) +- Fixed "Adding {.compile.} fails if nimcache already exists." + (`#2247 <https://github.com/Araq/Nim/issues/2247>`_) +- Fixed "Two different type names generated for a single type (C backend)" + (`#2250 <https://github.com/Araq/Nim/issues/2250>`_) +- Fixed "Ambigous call when it should not be" + (`#2229 <https://github.com/Araq/Nim/issues/2229>`_) +- Fixed "Make sure we can load root urls" + (`#2227 <https://github.com/Araq/Nim/issues/2227>`_) +- Fixed "Failure to slice a string with an int subrange type" + (`#794 <https://github.com/Araq/Nim/issues/794>`_) +- Fixed "documentation error" + (`#2205 <https://github.com/Araq/Nim/issues/2205>`_) +- Fixed "Code growth when using `const`" + (`#1940 <https://github.com/Araq/Nim/issues/1940>`_) +- Fixed "Instances of generic types confuse overload resolution" + (`#2220 <https://github.com/Araq/Nim/issues/2220>`_) +- Fixed "Compiler error when initializing sdl2's EventType" + (`#2316 <https://github.com/Araq/Nim/issues/2316>`_) +- Fixed "Parallel disjoint checking can't handle `<`, `items`, or arrays" + (`#2287 <https://github.com/Araq/Nim/issues/2287>`_) +- Fixed "Strings aren't copied in parallel loop" + (`#2286 <https://github.com/Araq/Nim/issues/2286>`_) +- Fixed "JavaScript compiler crash with tables" + (`#2298 <https://github.com/Araq/Nim/issues/2298>`_) +- Fixed "Range checker too restrictive" + (`#1845 <https://github.com/Araq/Nim/issues/1845>`_) +- Fixed "Failure to slice a string with an int subrange type" + (`#794 <https://github.com/Araq/Nim/issues/794>`_) +- Fixed "Remind user when compiling in debug mode" + (`#1868 <https://github.com/Araq/Nim/issues/1868>`_) +- Fixed "Compiler user guide has jumbled options/commands." + (`#1819 <https://github.com/Araq/Nim/issues/1819>`_) +- Fixed "using `method`: 1 in a objects constructor fails when compiling" + (`#1791 <https://github.com/Araq/Nim/issues/1791>`_) diff --git a/web/news/2015_05_04_version_0_11_2_released.rst b/web/news/2015_05_04_version_0_11_2_released.rst new file mode 100644 index 000000000..273182340 --- /dev/null +++ b/web/news/2015_05_04_version_0_11_2_released.rst @@ -0,0 +1,12 @@ +Version 0.11.2 released +================================== + +.. container:: metadata + + Posted by Andreas Rumpf on 04/05/2015 + +This is just a bugfix release that fixes the most pressing regressions we +introduced with version 0.11.0. The way types are computed was +changed significantly causing all sort of problems. Sorry for the +inconvenience; we grew overconfident our large test suite would prevent these +things. diff --git a/web/news/2015_10_16_first_nim_conference.rst b/web/news/2015_10_16_first_nim_conference.rst new file mode 100644 index 000000000..228bffd28 --- /dev/null +++ b/web/news/2015_10_16_first_nim_conference.rst @@ -0,0 +1,26 @@ +First Nim conference +==================== + +.. container:: metadata + + Posted by Dominik Picheta on 16/10/2015 + +.. raw::html + + <img src="../assets/zeo/banner.jpg" alt="First Nim conference in Ukraine!" width="682"/> + +This Autumn you have the unique opportunity to take part in the first Nim event +held in Kyiv and to meet the creator of the Nim programming language - +Andreas Rumpf. The event is hosted by Zeo Alliance and is taking place between +14-15 November 2015 in Kyiv, Ukraine. + +During the workshop you will learn: + +- The basics of the language including its safe and unsafe subsets. +- How to use Nim to develop web applications. +- How Nim's meta programming capabilities make Nim the ultimate glue language, + excellent at interoperability with C++, JavaScript, Java and others. +- Games in Nim and the ability to rapidly prototype without sacrificing speed. + +Registration is free, but the number of places is limited. More details +can be found `here <https://nimworkshop.splashthat.com/>`_. diff --git a/web/news/2015_10_27_version_0_12_0_released.rst b/web/news/2015_10_27_version_0_12_0_released.rst new file mode 100644 index 000000000..63088f9e2 --- /dev/null +++ b/web/news/2015_10_27_version_0_12_0_released.rst @@ -0,0 +1,403 @@ +2015-10-27 Version 0.12.0 released +================================== + +.. container:: metadata + + Posted by Dominik Picheta on 27/10/2015 + +The Nim community of developers is proud to announce the new version of the +Nim compiler. This has been a long time coming as the last release has been +made over 5 months ago! + +This release includes some changes which affect backwards compatibility, +one major change is that now the hash table ``[]`` operators now raise a +``KeyError`` exception when the key does not exist. + +Some of the more exciting new features include: the ability to unpack tuples +in any assignment context, the introduction of `NimScript <docs/nims.html>`_, +and improvements to the type inference of lambdas. + +There are of course many many many bug fixes included with this release. +We are getting closer and closer to a 1.0 release and are hoping that only +a few 0.x releases will be necessary before we are happy to release version 1.0. + +As always you can download the latest version of Nim from the +`download <download.html>`_ page. + +For a more detailed list of changes look below. Some of the upcoming breaking +changes are also documented in this forum +`thread <http://forum.nim-lang.org/t/1708>`_. + +Changes affecting backwards compatibility +----------------------------------------- +- The regular expression modules, ``re`` and ``nre`` now depend on version + 8.36 of PCRE. If you have an older version you may see a message similar + to ``could not import: pcre_free_study`` output when you start your + program. See `this issue <https://github.com/docopt/docopt.nim/issues/13>`_ + for more information. +- ``tables.[]``, ``strtabs.[]``, ``critbits.[]`` **now raise** + the ``KeyError`` **exception when the key does not exist**! Use the + new ``getOrDefault`` instead to get the old behaviour. Compile all your + code with ``-d:nimTableGet`` to get a listing of where your code + uses ``[]``! +- The ``rawsockets`` module has been renamed to ``nativesockets`` to avoid + confusion with TCP/IP raw sockets, so ``newNativeSocket`` should be used + instead of ``newRawSocket``. +- The ``miliseconds`` property of ``times.TimeInterval`` is now ``milliseconds``. + Code accessing that property is deprecated and code using ``miliseconds`` + during object initialization or as a named parameter of ``initInterval()`` + will need to be updated. +- ``std.logging`` functions no longer do formatting and semantically treat + their arguments just like ``echo`` does. Affected functions: ``log``, + ``debug``, ``info``, ``warn``, ``error``, ``fatal``. Custom subtypes of + ``Logger`` also need to be adjusted accordingly. +- Floating point numbers can now look like ``2d`` (float64) + and ``2f`` (float32) which means imports like ``import scene/2d/sprite`` + do not work anymore. Instead quotes have to be + used: ``import "scene/2d/sprite"``. The former code never was valid Nim. +- The Windows API wrapper (``windows.nim``) is now not part of the official + distribution anymore. Instead use the ``oldwinapi`` Nimble package. +- There is now a clear distinction between ``--os:standalone`` + and ``--gc:none``. So if you use ``--os:standalone`` ensure you also use + ``--gc:none``. ``--os:standalone`` without ``--gc:none`` is now a version + that doesn't depend on any OS but includes the GC. However this version + is currently untested! +- All procedures which construct a ``Socket``/``AsyncSocket`` now need to + specify the socket domain, type and protocol. The param name + ``typ: SockType`` (in ``newSocket``/``newAsyncSocket`` procs) was also + renamed to ``sockType``. The param ``af`` in the ``connect`` procs was + removed. This affects ``asyncnet``, ``asyncdispatch``, ``net``, and + ``rawsockets``. +- ``varargs[typed]`` and ``varargs[untyped]`` have been refined and now work + as expected. However ``varargs[untyped]`` is not an alias anymore for + ``varargs[expr]``. So if your code breaks for ``varargs[untyped]``, use + ``varargs[expr]`` instead. The same applies to ``varargs[typed]`` vs + ``varargs[stmt]``. +- ``sequtils.delete`` doesn't take confusing default arguments anymore. +- ``system.free`` was an error-prone alias to ``system.dealloc`` and has + been removed. +- ``macros.high`` never worked and the manual says ``high`` cannot be + overloaded, so we removed it with no deprecation cycle. +- To use the ``parallel`` statement you now have to + use the ``--experimental`` mode. +- Toplevel procs of calling convention ``closure`` never worked reliably + and are now deprecated and will be removed from the language. Instead you + have to insert type conversions + like ``(proc (a, b: int) {.closure.})(myToplevelProc)`` if necessary. +- The modules ``libffi``, ``sdl``, ``windows``, ``zipfiles``, ``libzip``, + ``zlib``, ``zzip``, ``dialogs``, ``expat``, ``graphics``, ``libcurl``, + ``sphinx`` have been moved out of the stdlib and are Nimble packages now. +- The constant fights between 32 and 64 bit DLLs on Windows have been put to + an end: The standard distribution now ships with 32 and 64 bit versions + of all the DLLs the standard library needs. This means that the following + DLLs are now split into 32 and 64 versions: + + * ``pcre.dll``: Split into ``pcre32.dll`` and ``pcre64.dll``. + * ``pdcurses.dll``: Split into ``pdcurses32.dll`` and ``pdcurses64.dll``. + * ``sqlite3.dll``: Split into ``sqlite3_32.dll`` and ``sqlite3_64.dll``. + * ``ssleay32.dll``: Split into ``ssleay32.dll`` and ``ssleay64.dll``. + * ``libeay32.dll``: Split into ``libeay32.dll`` and ``libeay64.dll``. + + Compile with ``-d:nimOldDLLs`` to make the stdlib use the old DLL names. +- Nim VM now treats objects as ``nkObjConstr`` nodes, and not ``nkPar`` nodes + as it was previously. Macros that generate ``nkPar`` nodes when object is + expected are likely to break. Macros that expect ``nkPar`` nodes to which + objects are passed are likely to break as well. +- Base methods now need to be annotated with the ``base`` pragma. This makes + multi methods less error-prone to use with the effect system. +- Nim's parser directive ``#!`` is now ``#?`` in order to produce no conflicts + with Unix's ``#!``. +- An implicit return type for an iterator is now deprecated. Use ``auto`` if + you want more type inference. +- The type ``auto`` is now a "multi-bind" metatype, so the following compiles: + + .. code-block:: nim + proc f(x, y: auto): auto = + result = $x & y + + echo f(0, "abc") +- The ``ftpclient`` module is now deprecated in favour of the + ``asyncftpclient`` module. +- In sequtils.nim renamed ``repeat`` function to ``cycle`` (concatenating + a sequence by itself the given times), and also introduced ``repeat``, + which repeats an element the given times. +- The function ``map`` is moved to sequtils.nim. The inplace ``map`` version + is renamed to ``apply``. +- The template ``mapIt`` now doesn't require the result's type parameter. + Also the inplace ``mapIt`` is renamed to ``apply``. +- The compiler is now stricter with what is allowed as a case object + discriminator. The following code used to compile but was not supported + completely and so now fails: + +.. code-block:: nim + type + DataType* {.pure.} = enum + Char = 1, + Int8 = 2, + Int16 = 3, + Int32 = 4, + Int64 = 5, + Float32 = 6, + Float64 = 7 + + DataSeq* = object + case kind* : DataType + of DataType.Char: charSeq* : seq[char] + of DataType.Int8: int8Seq* : seq[int8] + of DataType.Int16: int16Seq* : seq[int16] + of DataType.Int32: int32Seq* : seq[int32] + of DataType.Int64: int64Seq* : seq[int64] + of DataType.Float32: float32Seq* : seq[float32] + of DataType.Float64: float64Seq* : seq[float64] + + length* : int + + + +Library Additions +----------------- + +- The nre module has been added, providing a better interface to PCRE than re. +- The ``expandSymlink`` proc has been added to the ``os`` module. +- The ``tailDir`` proc has been added to the ``os`` module. +- Define ``nimPinToCpu`` to make the ``threadpool`` use explicit thread + affinities. This can speed up or slow down the thread pool; it's up to you + to benchmark it. +- ``strutils.formatFloat`` and ``formatBiggestFloat`` do not depend on the C + locale anymore and now take an optional ``decimalSep = '.'`` parameter. +- Added ``unicode.lastRune``, ``unicode.graphemeLen``. + + +Compiler Additions +------------------ + +- The compiler now supports a new configuration system based on + `NimScript <docs/nims.html>`_. +- The compiler finally considers symbol binding rules in templates and + generics for overloaded ``[]``, ``[]=``, ``{}``, ``{}=`` operators + (issue `#2599 <https://github.com/nim-lang/Nim/issues/2599>`_). +- The compiler now supports a `bitsize pragma <docs/manual.html#pragmas-bitsize-pragma>`_ + for constructing bitfields. +- Added a new ``--reportConceptFailures`` switch for better debugging of + concept related type mismatches. This can also be used to debug + ``system.compiles`` failures. + + +Language Additions +------------------ + +- ``system.unsafeAddr`` can be used to access the address of a ``let`` + variable or parameter for C interoperability. Since technically this + makes parameters and ``let`` variables mutable, it is considered even more + unsafe than the ordinary ``addr`` builtin. +- Added ``macros.getImpl`` that can be used to access the implementation of + a routine or a constant. This allows for example for user-defined inlining + of function calls. +- Tuple unpacking finally works in a non-var/let context: ``(x, y) = f()`` + is allowed. Note that this doesn't declare ``x`` and ``y`` variables, for + this ``let (x, y) = f()`` still needs to be used. +- ``when nimvm`` can now be used for compiletime versions of some code + sections. Click `here <docs/manual.html#when-nimvm-statement>`_ for details. +- Usage of the type ``NimNode`` in a proc now implicitly annotates the proc + with ``.compileTime``. This means generics work much better for ``NimNode``. + + +Bugfixes +-------- +- Fixed "Compiler internal error on iterator it(T: typedesc[Base]) called with it(Child), where Child = object of Base" + (`#2662 <https://github.com/Araq/Nim/issues/2662>`_) +- Fixed "repr() misses base object field in 2nd level derived object" + (`#2749 <https://github.com/Araq/Nim/issues/2749>`_) +- Fixed "nimsuggest doesn't work more than once on the non-main file" + (`#2694 <https://github.com/Araq/Nim/issues/2694>`_) +- Fixed "JS Codegen. Passing arguments by var in certain cases leads to invalid JS." + (`#2798 <https://github.com/Araq/Nim/issues/2798>`_) +- Fixed ""check" proc in unittest.nim prevents the propagation of changes to var parameters." + (`#964 <https://github.com/Araq/Nim/issues/964>`_) +- Fixed "Excessive letters in integer literals are not an error" + (`#2523 <https://github.com/Araq/Nim/issues/2523>`_) +- Fixed "Unicode dashes as "lisp'ish" alternative to hump and snake notation" + (`#2811 <https://github.com/Araq/Nim/issues/2811>`_) +- Fixed "Bad error message when trying to construct an object incorrectly" + (`#2584 <https://github.com/Araq/Nim/issues/2584>`_) +- Fixed "Determination of GC safety of globals is broken " + (`#2854 <https://github.com/Araq/Nim/issues/2854>`_) +- Fixed "v2 gc crashes compiler" + (`#2687 <https://github.com/Araq/Nim/issues/2687>`_) +- Fixed "Compile error using object in const array" + (`#2774 <https://github.com/Araq/Nim/issues/2774>`_) +- Fixed "httpclient async requests with method httpPOST isn't sending Content-Length header" + (`#2884 <https://github.com/Araq/Nim/issues/2884>`_) +- Fixed "Streams module not working with JS backend" + (`#2148 <https://github.com/Araq/Nim/issues/2148>`_) +- Fixed "Sign of certain short constants is wrong" + (`#1179 <https://github.com/Araq/Nim/issues/1179>`_) +- Fixed "Symlinks to directories reported as symlinks to files" + (`#1985 <https://github.com/Araq/Nim/issues/1985>`_) +- Fixed "64-bit literals broken on x86" + (`#2909 <https://github.com/Araq/Nim/issues/2909>`_) +- Fixed "import broken for certain names" + (`#2904 <https://github.com/Araq/Nim/issues/2904>`_) +- Fixed "Invalid UTF-8 strings in JavaScript" + (`#2917 <https://github.com/Araq/Nim/issues/2917>`_) +- Fixed "[JS][Codegen] Initialising object doesn't create unmentioned fields." + + (`#2617 <https://github.com/Araq/Nim/issues/2617>`_) +- Fixed "Table returned from proc computed at compile time is missing keys:" + (`#2297 <https://github.com/Araq/Nim/issues/2297>`_) +- Fixed "Clarify copyright status for some files" + (`#2949 <https://github.com/Araq/Nim/issues/2949>`_) +- Fixed "math.nim: trigonometry: radians to degrees conversion" + (`#2881 <https://github.com/Araq/Nim/issues/2881>`_) +- Fixed "xoring unsigned integers yields RangeError in certain conditions" + (`#2979 <https://github.com/Araq/Nim/issues/2979>`_) +- Fixed "Directly checking equality between procs" + (`#2985 <https://github.com/Araq/Nim/issues/2985>`_) +- Fixed "Compiler crashed, but there have to be meaningful error message" + (`#2974 <https://github.com/Araq/Nim/issues/2974>`_) +- Fixed "repr is broken" + (`#2992 <https://github.com/Araq/Nim/issues/2992>`_) +- Fixed "Ipv6 devel - add IPv6 support for asyncsockets, make AF_INET6 a default" + (`#2976 <https://github.com/Araq/Nim/issues/2976>`_) +- Fixed "Compilation broken on windows" + (`#2996 <https://github.com/Araq/Nim/issues/2996>`_) +- Fixed "'u64 literal conversion compiler error" + (`#2731 <https://github.com/Araq/Nim/issues/2731>`_) +- Fixed "Importing 'impure' libraries while using threads causes segfaults" + (`#2672 <https://github.com/Araq/Nim/issues/2672>`_) +- Fixed "Uncatched exception in async procedure on raise statement" + (`#3014 <https://github.com/Araq/Nim/issues/3014>`_) +- Fixed "nim doc2 fails in Mac OS X due to system.nim (possibly related to #1898)" + (`#3005 <https://github.com/Araq/Nim/issues/3005>`_) +- Fixed "IndexError when rebuilding Nim on iteration 2" + (`#3018 <https://github.com/Araq/Nim/issues/3018>`_) +- Fixed "Assigning large const set to variable looses some information" + (`#2880 <https://github.com/Araq/Nim/issues/2880>`_) +- Fixed "Inconsistent generics behavior" + (`#3022 <https://github.com/Araq/Nim/issues/3022>`_) +- Fixed "Compiler breaks on float64 division" + (`#3028 <https://github.com/Araq/Nim/issues/3028>`_) +- Fixed "Confusing error message comparing string to nil " + (`#2935 <https://github.com/Araq/Nim/issues/2935>`_) +- Fixed "convert 64bit number to float on 32bit" + (`#1463 <https://github.com/Araq/Nim/issues/1463>`_) +- Fixed "Type redefinition and construction will break nim check" + (`#3032 <https://github.com/Araq/Nim/issues/3032>`_) +- Fixed "XmlParser fails on very large XML files without new lines" + (`#2429 <https://github.com/Araq/Nim/issues/2429>`_) +- Fixed "Error parsing arguments with whitespaces" + (`#2874 <https://github.com/Araq/Nim/issues/2874>`_) +- Fixed "Crash when missing one arg and used a named arg" + (`#2993 <https://github.com/Araq/Nim/issues/2993>`_) +- Fixed "Wrong number of arguments in assert will break nim check" + (`#3044 <https://github.com/Araq/Nim/issues/3044>`_) +- Fixed "Wrong const definition will break nim check" + (`#3041 <https://github.com/Araq/Nim/issues/3041>`_) +- Fixed "Wrong set declaration will break nim check" + (`#3040 <https://github.com/Araq/Nim/issues/3040>`_) +- Fixed "Compiler segfault (type section)" + (`#2540 <https://github.com/Araq/Nim/issues/2540>`_) +- Fixed "Segmentation fault when compiling this code" + (`#3038 <https://github.com/Araq/Nim/issues/3038>`_) +- Fixed "Kill nim i" + (`#2633 <https://github.com/Araq/Nim/issues/2633>`_) +- Fixed "Nim check will break on wrong array declaration" + (`#3048 <https://github.com/Araq/Nim/issues/3048>`_) +- Fixed "boolVal seems to be broken" + (`#3046 <https://github.com/Araq/Nim/issues/3046>`_) +- Fixed "Nim check crashes on wrong set/array declaration inside ref object" + (`#3062 <https://github.com/Araq/Nim/issues/3062>`_) +- Fixed "Nim check crashes on incorrect generic arg definition" + (`#3051 <https://github.com/Araq/Nim/issues/3051>`_) +- Fixed "Nim check crashes on iterating nonexistent var" + (`#3053 <https://github.com/Araq/Nim/issues/3053>`_) +- Fixed "Nim check crashes on wrong param set declaration + iteration" + (`#3054 <https://github.com/Araq/Nim/issues/3054>`_) +- Fixed "Wrong sharing of static_t instantations" + (`#3112 <https://github.com/Araq/Nim/issues/3112>`_) +- Fixed "Automatically generated proc conflicts with user-defined proc when .exportc.'ed" + (`#3134 <https://github.com/Araq/Nim/issues/3134>`_) +- Fixed "getTypeInfo call crashes nim" + (`#3099 <https://github.com/Araq/Nim/issues/3099>`_) +- Fixed "Array ptr dereference" + (`#2963 <https://github.com/Araq/Nim/issues/2963>`_) +- Fixed "Internal error when `repr`-ing a type directly" + (`#3079 <https://github.com/Araq/Nim/issues/3079>`_) +- Fixed "unknown type name 'TNimType' after importing typeinfo module" + (`#2841 <https://github.com/Araq/Nim/issues/2841>`_) +- Fixed "Can export a template twice and from inside a block" + (`#1738 <https://github.com/Araq/Nim/issues/1738>`_) +- Fixed "C Codegen: C Types are defined after their usage in certain cases" + (`#2823 <https://github.com/Araq/Nim/issues/2823>`_) +- Fixed "s.high refers to the current seq instead of the old one" + (`#1832 <https://github.com/Araq/Nim/issues/1832>`_) +- Fixed "Error while unmarshaling null values" + (`#3149 <https://github.com/Araq/Nim/issues/3149>`_) +- Fixed "Inference of `static[T]` in sequences" + (`#3144 <https://github.com/Araq/Nim/issues/3144>`_) +- Fixed "Argument named "closure" to proc inside template interfere with closure pragma" + (`#3171 <https://github.com/Araq/Nim/issues/3171>`_) +- Fixed "Internal error with aliasing inside template" + (`#3158 <https://github.com/Araq/Nim/issues/3158>`_) +- Fixed "Cardinality of sets prints unexpected value" + (`#3135 <https://github.com/Araq/Nim/issues/3135>`_) +- Fixed "Nim crashes on const assignment from function returning var ref object" + (`#3103 <https://github.com/Araq/Nim/issues/3103>`_) +- Fixed "`repr` cstring" + (`#3080 <https://github.com/Araq/Nim/issues/3080>`_) +- Fixed "Nim check crashes on wrong enum declaration" + (`#3052 <https://github.com/Araq/Nim/issues/3052>`_) +- Fixed "Compiler assertion when evaluating template with static[T]" + (`#1858 <https://github.com/Araq/Nim/issues/1858>`_) +- Fixed "Erroneous overflow in iterators when compiler built with overflowChecks enabled" + (`#3140 <https://github.com/Araq/Nim/issues/3140>`_) +- Fixed "Unicode dashes as "lisp'ish" alternative to hump and snake notation" + (`#2811 <https://github.com/Araq/Nim/issues/2811>`_) +- Fixed "Calling discardable proc from a defer is an error." + (`#3185 <https://github.com/Araq/Nim/issues/3185>`_) +- Fixed "Defer statement at the end of a block produces ICE" + (`#3186 <https://github.com/Araq/Nim/issues/3186>`_) +- Fixed "Call to `createU` fails to compile" + (`#3193 <https://github.com/Araq/Nim/issues/3193>`_) +- Fixed "VM crash when accessing array's element" + (`#3192 <https://github.com/Araq/Nim/issues/3192>`_) +- Fixed "Unexpected proc invoked when different modules add procs to a type from a 3rd module" + (`#2664 <https://github.com/Araq/Nim/issues/2664>`_) +- Fixed "Nim crashes on conditional declaration inside a template" + (`#2670 <https://github.com/Araq/Nim/issues/2670>`_) +- Fixed "Iterator names conflict within different scopes" + (`#2752 <https://github.com/Araq/Nim/issues/2752>`_) +- Fixed "VM: Cannot assign int value to ref variable" + (`#1329 <https://github.com/Araq/Nim/issues/1329>`_) +- Fixed "Incorrect code generated for tagged unions with enums not starting at zero" + (`#3096 <https://github.com/Araq/Nim/issues/3096>`_) +- Fixed "Compile time procs using forward declarations are silently ignored" + (`#3066 <https://github.com/Araq/Nim/issues/3066>`_) +- Fixed "re binding error in generic" + (`#1965 <https://github.com/Araq/Nim/issues/1965>`_) +- Fixed "os.getCreationTime is incorrect/impossible on Posix systems" + (`#1058 <https://github.com/Araq/Nim/issues/1058>`_) +- Fixed "Improve error message for osproc.startProcess when command does not exist" + (`#2183 <https://github.com/Araq/Nim/issues/2183>`_) +- Fixed "gctest segfaults with --gc:markandsweep on x86_64" + (`#2305 <https://github.com/Araq/Nim/issues/2305>`_) +- Fixed "Coroutine changes break compilation on unsupported architectures" + (`#3245 <https://github.com/Araq/Nim/issues/3245>`_) +- Fixed "Bugfix: Windows 32bit TinyCC support issue fixed" + (`#3237 <https://github.com/Araq/Nim/issues/3237>`_) +- Fixed "db_mysql getValue() followed by exec() causing error" + (`#3220 <https://github.com/Araq/Nim/issues/3220>`_) +- Fixed "xmltree.newEntity creates xnCData instead of xnEntity" + (`#3282 <https://github.com/Araq/Nim/issues/3282>`_) +- Fixed "Methods and modules don't work together" + (`#2590 <https://github.com/Araq/Nim/issues/2590>`_) +- Fixed "String slicing not working in the vm" + (`#3300 <https://github.com/Araq/Nim/issues/3300>`_) +- Fixed "internal error: evalOp(mTypeOf)" + (`#3230 <https://github.com/Araq/Nim/issues/3230>`_) +- Fixed "#! source code prefix collides with Unix Shebang" + (`#2559 <https://github.com/Araq/Nim/issues/2559>`_) +- Fixed "wrong codegen for constant object" + (`#3195 <https://github.com/Araq/Nim/issues/3195>`_) +- Fixed "Doc comments inside procs with implicit returns don't work" + (`#1528 <https://github.com/Araq/Nim/issues/1528>`_) diff --git a/web/news/2016_01_18_andreas_rumpfs_talk_at_oscon_amsterdam.rst b/web/news/2016_01_18_andreas_rumpfs_talk_at_oscon_amsterdam.rst new file mode 100644 index 000000000..fcb4a8794 --- /dev/null +++ b/web/news/2016_01_18_andreas_rumpfs_talk_at_oscon_amsterdam.rst @@ -0,0 +1,13 @@ +Andreas Rumpf's talk at OSCON Amsterdam +================================================== + +.. container:: metadata + + Posted by Dominik Picheta on 18/01/2016 + +In case you have missed it, here is Andreas' Nim: An Overview talk at +OSCON Amsterdam. + +.. raw:: html + + <iframe width="560" height="315" src="https://www.youtube.com/embed/4rJEBs_Nnaw" frameborder="0" allowfullscreen></iframe> diff --git a/web/news/2016_01_18_version_0_13_0_released.rst b/web/news/2016_01_18_version_0_13_0_released.rst new file mode 100644 index 000000000..2c8e66fa3 --- /dev/null +++ b/web/news/2016_01_18_version_0_13_0_released.rst @@ -0,0 +1,182 @@ +Version 0.13.0 released +======================= + +.. container:: metadata + + Posted by Dominik Picheta on 18/01/2016 + +Once again we are proud to announce the latest release of the Nim compiler +and related tools. This release comes just 3 months after the last +release! + +A new version of Nimble which depends on this release, has also been +released. See `this <http://forum.nim-lang.org/t/1912>`_ forum thread for +more information about the Nimble release. + +This release of Nim includes over 116 bug fixes, many of which are related +to closures. The lambda lifting algorithm in the compiler has been completely +rewritten, and some changes have been made to the semantics of closures in +Nim as a result. These changes may affect backwards compatibility and are all +described in the section below. + +With this release, we are one step closer to Nim version 1.0. +The 1.0 release will be a big milestone for Nim, because after that version +is released there will be no more breaking changes made to the language +or the standard library. + +That being said, the next release will likely be Nim 0.14. It will focus on +improvements to the GC and concurrency. We will in particular be looking at +ways to add multi-core support to async await. Standard library improvements +are also on our roadmap but may not make it for Nim 0.14. + +As always you can download the latest version of Nim from the +`download <download.html>`_ page. + +Happy coding! + +Changes affecting backwards compatibility +----------------------------------------- + +- ``macros.newLit`` for ``bool`` now produces false/true symbols which + actually work with the bool datatype. +- When compiling to JS: ``Node``, ``NodeType`` and ``Document`` are no longer + defined. Use the types defined in ``dom.nim`` instead. +- The check ``x is iterator`` (used for instance in concepts) was always a + weird special case (you could not use ``x is proc``) and was removed from + the language. +- Top level routines cannot have the calling convention ``closure`` + anymore. +- The ``redis`` module has been moved out of the standard library. It can + now be installed via Nimble and is located here: + https://github.com/nim-lang/redis +- ``math.RunningStat`` and its associated procs have been moved from + the ``math`` module to a new ``stats`` module. + + +Syntax changes +~~~~~~~~~~~~~~ + +The parser now considers leading whitespace in front of operators +to determine if an operator is used in prefix or infix position. +This means that finally ``echo $foo`` is parsed as people expect, +which is as ``echo($foo)``. It used to be parsed as ``(echo) $ (foo)``. + +``echo $ foo`` continues to be parsed as ``(echo) $ (foo)``. + +This also means that ``-1`` is always parsed as prefix operator so +code like ``0..kArraySize div 2 -1`` needs to be changed to +``0..kArraySize div 2 - 1``. + +This release also adds multi-line comments to Nim. The syntax for them is: +``#[ comment here ]#``. For more details read the section of +the `manual <docs/manual.html#lexical-analysis-multiline-comments>`_. + +Iterator changes +~~~~~~~~~~~~~~~~ + +Implicit return type inference for iterators has been removed from the language. The following used to work: + +.. code-block:: nim + iterator it = + yield 7 + +This was a strange special case and has been removed. Now you need to write it like so which is consistent with procs: + +.. code-block:: nim + iterator it: auto = + yield 7 + + +Closure changes +~~~~~~~~~~~~~~~ + +The semantics of closures changed: Capturing variables that are in loops do not produce a new environment. Nim closures behave like JavaScript closures now. + +The following used to work as the environment creation used to be attached to the loop body: + +.. code-block:: nim + + proc outer = + var s: seq[proc(): int {.closure.}] = @[] + for i in 0 ..< 30: + let ii = i + s.add(proc(): int = return ii*ii) + +This behaviour has changed in 0.13.0 and now needs to be written as: + +.. code-block:: nim + + proc outer = + var s: seq[proc(): int {.closure.}] = @[] + for i in 0 ..< 30: + (proc () = + let ii = i + s.add(proc(): int = return ii*ii))() + +The reason is that environment creations are now only performed once +per proc call. This change is subtle and unfortunate, but: + +1. Affects almost no code out there. +2. Is easier to implement and we are at a point in Nim's development process where simple+stable wins over perfect-in-theory+unstable-in-practice. +3. Implies programmers are more in control of where memory is allocated which is beneficial for a systems programming language. + +Bugfixes +-------- + +The list below has been generated based on the commits in Nim's git +repository. As such it lists only the issues which have been closed +via a commit, for a full list see +`this link on Github <https://github.com/nim-lang/Nim/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222015-10-27+..+2016-01-19%22+>`_. + +- Fixed "Generic arguments cannot be used in templates (raising undeclared identifier)" + (`#3498 <https://github.com/nim-lang/Nim/issues/3498>`_) +- Fixed "multimethods: Error: internal error: cgmeth.genConv" + (`#3550 <https://github.com/nim-lang/Nim/issues/3550>`_) +- Fixed "nimscript - SIGSEGV in except block" + (`#3546 <https://github.com/nim-lang/Nim/issues/3546>`_) +- Fixed "Bool literals in macros do not work." + (`#3541 <https://github.com/nim-lang/Nim/issues/3541>`_) +- Fixed "Docs: nativesocket.html - 404" + (`#3582 <https://github.com/nim-lang/Nim/issues/3582>`_) +- Fixed ""not nil" return types never trigger an error or warning" + (`#2285 <https://github.com/nim-lang/Nim/issues/2285>`_) +- Fixed "No warning or error is raised even if not nil is specified " + (`#3222 <https://github.com/nim-lang/Nim/issues/3222>`_) +- Fixed "Incorrect fsmonitor add() filter logic" + (`#3611 <https://github.com/nim-lang/Nim/issues/3611>`_) +- Fixed ""nimble install nimsuggest" failed" + (`#3622 <https://github.com/nim-lang/Nim/issues/3622>`_) +- Fixed "compile time `excl ` cause SIGSEGV" + (`#3639 <https://github.com/nim-lang/Nim/issues/3639>`_) +- Fixed "Unable to echo unsigned ints at compile-time" + (`#2514 <https://github.com/nim-lang/Nim/issues/2514>`_) +- Fixed "Nested closure iterator produces internal error" + (`#1725 <https://github.com/nim-lang/Nim/issues/1725>`_) +- Fixed "C Error on walkDirRec closure" + (`#3636 <https://github.com/nim-lang/Nim/issues/3636>`_) +- Fixed "Error in generated c code" + (`#3201 <https://github.com/nim-lang/Nim/issues/3201>`_) +- Fixed "C Compile-time error with generic proc type." + (`#2659 <https://github.com/nim-lang/Nim/issues/2659>`_) +- Fixed "ICE dereferencing array pointer" + (`#2240 <https://github.com/nim-lang/Nim/issues/2240>`_) +- Fixed "Lambda lifting crash" + (`#2007 <https://github.com/nim-lang/Nim/issues/2007>`_) +- Fixed "Can't reference outer variables from a closure in an iterator" + (`#2604 <https://github.com/nim-lang/Nim/issues/2604>`_) +- Fixed "M&S collector breaks with nested for loops." + (`#603 <https://github.com/nim-lang/Nim/issues/603>`_) +- Fixed "Regression: bad C codegen" + (`#3723 <https://github.com/nim-lang/Nim/issues/3723>`_) +- Fixed "JS backend - handle bool type in case statement" + (`#3722 <https://github.com/nim-lang/Nim/issues/3722>`_) +- Fixed "linenoise compilation with cpp" + (`#3720 <https://github.com/nim-lang/Nim/issues/3720>`_) +- Fixed "(???,???) duplicate case label" + (`#3665 <https://github.com/nim-lang/Nim/issues/3665>`_) +- Fixed "linenoise compilation with cpp" + (`#3720 <https://github.com/nim-lang/Nim/issues/3720>`_) +- Fixed "Update list of backward incompatibilities for Nim 0.12.0 in the main site" + (`#3689 <https://github.com/nim-lang/Nim/issues/3689>`_) +- Fixed "Can't compile nimble with latest devel - codegen bug" + (`#3730 <https://github.com/nim-lang/Nim/issues/3730>`_) diff --git a/web/news/2016_01_27_nim_in_action_is_now_available.rst b/web/news/2016_01_27_nim_in_action_is_now_available.rst new file mode 100644 index 000000000..33bcb7947 --- /dev/null +++ b/web/news/2016_01_27_nim_in_action_is_now_available.rst @@ -0,0 +1,32 @@ +Nim in Action is now available! +=============================== + +.. container:: metadata + + Posted by Dominik Picheta on 27/01/2016 + +.. raw::html + + <a href="https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81"> + <img src="../assets/niminaction/banner.jpg" alt="New in Manning Early Access Program: Nim in Action!" width="682"/> + </a> + +We are proud to announce that *Nim in Action*, a book about the Nim programming +language, is now available! + +The book is available at this URL: +`https://www.manning.com/books/nim-in-action <https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81>`_ + +The first three chapters are available for download +as an eBook through Manning's Early Access program. You can download a free +sample of the book containing the first chapter as well! + +*Nim in Action* is currently being written and is expected to be completed by +Summer 2016. If you purchase the eBook you will start receiving new chapters +as they become available. You can also purchase the printed book together with +the eBook for a slightly higher price. + +If you do read the book, even if it's just the first chapter, then please share +any comments, suggestions and questions on the +`Nim forum <http://forum.nim-lang.org/t/1978>`_ or in +Manning's own `Author Online forum! <https://forums.manning.com/forums/nim-in-action>`_ diff --git a/web/news/2016_06_04_meet_our_bountysource_sponsors.rst b/web/news/2016_06_04_meet_our_bountysource_sponsors.rst new file mode 100644 index 000000000..0bfb472c5 --- /dev/null +++ b/web/news/2016_06_04_meet_our_bountysource_sponsors.rst @@ -0,0 +1,30 @@ +Meet our BountySource sponsors +============================== + +.. container:: metadata + + Posted by Dominik Picheta on 04/06/2016 + + +.. raw::html + + <a href="../sponsors.html"> + <img src="../assets/bountysource/meet_sponsors.png" alt="Meet our BountySource sponsors!" width="400"/> + </a> + +It has now been two months since we began our +`BountySource fundraiser <https://salt.bountysource.com/teams/nim>`_. We +promised to create a "Current sponsors" page and are happy to announce that +it is now live `here <http://nim-lang.org/sponsors.html>`_. + +We are happy to say that last month we raised more than in the previous month! +A staggering $862 was raised during that month and we thank each and every one +of you once again for your generous contributions. + +We are gearing up for a brand new release of Nim that includes many bug fixes +and some new features. In the meantime, as always, feel free to get in +touch with us via `Twitter <https://twitter.com/nim_lang>`_, the +`#nim channel on Freenode <http://webchat.freenode.net/?channels=nim>`_ +or via email at contact@nim-lang.org. + +Thanks for reading! diff --git a/web/news/2016_06_07_version_0_14_0_released.rst b/web/news/2016_06_07_version_0_14_0_released.rst new file mode 100644 index 000000000..6634d0053 --- /dev/null +++ b/web/news/2016_06_07_version_0_14_0_released.rst @@ -0,0 +1,476 @@ +Version 0.14.0 released +======================= + +.. container:: metadata + + Posted by Dominik Picheta on 07/06/2016 + +It's been a while since the last release, but we've been very busy in the +meantime. In +addition to working on Nim we have started a +`BountySource campaign <https://salt.bountysource.com/teams/nim>`_ and +announced the pre-release of a new Nim book titled +`Nim in Action <https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81>`_. +Our BountySource campaign has already been very successful, helping us raise +enough funds to surpass 4 of our monthly goals. The companies and individuals +that made this possible are listed on our brand new +`sponsors page <http://nim-lang.org/sponsors.html>`_. + +This release includes over 260 bug fixes. As mentioned in the previous release +announcement, one of the focuses of this release was going to be improvements +to the GC. Indeed, the most prominent fixes are related to the GC not collecting +cycles properly. This was a major problem that was triggered typically when +applications using asynchronous I/O were left running for long periods of time. + +There have also been many fixes to the way that the compiler sources are +installed. Some applications such as Nimble depend on these sources and they +are now included in the release tarballs. This should fix many of the problems +that users experienced trying to compile the Nimble package manager. + +Finally, you will find multiple changes in the standard library. Some of which +unfortunately affects backwards compatibility. This includes the ``random`` +procedures being moved to a new ``random`` module, HTTP headers being stored +in a new ``HttpHeaders`` object and the ``round`` procedure in the ``math`` module +being changed to return a ``float`` instead of an ``int``. You can find a full +list of such changes below. + +Together with the new release of Nim, we are also releasing a new version of +Nimble. The release notes for it are available on +`GitHub <https://github.com/nim-lang/nimble/blob/master/changelog.markdown#074---06062016>`_. + +As always you can download the latest version of Nim from the +`download <http://nim-lang.org/download.html>`_ page. + +We hope that you will like this new release. Let us know if you run into +any trouble, have any questions or want to give some feedback. You can get +in touch with us on the `Forum <http://forum.nim-lang.org/>`_, +`IRC <http://webchat.freenode.net/?channels=nim>`_, +`Twitter <http://twitter.com/nim_lang>`_, +or via email contact@nim-lang.org. + +Happy coding! + +Changes affecting backwards compatibility +----------------------------------------- + +- ``--out`` and ``--nimcache`` command line arguments are now relative to + current directory. Previously they were relative to project directory. +- The json module now stores the name/value pairs in objects internally as a + hash table of type ``fields*: Table[string, JsonNode]`` instead of a + sequence. This means that order is no longer preserved. When using the + ``table.mpairs`` iterator only the returned values can be modified, no + longer the keys. +- The deprecated Nim shebang notation ``#!`` was removed from the language. Use ``#?`` instead. +- Typeless parameters have been removed from the language since it would + clash with ``using``. +- Procedures in ``mersenne.nim`` (Mersenne Twister implementation) no longer + accept and produce ``int`` values which have platform-dependent size - + they use ``uint32`` instead. +- The ``strutils.unindent`` procedure has been rewritten. Its parameters now + match the parameters of ``strutils.indent``. See issue `#4037 <https://github.com/nim-lang/Nim/issues/4037>`_ + for more details. +- The ``matchers`` module has been deprecated. See issue `#2446 <https://github.com/nim-lang/Nim/issues/2446>`_ + for more details. +- The ``json.[]`` no longer returns ``nil`` when a key is not found. Instead it + raises a ``KeyError`` exception. You can compile with the ``-d:nimJsonGet`` + flag to get a list of usages of ``[]``, as well as to restore the operator's + previous behaviour. +- When using ``useMalloc``, an additional header containing the size of the + allocation will be allocated, to support zeroing memory on realloc as expected + by the language. With this change, ``alloc`` and ``dealloc`` are no longer + aliases for ``malloc`` and ``free`` - use ``c_malloc`` and ``c_free`` if + you need that. +- The ``json.%`` operator is now overloaded for ``object``, ``ref object`` and + ``openarray[T]``. +- The procs related to ``random`` number generation in ``math.nim`` have + been moved to its own ``random`` module and been reimplemented in pure + Nim. +- The path handling changed. The project directory is not added to the + search path automatically anymore. Add this line to your project's + config to get back the old behaviour: ``--path:"$projectdir"``. (The compiler + replaces ``$projectdir`` with your project's absolute directory when compiling, + so you don't need to replace ``$projectdir`` by your project's actual + directory!). See issue `#546 <https://github.com/nim-lang/Nim/issues/546>`_ + and `this forum thread <http://forum.nim-lang.org/t/2277>`_ for more + information. +- The ``round`` function in ``math.nim`` now returns a float and has been + corrected such that the C implementation always rounds up from .5 rather + than changing the operation for even and odd numbers. +- The ``round`` function now accepts a ``places`` argument to round to a + given number of places (e.g. round 4.35 to 4.4 if ``places`` is 1). +- In ``strutils.nim``, ``formatSize`` now returns a number representing the + size in conventional decimal format (e.g. 2.234GB meaning 2.234 GB rather + than meaning 2.285 GB as in the previous implementation). By default it + also uses IEC prefixes (KiB, MiB) etc and optionally uses colloquial names + (kB, MB etc) and the (SI-preferred) space. +- The ``==`` operator for ``cstring`` now implements a value comparison + for the C backend (using ``strcmp``), not reference comparisons anymore. + Convert the cstrings to pointers if you really want reference equality + for speed. +- HTTP headers are now stored in a ``HttpHeaders`` object instead of a + ``StringTableRef``. This object allows multiple values to be associated with + a single key. A new ``httpcore`` module implements it and it is used by + both ``asynchttpserver`` and ``httpclient``. + +The ``using`` statement +~~~~~~~~~~~~~~~~~~~~~~~ + +The ``using`` statement now has a different meaning. + +In version 0.13.0, it +was used to provide syntactic convenience for procedures that heavily use +a single contextual parameter. For example: + +.. code-block:: nim + var socket = newSocket() + using socket + + connect("google.com", Port(80)) + send("GET / HTTP/1.1\c\l") + + +The ``connect`` and ``send`` calls are both transformed so that they pass +``socket`` as the first argument: + +.. code-block:: nim + var socket = newSocket() + + socket.connect("google.com", Port(80)) + socket.send("GET / HTTP/1.1\c\l") + +Take a look at the old version of the +`manual <http://nim-lang.org/0.13.0/manual.html#statements-and-expressions-using-statement>`_ +to learn more about the old behaviour. + +In 0.14.0, +the ``using`` statement +instead provides a syntactic convenience for procedure definitions where the +same parameter names and types are used repeatedly. For example, instead of +writing: + +.. code-block:: nim + proc foo(c: Context; n: Node) = ... + proc bar(c: Context; n: Node, counter: int) = ... + proc baz(c: Context; n: Node) = ... + + +You can simply write: + +.. code-block:: nim + {.experimental.} + using + c: Context + n: Node + counter: int + + proc foo(c, n) = ... + proc bar(c, n, counter) = ... + proc baz(c, n) = ... + +Again, the +`manual <http://nim-lang.org/docs/manual.html#statements-and-expressions-using-statement>`_ +has more details. + +You can still achieve a similar effect to what the old ``using`` statement +tried to achieve by using the new experimental ``this`` pragma, documented +`here <http://nim-lang.org/docs/manual.html#overloading-resolution-automatic-self-insertions>`_. + +Generic type classes +~~~~~~~~~~~~~~~~~~~~ + +Generic type classes are now handled properly in the compiler, but this +means code like the following does not compile any longer: + +.. code-block:: nim + type + Vec3[T] = distinct array[3, T] + + proc vec3*[T](a, b, c: T): Vec3[T] = Vec3([a, b, c]) + +While every ``Vec3[T]`` is part of the ``Vec3`` type class, the reverse +is not true, not every ``Vec3`` is a ``Vec3[T]``. Otherwise there would +be a subtype relation between ``Vec3[int]`` and ``Vec3[float]`` and there +is none for Nim. The fix is to write this instead: + +.. code-block:: nim + type + Vec3[T] = distinct array[3, T] + + proc vec3*[T](a, b, c: T): Vec3[T] = Vec3[T]([a, b, c]) + +Note that in general we don't advise to use ``distinct array``, +use ``object`` instead. + + +Library Additions +----------------- + +- The rlocks module has been added providing a reentrant lock synchronization + primitive. +- A generic "sink operator" written as ``&=`` has been added to the +``system`` and the ``net`` modules. This operator is similar to the C++ +``<<`` operator which writes data to a stream. +- Added ``strscans`` module that implements a ``scanf`` for easy input extraction. +- Added a version of ``parseutils.parseUntil`` that can deal with a string + ``until`` token. The other versions are for ``char`` and ``set[char]``. +- Added ``splitDecimal`` to ``math.nim`` to split a floating point value + into an integer part and a floating part (in the range -1<x<1). +- Added ``trimZeros`` to ``strutils.nim`` to trim trailing zeros in a + floating point number. +- Added ``formatEng`` to ``strutils.nim`` to format numbers using engineering + notation. + + +Compiler Additions +------------------ + +- Added a new ``--noCppExceptions`` switch that allows to use default exception + handling (no ``throw`` or ``try``/``catch`` generated) when compiling to C++ + code. + +Language Additions +------------------ + +- Nim now supports a ``.this`` pragma for more notational convenience. + See `automatic-self-insertions <../docs/manual.html#overloading-resolution-automatic-self-insertions>`_ for more information. +- Nim now supports a different ``using`` statement for more convenience. + Consult `using-statement <../docs/manual.html#statements-and-expressions-using-statement>`_ for more information. +- ``include`` statements are not restricted to top level statements anymore. + +.. + - Nim now supports ``partial`` object declarations to mitigate the problems + that arise when types are mutually dependent and yet should be kept in + different modules. + +Bugfixes +-------- + +The list below has been generated based on the commits in Nim's git +repository. As such it lists only the issues which have been closed +via a commit, for a full list see +`this link on Github <https://github.com/nim-lang/Nim/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222016-01-19+..+2016-06-06%22+>`_. + + + - Fixed "Calling generic templates with explicit generic arguments crashes compiler" + (`#3496 <https://github.com/nim-lang/Nim/issues/3496>`_) + - Fixed "JS backend - strange utf-8 handling" + (`#3714 <https://github.com/nim-lang/Nim/issues/3714>`_) + - Fixed "execvpe is glibc specific" + (`#3759 <https://github.com/nim-lang/Nim/issues/3759>`_) + - Fixed "GC stack overflow with in data structures with circular references." + (`#1895 <https://github.com/nim-lang/Nim/issues/1895>`_) + - Fixed "Internal compiler error in genTraverseProc" + (`#3794 <https://github.com/nim-lang/Nim/issues/3794>`_) + - Fixed "unsafeAddr fails in generic context" + (`#3736 <https://github.com/nim-lang/Nim/issues/3736>`_) + - Fixed "Generic converters produce internal errors" + (`#3799 <https://github.com/nim-lang/Nim/issues/3799>`_) + - Fixed "Cannot have two anonymous iterators in one proc" + (`#3788 <https://github.com/nim-lang/Nim/issues/3788>`_) + - Fixed "pure/net.nim fails to compile with --taintMode:on on HEAD" + (`#3789 <https://github.com/nim-lang/Nim/issues/3789>`_) + - Fixed "Using break inside iterator may produce memory/resource leak" + (`#3802 <https://github.com/nim-lang/Nim/issues/3802>`_) + + - Fixed "--out and --nimcache wrong paths" + (`#3871 <https://github.com/nim-lang/Nim/issues/3871>`_) + - Fixed "Release 0.13.0: documentation build failure" + (`#3823 <https://github.com/nim-lang/Nim/issues/3823>`_) + - Fixed "https post request" + (`#3895 <https://github.com/nim-lang/Nim/issues/3895>`_) + - Fixed "writeFile regression in nimscript" + (`#3901 <https://github.com/nim-lang/Nim/issues/3901>`_) + - Fixed "Cannot convert variables to int16 at compile time" + (`#3916 <https://github.com/nim-lang/Nim/issues/3916>`_) + - Fixed "Error in concepts when using functions on typedesc" + (`#3686 <https://github.com/nim-lang/Nim/issues/3686>`_) + - Fixed "Multiple generic table types with different type signatures lead to compilation errors." + (`#3669 <https://github.com/nim-lang/Nim/issues/3669>`_) + - Fixed "Explicit arguments with overloaded procedure?" + (`#3836 <https://github.com/nim-lang/Nim/issues/3836>`_) + - Fixed "doc2 generates strange output for proc generated by template" + (`#3868 <https://github.com/nim-lang/Nim/issues/3868>`_) + - Fixed "Passing const value as static[] argument to immediate macro leads to infinite memory consumption by compiler" + (`#3872 <https://github.com/nim-lang/Nim/issues/3872>`_) + - Fixed "`..<` is not happy with `BiggestInt` from `intVal`" + (`#3767 <https://github.com/nim-lang/Nim/issues/3767>`_) + - Fixed "stdtmpl filter does not support anything apart from '#' metachar" + (`#3924 <https://github.com/nim-lang/Nim/issues/3924>`_) + - Fixed "lib/pure/net: Can't bind to ports >= 32768" + (`#3484 <https://github.com/nim-lang/Nim/issues/3484>`_) + - Fixed "int and float assignment compatibility badly broken for generics" + (`#3998 <https://github.com/nim-lang/Nim/issues/3998>`_) + - Fixed "Adding echo statement causes "type mismatch" error" + (`#3975 <https://github.com/nim-lang/Nim/issues/3975>`_) + - Fixed "Dynlib error messages should be written to stderr, not stdout" + (`#3987 <https://github.com/nim-lang/Nim/issues/3987>`_) + - Fixed "Tests regressions while using the devel branch" + (`#4005 <https://github.com/nim-lang/Nim/issues/4005>`_) + + - Fixed "Lambda lifting bug: wrong c code generation" + (`#3995 <https://github.com/nim-lang/Nim/issues/3995>`_) + - Fixed "VM crashes in asgnComplex" + (`#3973 <https://github.com/nim-lang/Nim/issues/3973>`_) + - Fixed "Unknown opcode opcNGetType" + (`#1152 <https://github.com/nim-lang/Nim/issues/1152>`_) + - Fixed "`&` operator mutates first operand when used in compileTime proc while assigning result to seq" + (`#3804 <https://github.com/nim-lang/Nim/issues/3804>`_) + - Fixed "''nil' statement is deprecated' in macro" + (`#3561 <https://github.com/nim-lang/Nim/issues/3561>`_) + - Fixed "vm crash when accessing seq with mitems iterator" + (`#3731 <https://github.com/nim-lang/Nim/issues/3731>`_) + - Fixed "`mitems` or `mpairs` does not work for `seq[NimNode]` or `array[T,NimNode]` in a macro" + (`#3859 <https://github.com/nim-lang/Nim/issues/3859>`_) + - Fixed "passing "proc `,`()" to nim check causes an infinite loop" + (`#4036 <https://github.com/nim-lang/Nim/issues/4036>`_) + - Fixed "--dynlibOverride does not work with {.push dynlib: name.}" + (`#3646 <https://github.com/nim-lang/Nim/issues/3646>`_) + - Fixed "system.readChars fails on big len" + (`#3752 <https://github.com/nim-lang/Nim/issues/3752>`_) + - Fixed "strutils.unindent" + (`#4037 <https://github.com/nim-lang/Nim/issues/4037>`_) + - Fixed "Compiler's infinite recursion in generic resolution" + (`#2006 <https://github.com/nim-lang/Nim/issues/2006>`_) + - Fixed "Linux: readLineFromStdin calls quit(0) upon EOF" + (`#3159 <https://github.com/nim-lang/Nim/issues/3159>`_) + - Fixed "Forum sign up not possible" + (`#2446 <https://github.com/nim-lang/Nim/issues/2446>`_) + - Fixed "Json module - SIGSEGV if key not exists" + (`#3107 <https://github.com/nim-lang/Nim/issues/3107>`_) + - Fixed "About asyncdispatch.await and exception" + (`#3964 <https://github.com/nim-lang/Nim/issues/3964>`_) + - Fixed "Need testcase for JS backend to ensure closure callbacks don't break" + (`#3132 <https://github.com/nim-lang/Nim/issues/3132>`_) + - Fixed "Unexpected behaviour of C++ templates in conjunction with N_NIMCALL" + (`#4093 <https://github.com/nim-lang/Nim/issues/4093>`_) + - Fixed "SIGSEGV at compile time when using a compileTime variable as counter" + (`#4097 <https://github.com/nim-lang/Nim/issues/4097>`_) + - Fixed "Compiler crash issue on 32-bit machines only" + (`#4089 <https://github.com/nim-lang/Nim/issues/4089>`_) + - Fixed "type mismatch: got (<type>) but expected 'outType' in mapIt" + (`#4124 <https://github.com/nim-lang/Nim/issues/4124>`_) + - Fixed "Generic type constraints broken?" + (`#4084 <https://github.com/nim-lang/Nim/issues/4084>`_) + - Fixed "Invalid C code generated" + (`#3544 <https://github.com/nim-lang/Nim/issues/3544>`_) + - Fixed "An exit variable in proc shadows exit function called by quit()" + (`#3471 <https://github.com/nim-lang/Nim/issues/3471>`_) + - Fixed "ubuntu 16.04 build error" + (`#4144 <https://github.com/nim-lang/Nim/issues/4144>`_) + - Fixed "Ambiguous identifier error should list all possible qualifiers" + (`#177 <https://github.com/nim-lang/Nim/issues/177>`_) + - Fixed "Parameters are not captured inside closures inside closure iterators" + (`#4070 <https://github.com/nim-lang/Nim/issues/4070>`_) + - Fixed "`$` For array crashes the compiler when assigned to const" + (`#4040 <https://github.com/nim-lang/Nim/issues/4040>`_) + + - Fixed "Default value for .importcpp enum is initialized incorrectly" + (`#4034 <https://github.com/nim-lang/Nim/issues/4034>`_) + - Fixed "Nim doesn't instantiate template parameter in cgen when using procedure return value in for-in loop" + (`#4110 <https://github.com/nim-lang/Nim/issues/4110>`_) + - Fixed "Compile-time SIGSEGV when invoking procedures that cannot be evaluated at compile time from a macro" + (`#3956 <https://github.com/nim-lang/Nim/issues/3956>`_) + - Fixed "Backtricks inside .emit pragma output incorrect name for types" + (`#3992 <https://github.com/nim-lang/Nim/issues/3992>`_) + - Fixed "typedef is generated for .importcpp enums" + (`#4145 <https://github.com/nim-lang/Nim/issues/4145>`_) + - Fixed "Incorrect C code generated for nnkEmpty node" + (`#950 <https://github.com/nim-lang/Nim/issues/950>`_) + - Fixed "Syntax error in config file appears as general exception without useful info" + (`#3763 <https://github.com/nim-lang/Nim/issues/3763>`_) + - Fixed "Converting .importcpp enum to string doesn't work when done inside procs" + (`#4147 <https://github.com/nim-lang/Nim/issues/4147>`_) + - Fixed "Enum template specifiers do not work for .importcpp enums when they are used as a parameter" + (`#4146 <https://github.com/nim-lang/Nim/issues/4146>`_) + - Fixed "Providing template specifier recursively for .importcpp type doesn't work" + (`#4148 <https://github.com/nim-lang/Nim/issues/4148>`_) + - Fixed "sizeof doesn't work for generics in vm" + (`#4153 <https://github.com/nim-lang/Nim/issues/4153>`_) + - Fixed "Creating list-like structures in a loop leaks memory indefinitely" + (`#3793 <https://github.com/nim-lang/Nim/issues/3793>`_) + - Fixed "Creating list-like structures in a loop leaks memory indefinitely" + (`#3793 <https://github.com/nim-lang/Nim/issues/3793>`_) + - Fixed "Enum items generated by a macro have wrong type." + (`#4066 <https://github.com/nim-lang/Nim/issues/4066>`_) + - Fixed "Memory leak with default GC" + (`#3184 <https://github.com/nim-lang/Nim/issues/3184>`_) + - Fixed "Rationals Overflow Error on 32-bit machine" + (`#4194 <https://github.com/nim-lang/Nim/issues/4194>`_) + + - Fixed "osproc waitForExit() is ignoring the timeout parameter" + (`#4200 <https://github.com/nim-lang/Nim/issues/4200>`_) + - Fixed "Regression: exception parseFloat("-0.0") " + (`#4212 <https://github.com/nim-lang/Nim/issues/4212>`_) + - Fixed "JS Codegen: Bad constant initialization order" + (`#4222 <https://github.com/nim-lang/Nim/issues/4222>`_) + - Fixed "Term-rewriting macros gives Error: wrong number of arguments" + (`#4227 <https://github.com/nim-lang/Nim/issues/4227>`_) + - Fixed "importcpp allowed in body of proc after push" + (`#4225 <https://github.com/nim-lang/Nim/issues/4225>`_) + - Fixed "pragma SIGSEGV" + (`#4001 <https://github.com/nim-lang/Nim/issues/4001>`_) + - Fixed "Restrict hints to the current project" + (`#2159 <https://github.com/nim-lang/Nim/issues/2159>`_) + - Fixed "`unlikely`/`likely` should be no-ops for the Javascript backend" + (`#3882 <https://github.com/nim-lang/Nim/issues/3882>`_) + - Fixed ".this pragma doesn't work for fields and procs defined for parent type" + (`#4177 <https://github.com/nim-lang/Nim/issues/4177>`_) + - Fixed "VM SIGSEV with compile-time Table" + (`#3729 <https://github.com/nim-lang/Nim/issues/3729>`_) + - Fixed "Error during compilation with cpp option on FreeBSD " + (`#3059 <https://github.com/nim-lang/Nim/issues/3059>`_) + - Fixed "Compiler doesn't keep type bounds" + (`#1713 <https://github.com/nim-lang/Nim/issues/1713>`_) + - Fixed "Stdlib: future: Shortcut proc definition doesn't support, varargs, seqs, arrays, or openarrays" + (`#4238 <https://github.com/nim-lang/Nim/issues/4238>`_) + - Fixed "Why don't ``asynchttpserver`` support request-body when ``put`` ``delete``?" + (`#4221 <https://github.com/nim-lang/Nim/issues/4221>`_) + - Fixed "Paths for includes in Nim documentation" + (`#2640 <https://github.com/nim-lang/Nim/issues/2640>`_) + - Fixed "Compile pragma doesn't work with relative import" + (`#1262 <https://github.com/nim-lang/Nim/issues/1262>`_) + - Fixed "Slurp doesn't work with relative imports" + (`#765 <https://github.com/nim-lang/Nim/issues/765>`_) + - Fixed "Make tilde expansion consistent" + (`#786 <https://github.com/nim-lang/Nim/issues/786>`_) + - Fixed "koch expects nim to be in path for tests?" + (`#3290 <https://github.com/nim-lang/Nim/issues/3290>`_) + - Fixed "Don't use relative imports for non relative modules (aka babel libs)" + (`#546 <https://github.com/nim-lang/Nim/issues/546>`_) + - Fixed ""echo" on general structs does not work" + (`#4236 <https://github.com/nim-lang/Nim/issues/4236>`_) + - Fixed "Changing math.round() and adding math.integer()" + (`#3473 <https://github.com/nim-lang/Nim/issues/3473>`_) + - Fixed "Mathematics module missing modf" + (`#4195 <https://github.com/nim-lang/Nim/issues/4195>`_) + - Fixed "Passing method to macro causes seg fault" + (`#1611 <https://github.com/nim-lang/Nim/issues/1611>`_) + - Fixed "Internal error with "discard quit"" + (`#3532 <https://github.com/nim-lang/Nim/issues/3532>`_) + - Fixed "SIGSEGV when using object variant in compile time" + (`#4207 <https://github.com/nim-lang/Nim/issues/4207>`_) + - Fixed "formatSize has incorrect prefix" + (`#4198 <https://github.com/nim-lang/Nim/issues/4198>`_) + - Fixed "Add compiler parameter to generate output from source code filters" + (`#375 <https://github.com/nim-lang/Nim/issues/375>`_) + - Fixed "Add engineering notation to string formatting functions" + (`#4197 <https://github.com/nim-lang/Nim/issues/4197>`_) + - Fixed "Very minor error in json documentation" + (`#4255 <https://github.com/nim-lang/Nim/issues/4255>`_) + - Fixed "can't compile when checking if closure == nil" + (`#4186 <https://github.com/nim-lang/Nim/issues/4186>`_) + - Fixed "Strange code gen for procs returning arrays" + (`#2259 <https://github.com/nim-lang/Nim/issues/2259>`_) + - Fixed "asynchttpserver may consume unbounded memory reading headers" + (`#3847 <https://github.com/nim-lang/Nim/issues/3847>`_) + + - Fixed "download page still implies master is default branch" + (`#4022 <https://github.com/nim-lang/Nim/issues/4022>`_) + - Fixed "Use standard compiler flags in build script" + (`#2128 <https://github.com/nim-lang/Nim/issues/2128>`_) + - Fixed "CentOS 6 (gcc-4.4.7) compilation failed (redefinition of typedef)" + (`#4272 <https://github.com/nim-lang/Nim/issues/4272>`_) + - Fixed "doc2 has issues with httpclient" + (`#4278 <https://github.com/nim-lang/Nim/issues/4278>`_) + - Fixed "tuples/tuple_with_nil fails without unsigned module" + (`#3579 <https://github.com/nim-lang/Nim/issues/3579>`_) diff --git a/web/news/2016_06_11_version_0_14_2_released.rst b/web/news/2016_06_11_version_0_14_2_released.rst new file mode 100644 index 000000000..cbfe52713 --- /dev/null +++ b/web/news/2016_06_11_version_0_14_2_released.rst @@ -0,0 +1,14 @@ +Version 0.14.2 released +======================= + +.. container:: metadata + + Posted by Andreas Rumpf on 09/06/2016 + +Version 0.14.2 is just a bugfix release that fixes the most pressing +regressions. In particular, the ``tar.xz`` now supports documentation +generation, and the Windows installers bundle the latest stable nimble +release. + +The news about the 0.14.0 release are still relevant, so check them out +`here <http://nim-lang.org/news/2016_06_07_version_0_14_0_released.html>`_. diff --git a/web/news/2016_06_23_launching_the_2016_nim_community_survey.rst b/web/news/2016_06_23_launching_the_2016_nim_community_survey.rst new file mode 100644 index 000000000..0b87577aa --- /dev/null +++ b/web/news/2016_06_23_launching_the_2016_nim_community_survey.rst @@ -0,0 +1,29 @@ +Launching the 2016 Nim Community Survey +======================================= + +.. container:: metadata + + Posted by Dominik Picheta on 23/06/2016 + +We are proud to announce the official +`2016 Nim Community Survey <http://goo.gl/forms/XJ3TPsaiIQe5HlTB2>`_! No matter +whether you use Nim today, have used Nim previously, or never used Nim before; +we want to know your opinions. +Your feedback will help the Nim project understand its strengths and +weaknesses, and to determine development priorities for the future. + +It shouldn't take you much longer than 5 to 10 minutes to complete this survey. +Submissions will be accepted until around the 23rd of July, depending on the +response rates. If you have any questions or feedback, please don't hesitate +to get in touch with us via email at survey@nim-lang.org or on the +`Nim Forum <http://forum.nim-lang.org>`_. + +We would appreciate your help in spreading the word about this survey. Share +the above link on your social network feeds, with your colleagues and in +other communities. + +Thank you to everyone that helped develop and test the survey! Once the +submission period ends, the results will be shown here and publicised via +Twitter, the Nim Forum and IRC. + +Thanks for your time! diff --git a/web/news/2016_08_06_bountysource_update_the_road_to_v10.rst b/web/news/2016_08_06_bountysource_update_the_road_to_v10.rst new file mode 100644 index 000000000..00ca7022e --- /dev/null +++ b/web/news/2016_08_06_bountysource_update_the_road_to_v10.rst @@ -0,0 +1,73 @@ +BountySource Update: The Road to v1.0 +===================================== + +.. container:: metadata + + Cross posted by Dominik Picheta on 06/08/2016 from + `Update #4 on BountySource <https://salt.bountysource.com/teams/nim/updates/4-the-road-to-v1-0>`_ + +We are now in the fourth month of the +`Nim BountySource fundraiser <https://salt.bountysource.com/teams/nim>`_ and +here is +this month's update[1]. Once again this month, we have beat our previous +donation record of $1280 having raised over $1600 over the course of July! +That's now 4 months in a row that your monthly donations have been increasing. +As always we are absolutely blown away by your contributions, +`myself <https://github.com/dom96>`_ and the +rest of the Nim team are extremely thankful for them. It's not only helping us +pay for the necessary expenses (like for example the server that +http://nim-lang.org runs on) but it also inspires us to keep going and to make +Nim the best programming language that it can be. + +As mentioned in +`last month's update <https://salt.bountysource.com/teams/nim/updates/3-engaging-with-our-community>`_, we have begun the process of engaging with +the Nim community through a survey. This survey has now been open for more than +a month and will be closing very soon (in about 2 days), so if you haven't +answered it yet, now would be a perfect time to do so. You can find the survey +here: http://nim-lang.org/survey. + +The survey itself has been designed for three types of people: Nim users, +ex-Nim users and people who have never used Nim before. This means that you +have no excuse not to answer it[2]. There are almost 700 submissions and after +the survey is finalised, a blog post will be written with a thorough analysis. + +It is my hope that the survey analysis will give the Nim team a good idea of +what needs to be implemented before version 1.0 can be released. Personally, I +hope to make a thorough review of the standard library to ensure that it is +ready for the "1.0 backwards compatibility lock"[3]. Although I myself have +been very busy lately[4], `Araq <http://github.com/Araq>`_ has been working very hard to fix +`High Priority <https://github.com/nim-lang/Nim/issues?q=is%3Aissue+is%3Aopen+label%3A%22High+Priority%22>`_ +issues, ahead of the 1.0 release. And as always, there has also been a +`lot of pull requests <https://github.com/nim-lang/Nim/pulse>`_ +from a wide range of Nim users. + +Lastly, I would like to mention +`Nim in Action <https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81>`_ once again. I have recently +finished the final chapter. All that remains now are corrections (based mainly +on your feedback, huge thanks to everyone for reading!) and a final review. +The book will then be put into production[5] with an estimated print date of +around the 20th of January (as usual delays are very much possible). If you +have not yet picked up the book, now would be a perfect time to do so. There +is still time for you to give feedback about the book, and for me to improve it +based on your remarks. It is not often that you can influence a book in this +way, and it is my hope that you will help me make this book one of the best +resources for learning Nim! + +As always, many thanks for reading and if you have any questions or feedback +feel free to get in touch via email at contact@nim-lang.org or via +`Twitter <https://twitter.com/nim_lang>`_. + +1 - These updates have so far been fairly regular and it is my hope to write +at least one a month. Yep, I am patting myself on the back :) + +2 - I joke of course, don't worry about if you don't have the time :) + +3 - After version 1.0, it is typical for a piece of software (especially +a programming language) to ensure that backwards compatibility is not broken +between further minor versions until a new major version such as 2.0 is released. + +4 - Writing a book, moving to Switzerland to start a new job, and playing +`Pokemon Go <https://pokemongostatus.org/>`_ has certainly kept me busy. + +5 - Basically the brilliant guys at `Manning <https://manning.com>`_ will +process the book so that it looks good in a printed format. diff --git a/web/news/nim_community_survey_results.rst b/web/news/nim_community_survey_results.rst new file mode 100644 index 000000000..49656f20a --- /dev/null +++ b/web/news/nim_community_survey_results.rst @@ -0,0 +1,317 @@ +Nim Community Survey Results +============================ + +.. container:: metadata + + Posted by Dominik Picheta on 20/08/2016 + +We have recently closed the 2016 Nim Community Survey. I am happy to +say that we have received exactly 790 responses, huge thanks go to the people +that took the time to respond. We're very thankful for this very valuable +feedback. + +This survey was inspired in part by the +`2016 State of Rust <https://blog.rust-lang.org/2016/06/30/State-of-Rust-Survey-2016.html>`_ +survey. You will note that many of the questions were modelled after +Rust's survey. One of the reasons for doing this was to allow us to easily +compare our results against the results obtained in the Rust survey. In +addition, we of course also liked many of their questions. + +Our survey ran from the 23rd of June 2016 until the 8th of August 2016. The +response numbers are impressive considering Nim's community size; at 790 they +make up just over 25% of the Rust survey's responses. + +The goal of this survey was to primarily determine how our community is using +Nim, in order to better understand how we should be improving it. In particular, +we wanted to know what people feel is missing from Nim in the lead up to +version 1.0. We have also asked our respondents about how well the Nim tools +worked, the challenges of adopting Nim, the resources that they used to learn +Nim and more. + +It is my hope that we will be able to run a similar survey in a years time, +doing so should give us an idea of whether we are improving. +With these general facts in mind, let's begin looking at specific questions. + +How did you find out about Nim? +------------------------------- + +The rationale for the first question was simple, we wanted to know where our +respondents found out about Nim. This is an interesting question for us, as +we do occassionally get users asking us why it took so long for them to hear +about Nim. It allows us to see how effective each website is at spreading the +word about Nim. + +.. raw::html + + <a href="../assets/news/images/survey/nim_found.png"> + <img src="../assets/news/images/survey/nim_found.png" alt="How did you find out about Nim?" style="width:100%"/> + </a> + +The majority of our respondents found Nim via Reddit, HackerNews or a search +engine such as Google. These results are not altogether surprising. There were +also a lot of "Other" responses, some of which were a bit more +interesting. These included multiple mentions of habrahabr.ru, Dr. Dobb's, +and lobste.rs. + +Do you use Nim? +--------------- + +Just like the Rust survey creators, we wanted to ensure that our survey was +open to both Nim users as well people who never used Nim. In addition to +those two groups, we have also included a third group of people: ex-Nim +users. All three are interesting, for many different reasons. +Nim users can tell us how they are using Nim and also how Nim's +tooling can improve. Ex-Nim users give us an +idea of why they stopped using Nim. Finally, respondents who never used Nim +can tell us the reasons for not adopting it. + +.. raw::html + + <a href="../assets/news/images/survey/do_you_use_nim.png"> + <img src="../assets/news/images/survey/do_you_use_nim.png" alt="Do you use Nim?" style="width:100%"/> + </a> + +It's nice to see that we have such a good range of respondents. The Rust survey +had a much larger number of Rust users amongst their respondents, with +no distinction between users that never used Rust and users that stopped using +Rust. + +.. raw::html + + <a href="https://blog.rust-lang.org/images/2016-06-Survey/do_you_use_rust.png"> + <img src="https://blog.rust-lang.org/images/2016-06-Survey/do_you_use_rust.png" alt="Do you use Rust?" style="width:100%"/> + </a> + +Should we consider your answers to be invalid? +---------------------------------------------- + +This was something I thought would be interesting to have, after I saw it +being used in another survey. While it does pinpoint possibly +invalid respondents, I have opted against filtering those out. Mainly because +that would require re-creating each of the charts generated by Google Forms +manually. + +.. raw::html + + <a href="../assets/news/images/survey/reliability.png"> + <img src="../assets/news/images/survey/reliability.png" alt="Should we consider your answers to be invalid?" style="width:100%"/> + </a> + +According to the responses to this question, around 94% of our responses +can be considered reliable. + +Nim users +--------- + +The following questions were answered only by the 38.9% of our respondents +who identified themselves as Nim users. + +How long have you been using Nim? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. raw::html + + <a href="../assets/news/images/survey/nim_time.png"> + <img src="../assets/news/images/survey/nim_time.png" alt="How long have you been using Nim?" style="width:100%"/> + </a> + +A large proportion of our Nim users were new. This is good news as it means that +our community is growing, with a large proportion of new Nim users that could +become long-term Nimians. In total, more than 35% of Nim users can be considered +new having used Nim for less than 3 months. With 18% of Nim users that can +be considered very new having used Nim for less than a month. +This could suggest that 18% of our users have only just found out about Nim in +the last week or so and have not yet got the chance to use it extensively. + +The high percentages of long term Nim users are encouraging. +They suggest +that many users are continuing to use Nim after making it through the first +few months. The sharp drop at 7-9 months is interesting, but may simply be +due to the fact that there were fewer newcomers during that period, or it +could be because our respondents are more likely to estimate that they have +been using Nim for a year or half a year rather than the awkward 7-9 months. + +.. raw::html + + <a href="../assets/news/images/survey/nim_time_rust.png"> + <img src="../assets/news/images/survey/nim_time_rust.png" alt="Time using Nim and Rust" style="width:100%"/> + </a> + +The results for Nim and Rust are actually remarkably similar. They both show a +drop at 7-9 months, although Rust's isn't as dramatic. Nim on the other hand +has a significantly higher percentage of new Nim users. + +Do you use Nim at work? +~~~~~~~~~~~~~~~~~~~~~~~ + +An important aspect of a language's adoption is whether it is being used for +"real" work. We wanted to know how many people are using Nim in their day +jobs and under what circumstances it is used. + +.. raw::html + + <a href="../assets/news/images/survey/nim_at_work.png"> + <img src="../assets/news/images/survey/nim_at_work.png" alt="Do you use Nim at work?" style="width:100%"/> + </a> + +While a vast majority of our users are not using Nim at work, more than 25% +of them are. It's encouraging to see such a high number already, even before +we have released version 1.0. In fact, this percentage is likely close to 30%, +because many of the "Other" responses mention using Nim for the likes of +internal tools or small scripts to help with the respondent's work. + +.. raw::html + + <a href="https://blog.rust-lang.org/images/2016-06-Survey/rust_at_work.png"> + <img src="https://blog.rust-lang.org/images/2016-06-Survey/rust_at_work.png" alt="Do you use Rust at work?" style="width:100%"/> + </a> + +Interestingly, a larger percentage of Nim users are using Nim at work than +Rust users. The sample sizes are of course vastly different, but it's still an +interesting result. Combined, nearly 1/5th of Rust users are using Rust +commercially whereas more than a quarter of Nim users are using Nim +commercially. + +Approximately how large are all the Nim projects that you work on? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Finding out how large the Nim projects worked on by Nim users are is also +very valuable. + +.. raw::html + + <a href="../assets/news/images/survey/project_size.png"> + <img src="../assets/news/images/survey/project_size.png" alt="Nim project size for all users" style="width:100%"/> + </a> + +This shows us that currently Nim is primarily being used for small scripts and +applications, with nearly 60% of the projects consisting of less than 1,000 +lines of code. This makes sense as many of our users are not using Nim +professionally, but are doing so in their spare time. + +.. raw::html + + <a href="../assets/news/images/survey/project_size_work.png"> + <img src="../assets/news/images/survey/project_size_work.png" alt="Nim project size for work users" style="width:100%"/> + </a> + +The numbers for part-time and full-time work users of Nim tell a different +story. Over 70% of the projects written by full-time users are between 10,001 +and 100,000 lines of code. Part-time users show a slightly different trend, +with many more small projects, the majority being between 1,000 and +10,000 lines of code. + +Overall it's good to see that there is a few large projects out there which are +composed of more than 100,000 lines of code. We expect to see the amount of +large projects to grow with time, especially with version 1.0 on the way. + +.. raw::html + + <a href="../assets/news/images/survey/project_size_nim_rust.png"> + <img src="../assets/news/images/survey/project_size_nim_rust.png" alt="Nim project size for work users (Nim vs. Rust)" style="width:100%"/> + </a> + +In comparison to Rust the proportion of project sizes for full-time users is +vastly different. This is likely due to our small sample size. Project sizes for +part-time users between Rust and Nim are somewhat similar, with differences of +around 10% for each project size. + +Do you plan to try to use Nim at work? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. raw::html + + <a href="../assets/news/images/survey/planning_to_use_at_work.png"> + <img src="../assets/news/images/survey/planning_to_use_at_work.png" alt="Planning to use Nim at work?" style="width:100%"/> + </a> + +It's also encouraging to see that over 50% of Nim users are planning to use +Nim at work! This is slightly more than Rust's 40% and should help Nim's +adoption into even more areas. + +Nim and its tools +~~~~~~~~~~~~~~~~~ + +In this section of the survey, we wanted to find out the tools that Nim +users are utilising when developing Nim applications. + +What editor(s) do you use when writing Nim? +___________________________________________ + +Programmers are very specific when it comes to their editor of choice, because +of that it's good to know which editor is most popular among our community. + +.. raw::html + + <a href="../assets/news/images/survey/editors.png"> + <img src="../assets/news/images/survey/editors.png" alt="Editors used by Nim users" style="width:100%"/> + </a> + +Looks like Vim is the winner with almost 30%. Followed by Sublime Text and +Visual Studio Code. Aporia, the Nim IDE, gets a respectable 15.5%. There was +also more than +17% of answers which included "Other" editors, such as: Notepad++, Geany, gedit, +and Kate. + +What operating system(s) do you compile for and run your Nim projects on? +_________________________________________________________________________ + +This question gave us information about the most popular target operating +systems, as well as some of the more obscure ones. We have asked this question +to find out the platforms on which Nim applications run on most frequently. + +.. raw::html + + <a href="../assets/news/images/survey/target_os.png"> + <img src="../assets/news/images/survey/target_os.png" alt="Target operating systems" style="width:100%"/> + </a> + +This question allowed multiple choices, so each percentage is out of the total +number of respondents for this question. For example, 80.7% of the +respondents selected "Linux" but only 26.6% selected OS X. + +This makes Linux by far the most popular target for Nim applications. +Some "Other" targets included: BSD (OpenBSD, FreeBSD), iOS, Android, and +JavaScript. +It's great to see Nim being used on such a wide variety of platforms. + +What operating system(s) do you develop Nim projects on? +________________________________________________________ + +With this question, we wanted to know what operating systems are used for +development. + +.. raw::html + + <a href="../assets/news/images/survey/dev_os.png"> + <img src="../assets/news/images/survey/dev_os.png" alt="Development operating systems" style="width:100%"/> + </a> + +This question also allowed multiple choices and ended up with very similar +results. + +You can see that Linux is also the most popular developmental +platform for Nim. But it's more popular as a target platform. + +Which version(s) of Nim do you use for your applications? +_________________________________________________________ + +.. raw::html + + <a href="../assets/news/images/survey/nim_versions.png"> + <img src="../assets/news/images/survey/nim_versions.png" alt="Version use" style="width:100%"/> + </a> + +At the time of this survey, version 0.14.2 was the latest stable release. +It's no wonder that it is the most commonly used release of Nim. It's good to +see that the older versions are not used as often. The high use of ``Git HEAD (devel)`` +(nightly builds) isn't surprising, Nim is still evolving rapidly and our +release schedule is not regular or frequent. + +Once we go past the 1.0 release, we expect to see much less use of the unstable +``devel`` branch. + + + + + diff --git a/web/news/version_0_15_released.rst b/web/news/version_0_15_released.rst new file mode 100644 index 000000000..1e35fb627 --- /dev/null +++ b/web/news/version_0_15_released.rst @@ -0,0 +1,108 @@ +Version 0.15.0 released +======================= + +.. container:: metadata + + Posted by Dominik Picheta on 17/09/2016 + +Some text here. + +Changes affecting backwards compatibility +----------------------------------------- + +- De-deprecated ``re.nim`` because we have too much code using it + and it got the basic API right. + +- ``split`` with ``set[char]`` as a delimiter in ``strutils.nim`` + no longer strips and splits characters out of the target string + by the entire set of characters. Instead, it now behaves in a + similar fashion to ``split`` with ``string`` and ``char`` + delimiters. Use ``splitWhitespace`` to get the old behaviour. +- The command invocation syntax will soon apply to open brackets + and curlies too. This means that code like ``a [i]`` will be + interpreted as ``a([i])`` and not as ``a[i]`` anymore. Likewise + ``f (a, b)`` means that the tuple ``(a, b)`` is passed to ``f``. + The compiler produces a warning for ``a [i]``:: + + Warning: a [b] will be parsed as command syntax; spacing is deprecated + + See `<https://github.com/nim-lang/Nim/issues/3898>`_ for the relevant + discussion. +- Overloading the special operators ``.``, ``.()``, ``.=``, ``()`` now + should be enabled via ``{.experimental.}``. +- ``immediate`` templates and macros are now deprecated. + Instead use ``untyped`` parameters. +- The metatype ``expr`` is deprecated. Use ``untyped`` instead. +- The metatype ``stmt`` is deprecated. Use ``typed`` instead. +- The compiler is now more picky when it comes to ``tuple`` types. The + following code used to compile, now it's rejected: + +.. code-block:: nim + + import tables + var rocketaims = initOrderedTable[string, Table[tuple[k: int8, v: int8], int64] ]() + rocketaims["hi"] = {(-1.int8, 0.int8): 0.int64}.toTable() + +Instead be consistent in your tuple usage and use tuple names for tuples +that have tuple name: + +.. code-block:: nim + + import tables + var rocketaims = initOrderedTable[string, Table[tuple[k: int8, v: int8], int64] ]() + rocketaims["hi"] = {(k: -1.int8, v: 0.int8): 0.int64}.toTable() + +- Now when you compile console application for Windows, console output + encoding is automatically set to UTF-8. + +Library Additions +----------------- + +- Added ``readHeaderRow`` and ``rowEntry`` to ``parsecsv.nim`` to provide + a lightweight alternative to python's ``csv.DictReader``. +- Added ``setStdIoUnbuffered`` proc to ``system.nim`` to enable unbuffered I/O. + +- Added ``center`` and ``rsplit`` to ``strutils.nim`` to + provide similar Python functionality for Nim's strings. + +- Added ``isTitle``, ``title``, ``swapCase``, ``isUpper``, ``toUpper``, + ``isLower``, ``toLower``, ``isAlpha``, ``isSpace``, and ``capitalize`` + to ``unicode.nim`` to provide unicode aware case manipulation and case + testing. + +- Added a new module ``lib/pure/strmisc.nim`` to hold uncommon string + operations. Currently contains ``partition``, ``rpartition`` + and ``expandTabs``. + +- Split out ``walkFiles`` in ``os.nim`` to three separate procs in order + to make a clear distinction of functionality. ``walkPattern`` iterates + over both files and directories, while ``walkFiles`` now only iterates + over files and ``walkDirs`` only iterates over directories. + +Compiler Additions +------------------ + +- The ``-d/--define`` flag can now optionally take a value to be used + by code at compile time. + +Nimscript Additions +------------------- + +- Finally it's possible to dis/enable specific hints and warnings in + Nimscript via the procs ``warning`` and ``hint``. +- Nimscript exports a proc named ``patchFile`` which can be used to + patch modules or include files for different Nimble packages, including + the ``stdlib`` package. + + +Language Additions +------------------ + +- Added ``{.intdefine.}`` and ``{.strdefine.}`` macros to make use of + (optional) compile time defines. +- If the first statement is an ``import system`` statement then ``system`` + is not imported implicitly anymore. This allows for code like + ``import system except echo`` or ``from system import nil``. + +Bugfixes +-------- diff --git a/web/question.txt b/web/question.rst index b3a357bb8..9de178c90 100644 --- a/web/question.txt +++ b/web/question.rst @@ -126,6 +126,7 @@ General FAQ -------------------------- - Nim IDE: https://github.com/nim-lang/Aporia + - Visual Studio Code: https://marketplace.visualstudio.com/items?itemName=kosz78.nim - Emacs: https://github.com/nim-lang/nim-mode - Vim: https://github.com/zah/nimrod.vim/ - Scite: Included @@ -136,7 +137,7 @@ General FAQ - LiClipse: http://www.liclipse.com/ (Eclipse based plugin) - Howl: Included - Notepad++: Available via `plugin <https://github.com/jangko/nppnim/releases>`_ - + .. container:: standout diff --git a/web/sponsors.csv b/web/sponsors.csv new file mode 100644 index 000000000..ac35b284c --- /dev/null +++ b/web/sponsors.csv @@ -0,0 +1,31 @@ +logo, name, url, this_month, all_time, since, level +assets/bountysource/secondspectrum.png,Second Spectrum,http://www.secondspectrum.com/,250,750,"May 5, 2016",250 +assets/bountysource/xored.svg,"Xored Software, Inc.",http://xored.com/,250,500,"Jun 20, 2016",250 +,avsej,http://avsej.net,75,85,"Jun 10, 2016",75 +,shkolnick-kun,https://github.com/shkolnick-kun,75,75,"Jul 6, 2016",75 +,flyx,http://flyx.org,35,140,"Apr 7, 2016",75 +,endragor,https://github.com/endragor,25,100,"Apr 7, 2016",25 +,euantorano,http://euantorano.co.uk,25,50,"Jun 7, 2016",25 +,FedericoCeratto,http://firelet.net,25,100,"Apr 7, 2016",25 +,"Adrian Veith",,25,100,"Apr 20, 2016",25 +,xxlabaza,https://github.com/xxlabaza,25,45,"Jun 17, 2016",25 +,"Yuriy Glukhov",,25,100,"Apr 6, 2016",25 +,"Jonathan Arnett",,10,30,"May 20, 2016",10 +,"Oskari Timperi",,10,20,"Jun 8, 2016",10 +,zachaysan,http://venn.lc,10,20,"Jun 7, 2016",10 +,"Matthew Baulch",,10,20,"Jun 7, 2016",10 +,RationalG,https://github.com/RationalG,10,20,"Jun 17, 2016",10 +,btbytes,https://www.btbytes.com/,10,40,"Apr 6, 2016",10 +,niebaopeng,https://github.com/niebaopeng,10,30,"Apr 15, 2016",10 +,moigagoo,http://sloth-ci.com,5,15,"May 13, 2016",5 +,calind,http://calindon.net,5,10,"Jun 7, 2016",5 +,swalf,https://github.com/swalf,5,35,"May 9, 2016",5 +,johnnovak,http://www.johnnovak.net/,5,20,"Apr 29, 2016",5 +,RyanMarcus,http://rmarcus.info,5,5,"Jul 19, 2016",5 +,Blumenversand,https://blumenversender.com/,5,5,"Jul 21, 2016",5 +,lenzenmi,https://github.com/lenzenmi,5,5,"Jul 28, 2016",5 +,"Handojo Goenadi",,5,20,"Apr 19, 2016",5 +,"Date in Asia",,5,5,"Jul 30, 2016",5 +,"Matthew Newton",,5,20,"Apr 20, 2016",5 +,"Michael D. Sklaroff",,1,4,"Apr 27, 2016",1 +,"Svend Knudsen",,1,4,"Apr 11, 2016",1 diff --git a/web/support.txt b/web/support.rst index 9a526605e..9a526605e 100644 --- a/web/support.txt +++ b/web/support.rst diff --git a/web/ticker.html b/web/ticker.html new file mode 100644 index 000000000..ecbfb5e3f --- /dev/null +++ b/web/ticker.html @@ -0,0 +1,25 @@ +<a class="news" href="$1news/2016_08_06_bountysource_update_the_road_to_v10.html"> + <h4>August 6, 2016</h4> + <p>BountySource Update: The Road to v1.0</p> +</a> + +<a class="news" href="$1news/2016_06_23_launching_the_2016_nim_community_survey.html"> + <h4>June 23, 2016</h4> + <p>Launching the 2016 Nim community survey!</p> +</a> + +<a class="news" href="$1news/2016_06_11_version_0_14_2_released.html"> + <h4>June 11, 2016</h4> + <p>Nim version 0.14.2 has been released!</p> +</a> + +<a class="news" href="$1news/2016_06_04_meet_our_bountysource_sponsors.html"> + <h4>June 04, 2016</h4> + <p>Meet our BountySource sponsors</p> +</a> + +<a class="news" href="$1news/2016_01_27_nim_in_action_is_now_available.html"> + <h4>January 27, 2016</h4> + <p>Nim in Action is now available!</p> +</a> +<a href="$1news.html" class="blue">See All News...</a> diff --git a/web/ticker.txt b/web/ticker.txt deleted file mode 100644 index 7e917e0f2..000000000 --- a/web/ticker.txt +++ /dev/null @@ -1,26 +0,0 @@ -<a class="news" href="news.html#Z2016-01-27-nim-in-action-is-now-available"> - <h4>January 27, 2016</h4> - <p>Nim in Action is now available!</p> -</a> - -<a class="news" href="news.html#Z2016-01-18-version-0-13-0-released"> - <h4>January 18, 2016</h4> - <p>Nim version 0.13.0 has been released!</p> -</a> - -<a class="news" href="news.html#Z2016-01-18-andreas-rumpf-s-talk-at-oscon-amsterdam"> - <h4>January 18, 2016</h4> - <p>Andreas Rumpf's talk at OSCON Amsterdam</p> -</a> - -<a class="news" href="news.html#Z2015-10-27-version-0-12-0-released"> - <h4>October 27, 2015</h4> - <p>Nim version 0.12.0 has been released!</p> -</a> - -<a class="news" href="news.html#Z2015-10-16-first-nim-conference"> - <h4>October 16, 2015</h4> - <p>First official Nim conference will be in Kyiv!</p> -</a> - -<a href="news.html" class="blue">See All News...</a> diff --git a/web/website.ini b/web/website.ini index d1f8a04bf..860ab9338 100644 --- a/web/website.ini +++ b/web/website.ini @@ -26,24 +26,24 @@ community: community news: news [Ticker] -file: ticker.txt +file: ticker.html [Documentation] -doc: "endb;intern;apis;lib;manual.txt;tut1;tut2;nimc;overview;filters" -doc: "tools;niminst;nimgrep;gc;estp;idetools;docgen;koch;backends.txt" -doc: "nimfix.txt;nimsuggest.txt;nep1.txt;nims.txt" -pdf: "manual.txt;lib;tut1;tut2;nimc;niminst;gc" +doc: "endb;intern;apis;lib;manual.rst;tut1.rst;tut2.rst;nimc;overview;filters" +doc: "tools;niminst;nimgrep;gc;estp;idetools;docgen;koch;backends.rst" +doc: "nimfix.rst;nimsuggest.rst;nep1.rst;nims.rst" +pdf: "manual.rst;lib.rst;tut1.rst;tut2.rst;nimc.rst;niminst.rst;gc.rst" srcdoc2: "system.nim;system/nimscript;pure/ospaths" srcdoc2: "core/macros;pure/marshal;core/typeinfo" -srcdoc2: "impure/re;pure/typetraits" +srcdoc2: "impure/re;impure/nre;pure/typetraits" srcdoc2: "pure/concurrency/threadpool.nim;pure/concurrency/cpuinfo.nim" srcdoc: "system/threads.nim;system/channels.nim;js/dom" srcdoc2: "pure/os;pure/strutils;pure/math;pure/matchers;pure/algorithm" -srcdoc2: "pure/stats;impure/nre;windows/winlean" -srcdoc2: "pure/complex;pure/times;pure/osproc;pure/pegs;pure/dynlib" +srcdoc2: "pure/stats;impure/nre;windows/winlean;pure/random" +srcdoc2: "pure/complex;pure/times;pure/osproc;pure/pegs;pure/dynlib;pure/strscans" srcdoc2: "pure/parseopt;pure/parseopt2;pure/hashes;pure/strtabs;pure/lexbase" srcdoc2: "pure/parsecfg;pure/parsexml;pure/parsecsv;pure/parsesql" -srcdoc2: "pure/streams;pure/terminal;pure/cgi;pure/unicode" +srcdoc2: "pure/streams;pure/terminal;pure/cgi;pure/unicode;pure/strmisc" srcdoc2: "pure/htmlgen;pure/parseutils;pure/browsers" srcdoc2: "impure/db_postgres;impure/db_mysql;impure/db_sqlite" srcdoc2: "pure/httpserver;pure/httpclient;pure/smtp;impure/ssl;pure/fsmonitor" @@ -64,7 +64,7 @@ srcdoc2: "pure/asyncfile;pure/asyncftpclient" srcdoc2: "pure/md5;pure/rationals" srcdoc2: "posix/posix" srcdoc2: "pure/fenv;pure/securehash" -srcdoc2: "pure/basic2d;pure/basic3d;pure/mersenne;pure/coro" +srcdoc2: "pure/basic2d;pure/basic3d;pure/mersenne;pure/coro;pure/httpcore" ; Note: everything under 'webdoc' doesn't get listed in the index, so wrappers ; should live here |
